@zerwiz/ymir 0.1.4 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/README.md +291 -0
- package/.agents/agents/bragi-marketer.md +111 -0
- package/.agents/agents/brokk.md +352 -0
- package/.agents/agents/forseti-reviewer.md +93 -0
- package/.agents/agents/frigg-consent.md +77 -0
- package/.agents/agents/groa-update.md +77 -0
- package/.agents/agents/hnoss-designer.md +69 -0
- package/.agents/agents/huginn-researcher.md +99 -0
- package/.agents/agents/jord-projects.md +78 -0
- package/.agents/agents/kvasir-scout.md +91 -0
- package/.agents/agents/mimir-planner.md +92 -0
- package/.agents/agents/muninn-stow.md +77 -0
- package/.agents/agents/saga-bearings.md +78 -0
- package/.agents/agents/sindri-developer.md +114 -0
- package/.agents/agents/snotra-documenter.md +105 -0
- package/.agents/agents/syn-recovery.md +78 -0
- package/.agents/agents/tyr-check.md +78 -0
- package/.agents/agents/urdh-hold.md +77 -0
- package/.agents/agents/volundr-smith.md +80 -0
- package/.agents/agents/vor-diagnostics.md +77 -0
- package/.agents/assets/agents/naming.md +50 -0
- package/.agents/assets/agents/registry.md +93 -0
- package/.agents/assets/agents/runtime.md +36 -0
- package/.agents/assets/agents/toon-tasks-cli.md +42 -0
- package/.agents/assets/templates/company_entity.template.md +37 -0
- package/.agents/backend/README.md +74 -0
- package/.agents/backend/backends/cmux.sh +649 -0
- package/.agents/backend/backends/herdr-eventwait.py +157 -0
- package/.agents/backend/backends/herdr-workspace-move.py +114 -0
- package/.agents/backend/backends/herdr.sh +3386 -0
- package/.agents/backend/backends/orca.sh +290 -0
- package/.agents/backend/backends/tmux.sh +357 -0
- package/.agents/backend/backends/zellij.sh +700 -0
- package/.agents/backend/fm-afk-launch.sh +654 -0
- package/.agents/backend/fm-afk-return.sh +241 -0
- package/.agents/backend/fm-afk-start.sh +172 -0
- package/.agents/backend/fm-arm-command-policy.mjs +960 -0
- package/.agents/backend/fm-arm-pretool-check.sh +200 -0
- package/.agents/backend/fm-backend-hometag-lib.sh +52 -0
- package/.agents/backend/fm-backend.sh +981 -0
- package/.agents/backend/fm-backlog-handoff.sh +909 -0
- package/.agents/backend/fm-backlog-receive.sh +187 -0
- package/.agents/backend/fm-backlog-transition-lib.sh +779 -0
- package/.agents/backend/fm-bearings-board.sh +204 -0
- package/.agents/backend/fm-bearings-snapshot.sh +546 -0
- package/.agents/backend/fm-bootstrap.sh +1568 -0
- package/.agents/backend/fm-branch-outcome.sh +203 -0
- package/.agents/backend/fm-branch-prompt.sh +101 -0
- package/.agents/backend/fm-brief.sh +452 -0
- package/.agents/backend/fm-busy-event.sh +234 -0
- package/.agents/backend/fm-busy-lib.sh +986 -0
- package/.agents/backend/fm-captain-hold.sh +1001 -0
- package/.agents/backend/fm-cd-command-policy.mjs +153 -0
- package/.agents/backend/fm-cd-pretool-check.sh +190 -0
- package/.agents/backend/fm-check-lib.sh +74 -0
- package/.agents/backend/fm-check-register.sh +42 -0
- package/.agents/backend/fm-check-unregister.sh +52 -0
- package/.agents/backend/fm-classify-lib.sh +1770 -0
- package/.agents/backend/fm-claude-stop-autoarm.sh +334 -0
- package/.agents/backend/fm-composer-lib.sh +1416 -0
- package/.agents/backend/fm-config-inherit-lib.sh +1202 -0
- package/.agents/backend/fm-config-push.sh +254 -0
- package/.agents/backend/fm-control-lib.sh +251 -0
- package/.agents/backend/fm-control.sh +878 -0
- package/.agents/backend/fm-crew-state.sh +625 -0
- package/.agents/backend/fm-cursor-lib.sh +243 -0
- package/.agents/backend/fm-decision-hold.sh +232 -0
- package/.agents/backend/fm-doc-audience-check.sh +269 -0
- package/.agents/backend/fm-dod-lib.sh +67 -0
- package/.agents/backend/fm-ensure-agents-md.sh +253 -0
- package/.agents/backend/fm-extension-launch-barrier.mjs +129 -0
- package/.agents/backend/fm-extension.mjs +2577 -0
- package/.agents/backend/fm-extension.sh +16 -0
- package/.agents/backend/fm-ff-lib.sh +420 -0
- package/.agents/backend/fm-fleet-snapshot.sh +1463 -0
- package/.agents/backend/fm-fleet-sync.sh +458 -0
- package/.agents/backend/fm-fleet-view.sh +96 -0
- package/.agents/backend/fm-gate-refuse-lib.sh +102 -0
- package/.agents/backend/fm-guard.sh +242 -0
- package/.agents/backend/fm-harness.sh +195 -0
- package/.agents/backend/fm-herdr-ci-cleanup.sh +115 -0
- package/.agents/backend/fm-herdr-lab.sh +346 -0
- package/.agents/backend/fm-herdr-session-cleanup.sh +337 -0
- package/.agents/backend/fm-home-seed.sh +984 -0
- package/.agents/backend/fm-home-summary-refresh.sh +255 -0
- package/.agents/backend/fm-hook-host-lib.sh +36 -0
- package/.agents/backend/fm-inactive-reconcile.sh +531 -0
- package/.agents/backend/fm-inbox.sh +399 -0
- package/.agents/backend/fm-install-actionlint.sh +84 -0
- package/.agents/backend/fm-install-herdr.sh +96 -0
- package/.agents/backend/fm-install-shellcheck.sh +84 -0
- package/.agents/backend/fm-install-treehouse.sh +95 -0
- package/.agents/backend/fm-kimi-turnend-hook.sh +276 -0
- package/.agents/backend/fm-lease-lib.sh +218 -0
- package/.agents/backend/fm-lease.sh +189 -0
- package/.agents/backend/fm-line-cap-lib.sh +51 -0
- package/.agents/backend/fm-lint-workflows.sh +137 -0
- package/.agents/backend/fm-lint.sh +591 -0
- package/.agents/backend/fm-lock-lib.sh +104 -0
- package/.agents/backend/fm-lock.sh +107 -0
- package/.agents/backend/fm-marker-lib.sh +12 -0
- package/.agents/backend/fm-merge-local.sh +74 -0
- package/.agents/backend/fm-merge-outcome-lib.sh +138 -0
- package/.agents/backend/fm-nm-run-lib.sh +124 -0
- package/.agents/backend/fm-on.sh +125 -0
- package/.agents/backend/fm-operational-input.sh +253 -0
- package/.agents/backend/fm-peek.sh +46 -0
- package/.agents/backend/fm-pending-reply-lib.sh +1409 -0
- package/.agents/backend/fm-pr-check.sh +135 -0
- package/.agents/backend/fm-pr-lib.sh +1016 -0
- package/.agents/backend/fm-pr-merge.sh +707 -0
- package/.agents/backend/fm-pr-poll.sh +110 -0
- package/.agents/backend/fm-primary-scope-lib.sh +33 -0
- package/.agents/backend/fm-procevent-extension-capture.pl +259 -0
- package/.agents/backend/fm-procevent-lavish.sh +628 -0
- package/.agents/backend/fm-procevent-lib.sh +892 -0
- package/.agents/backend/fm-procevent-quota.sh +290 -0
- package/.agents/backend/fm-procevent-remote-reply.sh +577 -0
- package/.agents/backend/fm-procevent-when.sh +504 -0
- package/.agents/backend/fm-procevent.sh +1466 -0
- package/.agents/backend/fm-project-mode.sh +92 -0
- package/.agents/backend/fm-project-origin-lib.sh +180 -0
- package/.agents/backend/fm-promote.sh +282 -0
- package/.agents/backend/fm-public-followup-emit.sh +260 -0
- package/.agents/backend/fm-public-followup-lib.sh +398 -0
- package/.agents/backend/fm-public-followup.sh +1330 -0
- package/.agents/backend/fm-push-transition-lib.sh +168 -0
- package/.agents/backend/fm-quota-axi-lib.sh +93 -0
- package/.agents/backend/fm-quota-choose.sh +384 -0
- package/.agents/backend/fm-remote-delta-read.sh +189 -0
- package/.agents/backend/fm-remote-doctor.sh +800 -0
- package/.agents/backend/fm-remote-entrypoint.sh +193 -0
- package/.agents/backend/fm-remote-file.sh +234 -0
- package/.agents/backend/fm-remote-home-provision.sh +259 -0
- package/.agents/backend/fm-remote-home-seed.sh +259 -0
- package/.agents/backend/fm-remote-inherit-push.sh +90 -0
- package/.agents/backend/fm-remote-inherit.sh +179 -0
- package/.agents/backend/fm-remote-job-lib.sh +1241 -0
- package/.agents/backend/fm-remote-job-reap-orphans.sh +137 -0
- package/.agents/backend/fm-remote-job-worker.sh +1124 -0
- package/.agents/backend/fm-remote-readiness-lib.sh +44 -0
- package/.agents/backend/fm-remote-secondmate-control.sh +340 -0
- package/.agents/backend/fm-review-diff.sh +158 -0
- package/.agents/backend/fm-secondmate-charter-lib.sh +44 -0
- package/.agents/backend/fm-secondmate-nudge-lib.sh +68 -0
- package/.agents/backend/fm-secondmate-parent-lib.sh +70 -0
- package/.agents/backend/fm-secondmate-reconcile.sh +371 -0
- package/.agents/backend/fm-secondmate-registry-lib.sh +311 -0
- package/.agents/backend/fm-secondmate-report.sh +88 -0
- package/.agents/backend/fm-send.sh +1059 -0
- package/.agents/backend/fm-session-lock-lib.sh +176 -0
- package/.agents/backend/fm-session-start.sh +970 -0
- package/.agents/backend/fm-sessionstart-cursor.sh +40 -0
- package/.agents/backend/fm-sessionstart-nudge.sh +45 -0
- package/.agents/backend/fm-sessionstart-run.sh +145 -0
- package/.agents/backend/fm-spawn.sh +3153 -0
- package/.agents/backend/fm-startup-memory-budget-lib.sh +224 -0
- package/.agents/backend/fm-startup-memory-budget.sh +94 -0
- package/.agents/backend/fm-startup-network.sh +642 -0
- package/.agents/backend/fm-stow-cascade.sh +251 -0
- package/.agents/backend/fm-subagent-pretool-check.sh +207 -0
- package/.agents/backend/fm-supervise-daemon.sh +1732 -0
- package/.agents/backend/fm-supervision-instructions.sh +215 -0
- package/.agents/backend/fm-supervision-lib.sh +90 -0
- package/.agents/backend/fm-supervisor-target-lib.sh +78 -0
- package/.agents/backend/fm-tangle-lib.sh +53 -0
- package/.agents/backend/fm-task-inbox-lib.sh +402 -0
- package/.agents/backend/fm-tasks-axi-lib.sh +123 -0
- package/.agents/backend/fm-teardown.sh +2922 -0
- package/.agents/backend/fm-test-isolation-proof.sh +588 -0
- package/.agents/backend/fm-test-run.sh +2191 -0
- package/.agents/backend/fm-timeout-lib.sh +141 -0
- package/.agents/backend/fm-timing-lib.sh +179 -0
- package/.agents/backend/fm-tmux-lib.sh +291 -0
- package/.agents/backend/fm-tool-update-check.sh +898 -0
- package/.agents/backend/fm-trace-context-lib.sh +227 -0
- package/.agents/backend/fm-transition-lib.sh +103 -0
- package/.agents/backend/fm-turnend-guard-cursor.sh +391 -0
- package/.agents/backend/fm-turnend-guard-grok.sh +90 -0
- package/.agents/backend/fm-turnend-guard.sh +432 -0
- package/.agents/backend/fm-update.sh +110 -0
- package/.agents/backend/fm-vendor-auth-probe.sh +190 -0
- package/.agents/backend/fm-voice-client.py +1373 -0
- package/.agents/backend/fm-voice-relay.py +1256 -0
- package/.agents/backend/fm-wake-drain.sh +592 -0
- package/.agents/backend/fm-wake-grant.sh +127 -0
- package/.agents/backend/fm-wake-lib.sh +1849 -0
- package/.agents/backend/fm-watch-arm.sh +602 -0
- package/.agents/backend/fm-watch-checkpoint.sh +109 -0
- package/.agents/backend/fm-watch.sh +1962 -0
- package/.agents/backend/fm-x-dismiss.sh +116 -0
- package/.agents/backend/fm-x-followup.sh +286 -0
- package/.agents/backend/fm-x-lib.sh +1000 -0
- package/.agents/backend/fm-x-link.sh +234 -0
- package/.agents/backend/fm-x-poll.sh +185 -0
- package/.agents/backend/fm-x-reply.sh +385 -0
- package/.agents/backend/fm_voice_frame.py +166 -0
- package/.agents/backend/fm_voice_records.py +574 -0
- package/.agents/backend/model-bridge.py +360 -0
- package/.agents/backend/opencode-go-bridge.py +30 -0
- package/.agents/bus/README.md +12 -0
- package/.agents/bus/messages.json +1 -0
- package/.agents/config/agents.machine.example.yaml +16 -0
- package/.agents/config/agents.yaml.example +72 -0
- package/.agents/config/crew-dispatch.json +22 -0
- package/.agents/config/cron.yaml +11 -0
- package/.agents/config/eindri-dispatch.json +67 -0
- package/.agents/config/eindri-harness +1 -0
- package/.agents/config/startup-memory-budget +1 -0
- package/.agents/filebrowser/.filebrowser.json +6 -0
- package/.agents/filebrowser/README.md +11 -0
- package/.agents/gateway/README.md +12 -0
- package/.agents/github/README.md +15 -0
- package/.agents/github/webhooks/.gitkeep +0 -0
- package/.agents/github/workflows/.gitkeep +0 -0
- package/.agents/harness/opencode/plugins/lib/rodd-operational-input.js +37 -0
- package/.agents/harness/opencode/plugins/package.json +4 -0
- package/.agents/harness/opencode/plugins/saga-sessionstart.js +77 -0
- package/.agents/harness/opencode/plugins/syn-cd-check.js +61 -0
- package/.agents/harness/opencode/plugins/syn-pretool-check.js +68 -0
- package/.agents/harness/opencode/plugins/syn-turnend-guard.js +105 -0
- package/.agents/harness/opencode/plugins/syn-watch-arm.js +521 -0
- package/.agents/memory/README.md +46 -0
- package/.agents/memory/runes_audit.md +5 -0
- package/.agents/migrations/0001-hodd-layout.sh +38 -0
- package/.agents/migrations/0002-a2a-mcp.sh +27 -0
- package/.agents/migrations/0003-private-data-separation.sh +121 -0
- package/.agents/migrations/0004-hoard-and-realms.sh +104 -0
- package/.agents/migrations/0005-roots-out-of-tree.sh +158 -0
- package/.agents/sandbox/Dockerfile.utgard +19 -0
- package/.agents/sandbox/README.md +14 -0
- package/.agents/sandbox/utgard.config.json +14 -0
- package/.agents/skills/README.md +59 -0
- package/.agents/skills/bragi-marketing/README.md +3 -0
- package/.agents/skills/bragi-marketing/SKILL.md +82 -0
- package/.agents/skills/eindri-homes/README.md +3 -0
- package/.agents/skills/eindri-homes/SKILL.md +291 -0
- package/.agents/skills/eindri-homes/assets/control-plane.md +122 -0
- package/.agents/skills/eindri-homes/assets/dispatch.md +80 -0
- package/.agents/skills/frigg-consent/README.md +3 -0
- package/.agents/skills/frigg-consent/SKILL.md +56 -0
- package/.agents/skills/galdr-ymirsystem/README.md +3 -0
- package/.agents/skills/galdr-ymirsystem/SKILL.md +159 -0
- package/.agents/skills/galdr-ymirsystem/assets/README.md +74 -0
- package/.agents/skills/galdr-ymirsystem/assets/brokk-distro-runtime.md +390 -0
- package/.agents/skills/galdr-ymirsystem/assets/build-method.md +113 -0
- package/.agents/skills/galdr-ymirsystem/assets/build-tool-categories.md +178 -0
- package/.agents/skills/galdr-ymirsystem/assets/eindri-orchestration.md +518 -0
- package/.agents/skills/galdr-ymirsystem/assets/eindri-profiles.md +136 -0
- package/.agents/skills/galdr-ymirsystem/assets/harness-integration/README.md +736 -0
- package/.agents/skills/galdr-ymirsystem/assets/harness-integration/claude-code.md +197 -0
- package/.agents/skills/galdr-ymirsystem/assets/harness-integration/codex.md +191 -0
- package/.agents/skills/galdr-ymirsystem/assets/harness-integration/cursor.md +193 -0
- package/.agents/skills/galdr-ymirsystem/assets/harness-integration/opencode.md +262 -0
- package/.agents/skills/galdr-ymirsystem/assets/harness-integration/pi.md +295 -0
- package/.agents/skills/galdr-ymirsystem/assets/hlidskjalf-ui.md +539 -0
- package/.agents/skills/galdr-ymirsystem/assets/installation.md +763 -0
- package/.agents/skills/galdr-ymirsystem/assets/local-models/HOST-DATA.example.md +72 -0
- package/.agents/skills/galdr-ymirsystem/assets/local-models.md +274 -0
- package/.agents/skills/galdr-ymirsystem/assets/memory-well.md +163 -0
- package/.agents/skills/galdr-ymirsystem/assets/nornir-jobs.md +415 -0
- package/.agents/skills/galdr-ymirsystem/assets/norse-naming.md +325 -0
- package/.agents/skills/galdr-ymirsystem/assets/odrerir-hall.md +133 -0
- package/.agents/skills/galdr-ymirsystem/assets/pi-boot/einherjar-spawn.schema.json +81 -0
- package/.agents/skills/galdr-ymirsystem/assets/pi-boot/herdr-profile.toml +75 -0
- package/.agents/skills/galdr-ymirsystem/assets/pi-boot/pi-profile.yml +101 -0
- package/.agents/skills/galdr-ymirsystem/assets/pi-boot/supervision-tree.yml +163 -0
- package/.agents/skills/galdr-ymirsystem/assets/pi-boot-guide.md +58 -0
- package/.agents/skills/galdr-ymirsystem/assets/porting-upstream-to-norse.md +301 -0
- package/.agents/skills/galdr-ymirsystem/assets/principles.md +385 -0
- package/.agents/skills/galdr-ymirsystem/assets/reference-adoption.md +99 -0
- package/.agents/skills/galdr-ymirsystem/assets/registry.md +379 -0
- package/.agents/skills/galdr-ymirsystem/assets/runtime-compliance.md +463 -0
- package/.agents/skills/galdr-ymirsystem/assets/runtime-components.md +553 -0
- package/.agents/skills/galdr-ymirsystem/assets/smidja.md +180 -0
- package/.agents/skills/galdr-ymirsystem/schemas/toon-schemas.md +190 -0
- package/.agents/skills/galdr-ymirsystem/scripts/README.md +45 -0
- package/.agents/skills/galdr-ymirsystem/scripts/bench-one.sh +248 -0
- package/.agents/skills/galdr-ymirsystem/scripts/compliance-check.sh +503 -0
- package/.agents/skills/galdr-ymirsystem/scripts/gpu-sample.sh +43 -0
- package/.agents/skills/galdr-ymirsystem/scripts/stop-all.sh +34 -0
- package/.agents/skills/galdr-ymirsystem/scripts/toon-check.py +188 -0
- package/.agents/skills/gjallarhorn-relay/README.md +3 -0
- package/.agents/skills/gjallarhorn-relay/SKILL.md +283 -0
- package/.agents/skills/groa-update/SKILL.md +90 -0
- package/.agents/skills/hamr-adapters/README.md +3 -0
- package/.agents/skills/hamr-adapters/SKILL.md +123 -0
- package/.agents/skills/hamr-adapters/references/common/control-and-recovery.md +37 -0
- package/.agents/skills/hamr-adapters/references/common/dispatch.md +32 -0
- package/.agents/skills/hamr-adapters/references/common/model-and-effort.md +42 -0
- package/.agents/skills/hamr-adapters/references/common/primary-hooks.md +40 -0
- package/.agents/skills/hamr-adapters/references/harness/claude.md +55 -0
- package/.agents/skills/hamr-adapters/references/harness/codex.md +43 -0
- package/.agents/skills/hamr-adapters/references/harness/cursor.md +75 -0
- package/.agents/skills/hamr-adapters/references/harness/grok.md +69 -0
- package/.agents/skills/hamr-adapters/references/harness/kimi.md +51 -0
- package/.agents/skills/hamr-adapters/references/harness/muse.md +70 -0
- package/.agents/skills/hamr-adapters/references/harness/opencode.md +42 -0
- package/.agents/skills/hamr-adapters/references/harness/pi.md +56 -0
- package/.agents/skills/herdr-panes/README.md +3 -0
- package/.agents/skills/herdr-panes/SKILL.md +92 -0
- package/.agents/skills/herdr-panes/assets/herdr-backend.md +346 -0
- package/.agents/skills/herdr-panes/assets/tmux-backend.md +115 -0
- package/.agents/skills/hnoss-design/DESIGN.md +19 -0
- package/.agents/skills/hnoss-design/README.md +3 -0
- package/.agents/skills/hnoss-design/SKILL.md +83 -0
- package/.agents/skills/hvild-afk/README.md +3 -0
- package/.agents/skills/hvild-afk/SKILL.md +241 -0
- package/.agents/skills/jord-projects/README.md +3 -0
- package/.agents/skills/jord-projects/SKILL.md +92 -0
- package/.agents/skills/lifecycle/SKILL.md +56 -0
- package/.agents/skills/lifecycle/smoke_test.sh +57 -0
- package/.agents/skills/lifecycle/start.sh +21 -0
- package/.agents/skills/lifecycle/status.sh +35 -0
- package/.agents/skills/lifecycle/stop.sh +16 -0
- package/.agents/skills/muninn-stow/README.md +3 -0
- package/.agents/skills/muninn-stow/SKILL.md +309 -0
- package/.agents/skills/no-mistakes/SKILL.md +379 -0
- package/.agents/skills/nornir-schedule/README.md +3 -0
- package/.agents/skills/nornir-schedule/SKILL.md +25 -0
- package/.agents/skills/nornir-schedule/assets/events.md +137 -0
- package/.agents/skills/nornir-schedule/assets/quota.md +131 -0
- package/.agents/skills/nsr-compliance/README.md +3 -0
- package/.agents/skills/nsr-compliance/SKILL.md +29 -0
- package/.agents/skills/nsr-compliance/assets/nsr/README.md +40 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/RULES/README.md +15 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/RULES/git-workflow.md +18 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/RULES/security.md +22 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/RULES/tech-stack.md +8 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/SKILL.md +71 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/config/core_four.yaml +16 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/gates/check_danger.sh +102 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/gates/check_wiring.sh +69 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/gates/validate_code.sh +29 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/gates/verify_docs.py +66 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/generate.sh +109 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/harness/envelopes/result_envelope.json +14 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/harness/envelopes/task_envelope.json +20 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/harness/runner.py +303 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/telemetry/logger.py +112 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/templates/compliance/README.md +18 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/templates/compliance/config/README.md +10 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/templates/compliance/gates/README.md +12 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/templates/compliance/gates/check_env.sh +23 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/templates/compliance/gates/check_paths.sh +20 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/templates/compliance/gates/check_platform.sh +23 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/templates/compliance/harness/README.md +11 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/templates/compliance/harness/envelopes/README.md +10 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/templates/compliance/installer/README.md +10 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/templates/compliance/installer/install.sh +125 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/templates/compliance/telemetry/README.md +11 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/README.md +20 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/features/README.md +36 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/features/SKILL-template.md +14 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/git_ops/README.md +25 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/git_ops/SKILL.md +13 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/git_ops/create_branch.sh +6 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/git_ops/safe_commit.sh +9 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/git_ops/sync_upstream.sh +6 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/lifecycle/README.md +27 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/lifecycle/SKILL.md +15 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/lifecycle/smoke_test.sh +12 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/lifecycle/start.sh +19 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/lifecycle/status.sh +12 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/lifecycle/stop.sh +15 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/README.md +18 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/config/README.md +10 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/config/core_four.yaml +13 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/gates/README.md +12 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/gates/check_danger.sh +102 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/gates/check_env.sh +23 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/gates/check_paths.sh +20 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/gates/check_platform.sh +23 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/gates/check_wiring.sh +69 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/harness/README.md +11 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/harness/envelopes/README.md +10 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/harness/envelopes/result_envelope.json +14 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/harness/envelopes/task_envelope.json +20 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/harness/runner.py +303 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/installer/README.md +10 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/installer/install.sh +125 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/telemetry/README.md +11 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/telemetry/logger.py +112 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/BEST_PRACTICES/README.md +14 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/BEST_PRACTICES/api.md +17 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/BEST_PRACTICES/compliance-wiring.md +155 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/BEST_PRACTICES/frontend.md +15 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/CI_CD/README.md +11 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/CI_CD/deployment/README.md +23 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/CI_CD/deployment/clients/README.md +16 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/CI_CD/deployment/clients/template-README.md +11 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/CI_CD/deployment/deploy.sh +49 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/CI_CD/deployment/envs/README.md +12 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/CI_CD/deployment/envs/development.env.example +17 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/CI_CD/deployment/envs/production.env.example +17 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/CI_CD/deployment/envs/staging.env.example +17 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/CI_CD/deployment/tenants/README.md +11 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/CI_CD/deployment/tenants/template.env.example +8 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/DEVELOPER_SETUP/README.md +26 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/DEVELOPER_SETUP/agent.md +7 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/DEVELOPER_SETUP/developers/dev-1.md +20 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/DEVELOPER_SETUP/linux.md +9 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/DEVELOPER_SETUP/macos.md +10 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/DEVELOPER_SETUP/shared.md +19 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/DEVELOPER_SETUP/windows.md +14 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/HOSTING/README.md +25 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/HOSTING/_template.md +41 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/HOSTING/hosting-1.md +41 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/README.md +21 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/RUNBOOK/README.md +15 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/RUNBOOK/deployment.md +19 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/RUNBOOK/disaster-recovery.md +15 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/RUNBOOK/ops-tasks.md +14 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/features/README.md +16 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/project-skills.md +146 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/research/README.md +11 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/research/software-compliance-research.md +269 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/working-with-agents.md +260 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/root/AGENTS.md +38 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/root/ARCHITECTURE.md +36 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/root/BEST_PRACTICES.md +27 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/root/CI_CD.md +23 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/root/FEATURES.md +15 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/root/README.md +22 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/root/RULES.md +37 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/root/STRUCTURE.md +45 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/root/TECH_STACK.md +28 -0
- package/.agents/skills/nsr-compliance/assets/nsr/scaffold-spec.md +63 -0
- package/.agents/skills/nsr-compliance/assets/nsr/scripts/scaffold.sh +103 -0
- package/.agents/skills/nsr-compliance/assets/nsr/scripts/validate.sh +52 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/SKILL.md +71 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/config/core_four.yaml +16 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/gates/check_danger.sh +102 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/gates/check_wiring.sh +69 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/gates/validate_code.sh +29 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/gates/verify_docs.py +66 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/generate.sh +109 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/harness/envelopes/result_envelope.json +14 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/harness/envelopes/task_envelope.json +20 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/harness/runner.py +303 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/telemetry/logger.py +112 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/templates/compliance/README.md +18 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/templates/compliance/config/README.md +10 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/templates/compliance/gates/README.md +12 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/templates/compliance/gates/check_env.sh +23 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/templates/compliance/gates/check_paths.sh +20 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/templates/compliance/gates/check_platform.sh +23 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/templates/compliance/harness/README.md +11 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/templates/compliance/harness/envelopes/README.md +10 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/templates/compliance/installer/README.md +10 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/templates/compliance/installer/install.sh +125 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/templates/compliance/telemetry/README.md +11 -0
- package/.agents/skills/pr-ops/README.md +3 -0
- package/.agents/skills/pr-ops/SKILL.md +109 -0
- package/.agents/skills/ratatoskr-a2a/README.md +3 -0
- package/.agents/skills/ratatoskr-a2a/SKILL.md +148 -0
- package/.agents/skills/rules-check-drift/SKILL.md +81 -0
- package/.agents/skills/saga-bearings/README.md +3 -0
- package/.agents/skills/saga-bearings/SKILL.md +26 -0
- package/.agents/skills/saga-bearings/assets/bearings.md +161 -0
- package/.agents/skills/saga-bearings/assets/board-template.html +731 -0
- package/.agents/skills/saga-bearings/assets/recap.md +56 -0
- package/.agents/skills/syn-recovery/README.md +3 -0
- package/.agents/skills/syn-recovery/SKILL.md +54 -0
- package/.agents/skills/tyr-check/README.md +3 -0
- package/.agents/skills/tyr-check/SKILL.md +218 -0
- package/.agents/skills/urdh-hold/README.md +3 -0
- package/.agents/skills/urdh-hold/SKILL.md +25 -0
- package/.agents/skills/urdh-hold/assets/decisions.md +15 -0
- package/.agents/skills/urdh-hold/assets/hold.md +54 -0
- package/.agents/skills/vor-diagnostics/README.md +3 -0
- package/.agents/skills/vor-diagnostics/SKILL.md +68 -0
- package/.agents/skills/ymir-host/README.md +3 -0
- package/.agents/skills/ymir-host/SKILL.md +35 -0
- package/.agents/skills/ymir-host/assets/desktop.md +36 -0
- package/.agents/skills/ymir-host/assets/install.md +69 -0
- package/.agents/skills/ymir-host/assets/omarchy.md +297 -0
- package/.agents/skills/ymir-host/assets/runbooks.md +15 -0
- package/.agents/skills/ymir-host/assets/thjazi.md +220 -0
- package/.agents/skills/ymir-host/assets/update.md +64 -0
- package/.agents/tests/assets/board-render-harness.mjs +117 -0
- package/.agents/tests/cmux-test-safety.sh +51 -0
- package/.agents/tests/fixtures.sh +291 -0
- package/.agents/tests/fm-afk-inject-e2e.test.sh +428 -0
- package/.agents/tests/fm-afk-inject-herdr-e2e.test.sh +536 -0
- package/.agents/tests/fm-afk-launch.test.sh +961 -0
- package/.agents/tests/fm-afk-pi-herdr-return-e2e.test.sh +297 -0
- package/.agents/tests/fm-afk-return.test.sh +283 -0
- package/.agents/tests/fm-arm-pretool-check.test.sh +482 -0
- package/.agents/tests/fm-ask-user-authority.test.sh +42 -0
- package/.agents/tests/fm-backend-autodetect-smoke.test.sh +176 -0
- package/.agents/tests/fm-backend-cmux-smoke.test.sh +188 -0
- package/.agents/tests/fm-backend-cmux.test.sh +1166 -0
- package/.agents/tests/fm-backend-herdr-eventwait-smoke.test.sh +136 -0
- package/.agents/tests/fm-backend-herdr-eventwait.test.py +106 -0
- package/.agents/tests/fm-backend-herdr-focus-flash-e2e.test.sh +409 -0
- package/.agents/tests/fm-backend-herdr-launcher-workspace-e2e.test.sh +437 -0
- package/.agents/tests/fm-backend-herdr-presentation-e2e.test.sh +1424 -0
- package/.agents/tests/fm-backend-herdr-prune-safety-e2e.test.sh +180 -0
- package/.agents/tests/fm-backend-herdr-respawn-idem-e2e.test.sh +182 -0
- package/.agents/tests/fm-backend-herdr-smoke.test.sh +355 -0
- package/.agents/tests/fm-backend-herdr-workspace-per-home-e2e.test.sh +251 -0
- package/.agents/tests/fm-backend-herdr.test.sh +4606 -0
- package/.agents/tests/fm-backend-orca.test.sh +1357 -0
- package/.agents/tests/fm-backend-tmux-smoke.test.sh +173 -0
- package/.agents/tests/fm-backend-zellij-smoke.test.sh +205 -0
- package/.agents/tests/fm-backend-zellij.test.sh +1360 -0
- package/.agents/tests/fm-backend.test.sh +1142 -0
- package/.agents/tests/fm-backlog-atomicity.test.sh +2302 -0
- package/.agents/tests/fm-backlog-handoff.test.sh +1360 -0
- package/.agents/tests/fm-bearings-board-render.test.sh +135 -0
- package/.agents/tests/fm-bearings-board.test.sh +414 -0
- package/.agents/tests/fm-bearings-snapshot.test.sh +1990 -0
- package/.agents/tests/fm-bootstrap-network-parallel.test.sh +316 -0
- package/.agents/tests/fm-bootstrap.test.sh +1178 -0
- package/.agents/tests/fm-branch-supervision.test.sh +552 -0
- package/.agents/tests/fm-brief.test.sh +785 -0
- package/.agents/tests/fm-busy-adapter-wiring.test.sh +324 -0
- package/.agents/tests/fm-busy-state.test.sh +465 -0
- package/.agents/tests/fm-calm-pi-extension.test.sh +3979 -0
- package/.agents/tests/fm-captain-hold-lifecycle.test.sh +1187 -0
- package/.agents/tests/fm-cd-pretool-check.test.sh +400 -0
- package/.agents/tests/fm-check-unregister.test.sh +198 -0
- package/.agents/tests/fm-classify-corr-token.test.sh +543 -0
- package/.agents/tests/fm-classify-decision-key.test.sh +340 -0
- package/.agents/tests/fm-claude-stop-autoarm-live-e2e.test.sh +164 -0
- package/.agents/tests/fm-claude-stop-autoarm.test.sh +1190 -0
- package/.agents/tests/fm-cmux-claude-composer-live-e2e.test.sh +110 -0
- package/.agents/tests/fm-codex-continuity-live-e2e.test.sh +55 -0
- package/.agents/tests/fm-composer-ghost.test.sh +716 -0
- package/.agents/tests/fm-composer-lib.test.sh +666 -0
- package/.agents/tests/fm-composer-matrix-live-e2e.test.sh +220 -0
- package/.agents/tests/fm-control-herdr-smoke.test.sh +149 -0
- package/.agents/tests/fm-control-relaunch.test.sh +1530 -0
- package/.agents/tests/fm-control.test.sh +910 -0
- package/.agents/tests/fm-crew-state.test.sh +1610 -0
- package/.agents/tests/fm-cursor-harness.test.sh +404 -0
- package/.agents/tests/fm-cursor-primary-live-e2e.test.sh +217 -0
- package/.agents/tests/fm-cursor-primary.test.sh +708 -0
- package/.agents/tests/fm-daemon.test.sh +2740 -0
- package/.agents/tests/fm-documentation-audiences.test.sh +141 -0
- package/.agents/tests/fm-ensure-agents-md.test.sh +371 -0
- package/.agents/tests/fm-extension-binding.test.sh +2187 -0
- package/.agents/tests/fm-fleet-snapshot-view.test.sh +816 -0
- package/.agents/tests/fm-fleet-sync.test.sh +721 -0
- package/.agents/tests/fm-gate-refuse.test.sh +356 -0
- package/.agents/tests/fm-gitignore-config.test.sh +89 -0
- package/.agents/tests/fm-gotmp.test.sh +230 -0
- package/.agents/tests/fm-grok-continuity-live-e2e.test.sh +112 -0
- package/.agents/tests/fm-grok-harness.test.sh +117 -0
- package/.agents/tests/fm-grok-stop-live-e2e.test.sh +219 -0
- package/.agents/tests/fm-guard-stale-banner.test.sh +709 -0
- package/.agents/tests/fm-harness-adapter-instructions-live-e2e.test.sh +134 -0
- package/.agents/tests/fm-harness-adapter-references.test.sh +30 -0
- package/.agents/tests/fm-harness-liveness-drift-live-e2e.test.sh +151 -0
- package/.agents/tests/fm-herdr-lab.test.sh +243 -0
- package/.agents/tests/fm-herdr-session-cleanup-e2e.test.sh +146 -0
- package/.agents/tests/fm-herdr-session-cleanup.test.sh +329 -0
- package/.agents/tests/fm-herdr-submit-confirm-live-e2e.test.sh +125 -0
- package/.agents/tests/fm-herdr-version-floor-live-e2e.test.sh +144 -0
- package/.agents/tests/fm-home-summary-refresh.test.sh +971 -0
- package/.agents/tests/fm-inactive-reconcile.test.sh +485 -0
- package/.agents/tests/fm-kimi-harness.test.sh +687 -0
- package/.agents/tests/fm-lint-workflows.test.sh +532 -0
- package/.agents/tests/fm-lint.test.sh +1023 -0
- package/.agents/tests/fm-muse-harness.test.sh +932 -0
- package/.agents/tests/fm-muse-signals-live-e2e.test.sh +205 -0
- package/.agents/tests/fm-no-mistakes-required.test.sh +72 -0
- package/.agents/tests/fm-on.test.sh +507 -0
- package/.agents/tests/fm-opencode-primary-live-e2e.test.sh +357 -0
- package/.agents/tests/fm-operational-input.test.sh +160 -0
- package/.agents/tests/fm-peek-remote.test.sh +110 -0
- package/.agents/tests/fm-pending-reply.test.sh +1288 -0
- package/.agents/tests/fm-pi-branch-extension.test.sh +3182 -0
- package/.agents/tests/fm-pi-branch-live-e2e.test.sh +527 -0
- package/.agents/tests/fm-pi-primary-live-e2e.test.sh +345 -0
- package/.agents/tests/fm-pi-primary-types.test.sh +68 -0
- package/.agents/tests/fm-pi-watch-extension.test.sh +2843 -0
- package/.agents/tests/fm-pr-check-security.test.sh +2140 -0
- package/.agents/tests/fm-pr-merge.test.sh +2135 -0
- package/.agents/tests/fm-procevent-quota.test.sh +225 -0
- package/.agents/tests/fm-procevent-when.test.sh +406 -0
- package/.agents/tests/fm-procevent.test.sh +1745 -0
- package/.agents/tests/fm-project-origin.test.sh +110 -0
- package/.agents/tests/fm-public-followup.test.sh +2272 -0
- package/.agents/tests/fm-quota-array-dispatch-live-e2e.test.sh +526 -0
- package/.agents/tests/fm-quota-choose.test.sh +626 -0
- package/.agents/tests/fm-remote-backlog-handoff.test.sh +453 -0
- package/.agents/tests/fm-remote-doctor.test.sh +625 -0
- package/.agents/tests/fm-remote-entrypoint.test.sh +59 -0
- package/.agents/tests/fm-remote-job-orphan-reap.test.sh +205 -0
- package/.agents/tests/fm-remote-job.test.sh +768 -0
- package/.agents/tests/fm-remote-reply.test.sh +520 -0
- package/.agents/tests/fm-remote-secondmate-lifecycle-e2e.test.sh +1256 -0
- package/.agents/tests/fm-remote-secondmate-parent-binding.test.sh +326 -0
- package/.agents/tests/fm-remote-secondmate-trace-context.test.sh +310 -0
- package/.agents/tests/fm-remote-transport-lanes.test.sh +431 -0
- package/.agents/tests/fm-review-diff.test.sh +176 -0
- package/.agents/tests/fm-secondmate-harness.test.sh +2600 -0
- package/.agents/tests/fm-secondmate-lifecycle-e2e.test.sh +259 -0
- package/.agents/tests/fm-secondmate-liveness.test.sh +559 -0
- package/.agents/tests/fm-secondmate-reconcile.test.sh +732 -0
- package/.agents/tests/fm-secondmate-safety.test.sh +2972 -0
- package/.agents/tests/fm-secondmate-sync.test.sh +913 -0
- package/.agents/tests/fm-send-inbox-doorbell-live-e2e.test.sh +206 -0
- package/.agents/tests/fm-send-inbox.test.sh +352 -0
- package/.agents/tests/fm-send-popup-settle.test.sh +167 -0
- package/.agents/tests/fm-send-remote-delivery.test.sh +796 -0
- package/.agents/tests/fm-send-resolve-key.test.sh +551 -0
- package/.agents/tests/fm-send-secondmate-marker-herdr-e2e.test.sh +187 -0
- package/.agents/tests/fm-send-secondmate-marker.test.sh +278 -0
- package/.agents/tests/fm-send-settle.test.sh +147 -0
- package/.agents/tests/fm-send-strict.test.sh +241 -0
- package/.agents/tests/fm-session-lock-ancestry.test.sh +365 -0
- package/.agents/tests/fm-session-start.test.sh +2515 -0
- package/.agents/tests/fm-sessionstart-hook-live-e2e.test.sh +631 -0
- package/.agents/tests/fm-sessionstart-instruction-refresh-live-e2e.test.sh +230 -0
- package/.agents/tests/fm-sessionstart-nudge.test.sh +1039 -0
- package/.agents/tests/fm-shared-captain-inheritance.test.sh +404 -0
- package/.agents/tests/fm-spawn-batch.test.sh +148 -0
- package/.agents/tests/fm-spawn-dispatch-profile.test.sh +827 -0
- package/.agents/tests/fm-spawn-pool-base-freshen.test.sh +440 -0
- package/.agents/tests/fm-spawn-worktree-settle.test.sh +147 -0
- package/.agents/tests/fm-startup-memory-budget.test.sh +333 -0
- package/.agents/tests/fm-startup-network.test.sh +712 -0
- package/.agents/tests/fm-stow-cascade.test.sh +370 -0
- package/.agents/tests/fm-subagent-pretool-check.test.sh +291 -0
- package/.agents/tests/fm-supervision-events.test.sh +156 -0
- package/.agents/tests/fm-supervision-instructions.test.sh +189 -0
- package/.agents/tests/fm-tangle-guard.test.sh +277 -0
- package/.agents/tests/fm-task-delivery.test.sh +420 -0
- package/.agents/tests/fm-task-inbox.test.sh +513 -0
- package/.agents/tests/fm-teardown-endpoint-safety.test.sh +374 -0
- package/.agents/tests/fm-teardown.test.sh +2665 -0
- package/.agents/tests/fm-test-fixture-cleanup.test.sh +172 -0
- package/.agents/tests/fm-test-fixtures.test.sh +132 -0
- package/.agents/tests/fm-test-isolation-proof.test.sh +278 -0
- package/.agents/tests/fm-test-run.test.sh +1196 -0
- package/.agents/tests/fm-tmux-agent-liveness.test.sh +354 -0
- package/.agents/tests/fm-tmux-submit-busy.test.sh +340 -0
- package/.agents/tests/fm-tool-update-check.test.sh +1044 -0
- package/.agents/tests/fm-trace-context-lib.test.sh +253 -0
- package/.agents/tests/fm-trace-context-spawn.test.sh +581 -0
- package/.agents/tests/fm-transition-lib.test.sh +54 -0
- package/.agents/tests/fm-turnend-guard.test.sh +1822 -0
- package/.agents/tests/fm-update.test.sh +304 -0
- package/.agents/tests/fm-vendor-auth-probe.test.sh +395 -0
- package/.agents/tests/fm-voice-relay.test.sh +4741 -0
- package/.agents/tests/fm-wake-daemon-lifecycle-e2e.test.sh +171 -0
- package/.agents/tests/fm-wake-drain-open-decisions-cursor.test.sh +356 -0
- package/.agents/tests/fm-wake-drain-open-decisions.test.sh +226 -0
- package/.agents/tests/fm-wake-drain-unread-status.test.sh +379 -0
- package/.agents/tests/fm-wake-queue.test.sh +1224 -0
- package/.agents/tests/fm-watch-arm.test.sh +821 -0
- package/.agents/tests/fm-watch-checkpoint.test.sh +87 -0
- package/.agents/tests/fm-watch-recovery-loop.test.sh +223 -0
- package/.agents/tests/fm-watch-triage.test.sh +3889 -0
- package/.agents/tests/fm-watcher-lock.test.sh +1133 -0
- package/.agents/tests/fm-x-mode.test.sh +3042 -0
- package/.agents/tests/gleipnir-guard-pretool-check.test.sh +51 -0
- package/.agents/tests/gleipnir-machine-lock.test.sh +130 -0
- package/.agents/tests/herdr-test-safety.sh +42 -0
- package/.agents/tests/lib.sh +315 -0
- package/.agents/tests/remote-herdr-fixture.sh +125 -0
- package/.agents/tests/secondmate-helpers.sh +193 -0
- package/.agents/tests/smoke.test.sh +93 -0
- package/.agents/tests/wake-helpers.sh +338 -0
- package/.agents/tests/zellij-test-safety.sh +60 -0
- package/.agents/tools/README.md +10 -0
- package/.agents/tools/bin/tasks-cli +267 -0
- package/.agents/tools/tasks-cli.ts +216 -0
- package/AGENTS.md +498 -0
- package/CHANGELOG.md +1913 -0
- package/LICENSE +1 -1
- package/NOTICE +63 -0
- package/README.md +114 -60
- package/RULES/01-domains.md +51 -0
- package/RULES/02-agents.md +78 -0
- package/RULES/03-houses.md +61 -0
- package/RULES/04-hoard.md +97 -0
- package/RULES/05-platforms.md +67 -0
- package/RULES/06-append-only.md +108 -0
- package/RULES/07-config.md +101 -0
- package/RULES/08-delivery-gate.md +75 -0
- package/RULES/09-electron.md +58 -0
- package/RULES/README.md +22 -0
- package/bin/README.md +107 -0
- package/bin/a2a-mcp.sh +133 -0
- package/bin/a2a-serve.py +103 -0
- package/bin/a2a-serve.sh +8 -0
- package/bin/a2a-talk.sh +50 -0
- package/bin/a2abridge-ensure.sh +108 -0
- package/bin/agent-run.sh +43 -0
- package/bin/agents-config.sh +256 -0
- package/bin/apodex-smoke-test.sh +105 -0
- package/bin/bifrost-bridge.sh +131 -0
- package/bin/bifrost-ingress.sh +77 -0
- package/bin/bootstrap-macos.sh +112 -0
- package/bin/bootstrap-windows.ps1 +112 -0
- package/bin/branch-guard.sh +64 -0
- package/bin/brokk +73 -0
- package/bin/brokk-classify-lib.sh +1770 -0
- package/bin/brokk-lease-lib.sh +222 -0
- package/bin/brokk-lease.sh +193 -0
- package/bin/brokk-lint.sh +65 -0
- package/bin/brokk-send.sh +51 -0
- package/bin/brokk-update.sh +9 -0
- package/bin/brokk-wake-grant.sh +127 -0
- package/bin/brokk-wake-lib.sh +1849 -0
- package/bin/changelog-assemble.sh +154 -0
- package/bin/changelog-guard.sh +197 -0
- package/bin/crash-sense.sh +173 -0
- package/bin/design-check.sh +80 -0
- package/bin/design-icon.sh +153 -0
- package/bin/desktop-place.sh +224 -0
- package/bin/docs-guard.sh +39 -0
- package/bin/editor-place.sh +225 -0
- package/bin/eindri-acclaim.sh +47 -0
- package/bin/eindri-control.sh +28 -0
- package/bin/eindri-role.sh +83 -0
- package/bin/eindri-seat.sh +83 -0
- package/bin/eindri-seen.sh +54 -0
- package/bin/eindri-send.sh +15 -0
- package/bin/eindri-start.sh +98 -0
- package/bin/eindri-watch.sh +57 -0
- package/bin/einherjar-spawn.sh +576 -0
- package/bin/eir-doctor.sh +158 -0
- package/bin/erindi-brief.sh +271 -0
- package/bin/feature-inventory.sh +73 -0
- package/bin/fleet-apply.sh +81 -0
- package/bin/github-deploy.sh +94 -0
- package/bin/gjallarhorn-expose.sh +136 -0
- package/bin/gjallarhorn-notify.sh +71 -0
- package/bin/gjallarhorn-purge.sh +67 -0
- package/bin/gjallarhorn-tunnel.sh +85 -0
- package/bin/gleipnir-lock-lib.sh +284 -0
- package/bin/groa-update.sh +130 -0
- package/bin/hall-snapshot.sh +111 -0
- package/bin/hamr-harness.sh +246 -0
- package/bin/heimdall-ssh-keys.sh +374 -0
- package/bin/herdr-agents.py +44 -0
- package/bin/herdr-ensure.sh +104 -0
- package/bin/herdr-run.sh +452 -0
- package/bin/hermes-ensure.sh +88 -0
- package/bin/hlidskjalf-agents.sh +172 -0
- package/bin/hlidskjalf-usage.sh +196 -0
- package/bin/hoard-guard.sh +135 -0
- package/bin/hoard-lib.sh +105 -0
- package/bin/hodd.sh +102 -0
- package/bin/host-sense.sh +199 -0
- package/bin/huginn-research-worker.sh +121 -0
- package/bin/local-model-lock.sh +70 -0
- package/bin/mimir-bridge.py +259 -0
- package/bin/mimir-bridge.sh +100 -0
- package/bin/mimir-ingest.sh +131 -0
- package/bin/mimir-reflect.sh +27 -0
- package/bin/mimir.sh +113 -0
- package/bin/mjollnir-webhook.sh +72 -0
- package/bin/mjollnir.sh +104 -0
- package/bin/model-hardware.sh +96 -0
- package/bin/model-resolve.sh +160 -0
- package/bin/models-detect.sh +178 -0
- package/bin/nornir-cron-start.sh +162 -0
- package/bin/nornir-job-daily-briefing.sh +191 -0
- package/bin/nornir-job-git-sync.sh +152 -0
- package/bin/nornir-job-hall-snapshot.sh +28 -0
- package/bin/nornir-job-memory-housekeeping.sh +114 -0
- package/bin/nornir-job-nsr-compliance.sh +54 -0
- package/bin/nornir-job-observer.sh +166 -0
- package/bin/nornir-job-skillopt-sleep.sh +76 -0
- package/bin/npm-publish.sh +136 -0
- package/bin/omarchy-hook-install.sh +60 -0
- package/bin/omarchy-install.sh +161 -0
- package/bin/omarchy-plugins.sh +187 -0
- package/bin/omarchy-sense.sh +162 -0
- package/bin/perm-guard.sh +30 -0
- package/bin/pi-agent.sh +68 -0
- package/bin/pi-ensure.sh +77 -0
- package/bin/pi-local.sh +37 -0
- package/bin/pi-model-agnostic.sh +71 -0
- package/bin/pi-seat.sh +63 -0
- package/bin/prereq-ensure.sh +166 -0
- package/bin/private-guard.sh +58 -0
- package/bin/project-git.sh +69 -0
- package/bin/public-guard.sh +71 -0
- package/bin/ratatoskr.sh +49 -0
- package/bin/realm-lib.sh +32 -0
- package/bin/repo-scrub.sh +62 -0
- package/bin/rodd-operational-input.sh +250 -0
- package/bin/runes-append.sh +198 -0
- package/bin/saga-session-start.sh +172 -0
- package/bin/saga-sessionstart-run.sh +79 -0
- package/bin/saga-wake-drain.sh +41 -0
- package/bin/secret-guard.sh +63 -0
- package/bin/sessrumnir-ensure.sh +100 -0
- package/bin/sessrumnir-sync.sh +237 -0
- package/bin/sessrumnir.sh +104 -0
- package/bin/skillopt-setup.sh +76 -0
- package/bin/skuld-branch-outcome.sh +203 -0
- package/bin/skuld-branch-prompt.sh +101 -0
- package/bin/smidja-bootstrap.sh +94 -0
- package/bin/smidja-observe.sh +86 -0
- package/bin/syn-arm-pretool-check.sh +57 -0
- package/bin/syn-asset-pretool-check.sh +89 -0
- package/bin/syn-cd-pretool-check.sh +28 -0
- package/bin/syn-guard-pretool-check.sh +83 -0
- package/bin/syn-spawn-pretool-check.sh +58 -0
- package/bin/syn-turnend-guard.sh +36 -0
- package/bin/syn-watch-arm.sh +99 -0
- package/bin/tailscale-sync.sh +203 -0
- package/bin/telegram-bot.sh +82 -0
- package/bin/toolchain.sh +74 -0
- package/bin/ui-truth-check.sh +75 -0
- package/bin/utgard.sh +111 -0
- package/bin/valhalla.sh +115 -0
- package/bin/valknut-load.sh +298 -0
- package/bin/vor-crew-state.sh +124 -0
- package/bin/wedge-notify.sh +60 -0
- package/bin/workspace-provision.sh +80 -0
- package/bin/workspace-rag.sh +163 -0
- package/bin/wyrd-db.sh +47 -0
- package/bin/yggdrasil.sh +152 -0
- package/bin/ymir-edit.sh +145 -0
- package/bin/ymir-install.sh +806 -0
- package/bin/ymir-invite.sh +93 -0
- package/bin/ymir-isolation.sh +88 -0
- package/bin/ymir-migrate.sh +61 -0
- package/bin/ymir-plan.sh +356 -0
- package/bin/ymir-platform.sh +348 -0
- package/bin/ymir-say.sh +110 -0
- package/bin/ymir-setup-auth.sh +119 -0
- package/bin/ymir-validate.sh +193 -0
- package/bin/ymir.js +74 -16
- package/docs/Architecture.md +339 -0
- package/docs/design.md +351 -0
- package/docs/installations/README.md +63 -0
- package/docs/installations/ymir-home.md +93 -0
- package/docs/integration.md +105 -0
- package/docs/lore.md +1060 -0
- package/docs/research/agent-harnesses.md +290 -0
- package/docs/runbooks/README.md +19 -0
- package/docs/runbooks/a2a.md +91 -0
- package/docs/runbooks/agent-permissions.md +99 -0
- package/docs/runbooks/agents.md +57 -0
- package/docs/runbooks/models.md +69 -0
- package/docs/runbooks/realm-onboarding.md +60 -0
- package/docs/runbooks/secrets-and-hoard.md +60 -0
- package/docs/runbooks/updates-and-migrations.md +110 -0
- package/docs/runbooks/worktrees-isolation.md +72 -0
- package/docs/session-start.md +105 -0
- package/docs/skillopt-integration.md +321 -0
- package/docs/sync-system.md +66 -0
- package/docs/workspaces.md +104 -0
- package/docs/ymir-rut.md +260 -0
- package/package.json +54 -14
- package/packaging/build.sh +74 -0
- package/packaging/macos/Ymir Installer.command +43 -0
- package/packaging/windows/ymir-setup.nsi +74 -0
- package/scripts/README.md +22 -0
- package/scripts/electron.sh +269 -0
- package/scripts/lower.sh +34 -0
- package/scripts/raise.sh +42 -0
- package/scripts/start.sh +217 -0
- package/scripts/stop.sh +113 -0
- package/install.sh +0 -35
|
@@ -0,0 +1,1962 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# Firstmate watcher.
|
|
3
|
+
# Classifies supervision wakes in bash. In normal mode it absorbs benign wakes
|
|
4
|
+
# and keeps blocking; it queues and exits only for actionable wakes.
|
|
5
|
+
# The no-verb signal and stale path is absorb-only-on-positive-evidence: a wake
|
|
6
|
+
# is absorbed only when the crew shows it is still working through an actively
|
|
7
|
+
# running no-mistakes step or a backend busy signal. A home that opts in with
|
|
8
|
+
# config/turnend-churn-absorb lets a bare turn-end also use bounded pane churn
|
|
9
|
+
# since the previous poll. Every other no-verb wake surfaces, so a crew
|
|
10
|
+
# that finishes (or stops and waits) is never silently swallowed. A declared wait,
|
|
11
|
+
# either a paused: external wait or a verified captain-held transfer, is the
|
|
12
|
+
# separate idle absorb case and re-surfaces only on its long bounded cadence,
|
|
13
|
+
# although its initial no-verb status signal still surfaces in normal mode.
|
|
14
|
+
# While state/.afk exists, the daemon owns triage and this watcher queues and exits
|
|
15
|
+
# on every wake. Printed reason lines:
|
|
16
|
+
# signal: <file>... status/turn-end signals, surfaced when a listed status
|
|
17
|
+
# span has a captain-relevant event OR a no-verb signal lacks
|
|
18
|
+
# positive execution evidence, unless afk is active
|
|
19
|
+
# stale: <window> a provably-working stale is ALWAYS absorbed (with a wedge
|
|
20
|
+
# timer) regardless of what the status log says - an active
|
|
21
|
+
# run-step or busy pane outranks even a captain-relevant log
|
|
22
|
+
# line, since the crew's own log gets no new entry once
|
|
23
|
+
# firstmate hands it to a no-mistakes validation. A declared
|
|
24
|
+
# external-wait pause or verified captain-held transfer is
|
|
25
|
+
# absorbed instead with its own long re-surface cadence,
|
|
26
|
+
# never as a wedge, and that recheck reason names which
|
|
27
|
+
# human the wait is on. Only when neither absorb class
|
|
28
|
+
# applies does the log's last line decide:
|
|
29
|
+
# terminal (captain-relevant) or non-terminal (no verb),
|
|
30
|
+
# both surfaced at once. A provably-working stale past the
|
|
31
|
+
# wedge threshold also surfaces, with an "escalation N"
|
|
32
|
+
# count in the reason; at FM_WEDGE_DEMAND_INSPECT_COUNT
|
|
33
|
+
# consecutive escalations on the SAME pane, the reason
|
|
34
|
+
# also carries a "demand-deep-inspection" marker so the
|
|
35
|
+
# wake payload itself, not just repetition, forces a
|
|
36
|
+
# closer look instead of another routine supervision
|
|
37
|
+
# resume. Unless afk is active. A pane whose own task
|
|
38
|
+
# worktree was written during the quiet window is
|
|
39
|
+
# deferred rather than escalated (wedge_defer_writing),
|
|
40
|
+
# because files appearing there are liveness the pane and
|
|
41
|
+
# the run step cannot show; that deferral still
|
|
42
|
+
# re-surfaces once per PAUSE_RESURFACE_SECS, and a pane
|
|
43
|
+
# that writes nothing keeps the unchanged schedule.
|
|
44
|
+
# A genuinely busy pane
|
|
45
|
+
# (window_is_busy true) is exempt from the above, but
|
|
46
|
+
# only up to BUSY_TURN_MAX_SECS with no completed turn
|
|
47
|
+
# (state/<id>.turn-ended, or the spawn record before any
|
|
48
|
+
# turn completes). Past that bound, a declared external
|
|
49
|
+
# wait or verified captain-held transfer uses the long
|
|
50
|
+
# pause recheck cadence (under afk it is instead handed
|
|
51
|
+
# to the daemon as this plain reason, once per
|
|
52
|
+
# declaration; busy_turn_bound_check owns that handoff);
|
|
53
|
+
# every other pane goes through the same wedge timer and
|
|
54
|
+
# surfaces with the identical "stale: ..." reason,
|
|
55
|
+
# escalation count, and demand-deep-inspection marker,
|
|
56
|
+
# for human inspection only - never an automatic
|
|
57
|
+
# interrupt, signal, or restart of the worker or its
|
|
58
|
+
# tool process.
|
|
59
|
+
# stale: <window> (unread firstmate instruction: ...)
|
|
60
|
+
# the steering-inbox ladder spent its delivery-attempt
|
|
61
|
+
# budget on an idle pane without an acknowledgement
|
|
62
|
+
# stale: <window> (steering-inbox ladder bookkeeping unwritable: ...)
|
|
63
|
+
# an unhandled record's ladder cannot advance; quiet
|
|
64
|
+
# successful attempts never wake firstmate
|
|
65
|
+
# (bin/fm-task-inbox-lib.sh owns the ladder policy)
|
|
66
|
+
# check: <script>: <out> authenticated check output, always actionable
|
|
67
|
+
# check: process-event result captured: <keys>
|
|
68
|
+
# a durably captured process-to-event result is queued
|
|
69
|
+
# and has not been surfaced yet; reported once per
|
|
70
|
+
# captured generation, never again while that record
|
|
71
|
+
# stays queued and never once it is acknowledged
|
|
72
|
+
# check: rejected unauthenticated state checks: <paths>
|
|
73
|
+
# unsafe state checks were refused without execution
|
|
74
|
+
# check: rejected unauthenticated PR poll retirement receipts: <paths>
|
|
75
|
+
# invalid pending retirements were preserved without
|
|
76
|
+
# running a check or removing poll artifacts
|
|
77
|
+
# heartbeat fleet-scan backstop found an unsurfaced captain-relevant
|
|
78
|
+
# status, unless afk is active
|
|
79
|
+
# check: inactive-outcome bounded poll-loop reconciliation found a suspicious
|
|
80
|
+
# inactive terminal outcome that still lacks its durable
|
|
81
|
+
# upstream receipt
|
|
82
|
+
# check: secondmate wake-loop stalled: mate=<id> row=<seq> age=<seconds>s
|
|
83
|
+
# the oldest valid row in an endpoint-recorded local
|
|
84
|
+
# secondmate home's durable wake queue exceeded
|
|
85
|
+
# FM_SECONDMATE_WAKE_STALL_SECS; observation is read-only
|
|
86
|
+
# and one parent receipt suppresses repeats for that row
|
|
87
|
+
# For normal supervision, resume the session-start primary-harness protocol
|
|
88
|
+
# after each printed reason. Direct duplicate invocations of this script still
|
|
89
|
+
# no-op through the watcher singleton lock.
|
|
90
|
+
set -u
|
|
91
|
+
|
|
92
|
+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
93
|
+
FM_ROOT="${FM_ROOT_OVERRIDE:-$(cd "$SCRIPT_DIR/.." && pwd)}"
|
|
94
|
+
FM_HOME="${FM_HOME:-${FM_ROOT_OVERRIDE:-$FM_ROOT}}"
|
|
95
|
+
STATE="${FM_STATE_OVERRIDE:-$FM_HOME/state}"
|
|
96
|
+
CONFIG="${FM_CONFIG_OVERRIDE:-$FM_HOME/config}"
|
|
97
|
+
mkdir -p "$STATE"
|
|
98
|
+
|
|
99
|
+
# The native event fast-path and only its true dependencies have one narrow
|
|
100
|
+
# production owner. The Herdr event-wait smoke test consumes this same owner
|
|
101
|
+
# without sourcing the entire watcher graph.
|
|
102
|
+
# The shared transition owner is a canonical lint root itself. Stop duplicate
|
|
103
|
+
# source-graph expansion here: following its backend graph from this large
|
|
104
|
+
# runtime can exceed the bounded CI lint worker while adding no uncovered file.
|
|
105
|
+
# shellcheck source=/dev/null
|
|
106
|
+
. "$SCRIPT_DIR/fm-push-transition-lib.sh"
|
|
107
|
+
# shellcheck source=bin/fm-pr-lib.sh
|
|
108
|
+
. "$SCRIPT_DIR/fm-pr-lib.sh"
|
|
109
|
+
# Single owner of durable merge-outcome publication, shared with
|
|
110
|
+
# bin/fm-pr-merge.sh so self and poll origins use the same role-routed outcome.
|
|
111
|
+
# The watcher still owns immediate delivery of its actionable poll result and
|
|
112
|
+
# poll retirement.
|
|
113
|
+
# This library is a canonical lint root in its own right, and it reaches the
|
|
114
|
+
# wake queue, PR identity, and secondmate parent libraries. Keep it an analysis
|
|
115
|
+
# boundary here for the same reason as the transition and inbox owners above and
|
|
116
|
+
# below: following its graph from this large runtime exceeds the bounded CI lint
|
|
117
|
+
# worker while adding no uncovered file.
|
|
118
|
+
# shellcheck source=/dev/null
|
|
119
|
+
. "$SCRIPT_DIR/fm-merge-outcome-lib.sh"
|
|
120
|
+
# shellcheck source=bin/fm-x-lib.sh
|
|
121
|
+
. "$SCRIPT_DIR/fm-x-lib.sh"
|
|
122
|
+
# shellcheck source=bin/fm-check-lib.sh
|
|
123
|
+
. "$SCRIPT_DIR/fm-check-lib.sh"
|
|
124
|
+
# Parent-owned secondmate missed-report guards: durable pending-reply
|
|
125
|
+
# expectations created by fm-send on marked secondmate requests. The tick is
|
|
126
|
+
# cheap when no records exist and never scrapes secondmate conversation.
|
|
127
|
+
# shellcheck source=bin/fm-pending-reply-lib.sh
|
|
128
|
+
. "$SCRIPT_DIR/fm-pending-reply-lib.sh"
|
|
129
|
+
# shellcheck source=bin/fm-busy-lib.sh
|
|
130
|
+
. "$SCRIPT_DIR/fm-busy-lib.sh"
|
|
131
|
+
# Steering-inbox loss detection: bin/fm-task-inbox-lib.sh owns the record,
|
|
132
|
+
# doorbell, and re-ring ladder contracts; this watcher only supplies the busy
|
|
133
|
+
# gate and the wake emission (inbox_steer_check below).
|
|
134
|
+
# shellcheck source=bin/fm-task-inbox-lib.sh
|
|
135
|
+
. "$SCRIPT_DIR/fm-task-inbox-lib.sh"
|
|
136
|
+
|
|
137
|
+
WATCH_LOCK="$STATE/.watch.lock"
|
|
138
|
+
WATCH_PATH="$SCRIPT_DIR/fm-watch.sh"
|
|
139
|
+
WATCHER_DOWNTIME_MARKER="$STATE/.watcher-down"
|
|
140
|
+
WATCHER_STALE_GRACE=${FM_WATCHER_STALE_GRACE:-${FM_GUARD_GRACE:-300}}
|
|
141
|
+
# The singleton-lock acquisition, EXIT trap, and the blocking supervision loop
|
|
142
|
+
# all live below the source guard at the very bottom of this file (see "Main
|
|
143
|
+
# entry"). Sourcing this file for unit tests therefore loads the functions -
|
|
144
|
+
# including the event-wait splice below - and returns before acquiring the lock
|
|
145
|
+
# or starting the loop. Running it as a script executes the runtime exactly as
|
|
146
|
+
# before, byte-for-byte.
|
|
147
|
+
|
|
148
|
+
# Portable stat. macOS (BSD) stat uses `-f <fmt>`; Linux (GNU) stat uses `-c <fmt>`.
|
|
149
|
+
# Do NOT use the `stat -f <fmt> ... || stat -c <fmt> ...` fallback form: on Linux
|
|
150
|
+
# `stat -f` is *filesystem* stat and writes a partial filesystem dump ("File: ...",
|
|
151
|
+
# "Blocks: ...") to stdout before failing, so the fallback's correct output gets
|
|
152
|
+
# appended to that garbage. Arithmetic under `set -u` then aborts on the stray
|
|
153
|
+
# token (e.g. the word "File" read as an unset variable), which silently kills the
|
|
154
|
+
# watcher mid-cycle. Detect the platform once and pick the right form.
|
|
155
|
+
if [ "$(uname)" = Darwin ]; then
|
|
156
|
+
stat_mtime() { stat -f %m "$1" 2>/dev/null; } # epoch seconds of mtime
|
|
157
|
+
else
|
|
158
|
+
stat_mtime() { stat -c %Y "$1" 2>/dev/null; }
|
|
159
|
+
fi
|
|
160
|
+
# bin/fm-classify-lib.sh owns status reported-state signatures and presentation
|
|
161
|
+
# markers, while bin/fm-wake-lib.sh owns their wake-facing routing, the legacy
|
|
162
|
+
# turn-ended signature, annotation staleness checks, and guarded bookkeeping writes.
|
|
163
|
+
|
|
164
|
+
POLL=${FM_POLL:-15} # seconds between cycles
|
|
165
|
+
HEARTBEAT=${FM_HEARTBEAT:-600} # base seconds between heartbeat scans
|
|
166
|
+
HEARTBEAT_MAX=${FM_HEARTBEAT_MAX:-7200} # heartbeat backoff cap
|
|
167
|
+
CHECK_INTERVAL=${FM_CHECK_INTERVAL:-300} # seconds between *.check.sh sweeps
|
|
168
|
+
CHECK_TIMEOUT=${FM_CHECK_TIMEOUT:-30} # seconds allowed per *.check.sh
|
|
169
|
+
HOME_SUMMARY_INTERVAL=${FM_HOME_SUMMARY_INTERVAL:-300}
|
|
170
|
+
case "$HOME_SUMMARY_INTERVAL" in
|
|
171
|
+
''|*[!0-9]*|0) HOME_SUMMARY_INTERVAL=300 ;;
|
|
172
|
+
esac
|
|
173
|
+
SIGNAL_GRACE=${FM_SIGNAL_GRACE:-30} # seconds to linger after a signal so trailing
|
|
174
|
+
# signals (a status write, then the same turn's
|
|
175
|
+
# turn-end hook) coalesce into one wake
|
|
176
|
+
TURNEND_CHURN_ABSORB_SECS=${FM_TURNEND_CHURN_ABSORB_SECS:-900} # longest a task's
|
|
177
|
+
# bare turn-ends may be deferred on pane-churn
|
|
178
|
+
# evidence alone (signal_turnend_panes_churned)
|
|
179
|
+
# Busy state is decided by the semantic contract in bin/fm-busy-lib.sh, which
|
|
180
|
+
# is the single owner of per-harness sources, source attribution, and the one
|
|
181
|
+
# remaining rendered-text fallback (Grok only).
|
|
182
|
+
# Always-on wake triage: most wakes during a long crew validation are benign (a
|
|
183
|
+
# working: note or turn-end while a pipeline runs, a no-change heartbeat). Rather
|
|
184
|
+
# than wake firstmate's LLM for each, this watcher classifies every wake in bash
|
|
185
|
+
# and ABSORBS the benign majority - it advances the suppression marker, logs to a
|
|
186
|
+
# debug log, and keeps blocking WITHOUT enqueuing or exiting. The no-verb signal
|
|
187
|
+
# / stale path is absorb-only-on-positive-evidence. The shared proof is an actively
|
|
188
|
+
# running no-mistakes step or a busy pane via crew_is_provably_working over
|
|
189
|
+
# fm-crew-state.sh; where config/turnend-churn-absorb opts in, a bare turn-end alone
|
|
190
|
+
# may also use bounded pane churn since the previous poll.
|
|
191
|
+
# Every other crew that stopped its turn is SURFACED, so a finish reported
|
|
192
|
+
# only through interactive pane menus (no done: status) is never swallowed. An
|
|
193
|
+
# ACTIONABLE wake (a captain-relevant signal, a no-verb signal without either
|
|
194
|
+
# eligible proof, any check, a stale pane whose crew is not provably working, a
|
|
195
|
+
# provably-working stale past the threshold, or anything unknown) is written to
|
|
196
|
+
# the durable queue and exits. That wakes the LLM through the background-task
|
|
197
|
+
# completion. The same classifier
|
|
198
|
+
# (fm-classify-lib.sh) backs the away-mode daemon; while state/.afk exists the
|
|
199
|
+
# daemon owns triage, so this watcher reverts to one-shot (enqueue + exit on every
|
|
200
|
+
# wake) and never double-triages - and never runs the costly provably-working read.
|
|
201
|
+
STALE_ESCALATE_SECS=${FM_STALE_ESCALATE_SECS:-240} # idle secs before a provably-working stale escalates as a possible wedge
|
|
202
|
+
# A busy pane is unconditional proof of liveness with no built-in duration bound,
|
|
203
|
+
# so a hung foreground call can remain hidden even while its rendered busy
|
|
204
|
+
# footer changes every poll. BUSY_TURN_MAX_SECS bounds how long any busy pane
|
|
205
|
+
# may go with no completed turn: once its task's
|
|
206
|
+
# state/<id>.turn-ended marker (or, before any turn has completed, the task's
|
|
207
|
+
# spawn record) is this old, busy_turn_over_age routes the pane through
|
|
208
|
+
# busy_turn_bound_check, which hands a crossed bound to the same
|
|
209
|
+
# STALE_ESCALATE_SECS-paced wedge_timer_check used for a provably-working
|
|
210
|
+
# non-busy stale - so it escalates via the existing stale reason, escalation
|
|
211
|
+
# counter, and demand-deep-inspection marker for human inspection only, never an
|
|
212
|
+
# automatic interrupt, signal, or restart - unless the crew declared the wait
|
|
213
|
+
# itself, which takes the long pause cadence instead. A completed turn touches
|
|
214
|
+
# turn-ended and resets the age. Set generously above any legitimate interval
|
|
215
|
+
# between completed turns, including long tool calls, builds, or test runs.
|
|
216
|
+
BUSY_TURN_MAX_SECS=${FM_BUSY_TURN_MAX_SECS:-3600}
|
|
217
|
+
# A local secondmate's foreign queue is checked on every poll, but only after this
|
|
218
|
+
# bounded age can it produce a parent notification.
|
|
219
|
+
SECONDMATE_WAKE_STALL_SECS=${FM_SECONDMATE_WAKE_STALL_SECS:-60}
|
|
220
|
+
# A crew that declared a pause is idling on a known external wait, so its stale
|
|
221
|
+
# pane is absorbed rather than wedge-escalated.
|
|
222
|
+
# A captain-held or paused crew whose agent has confidently exited uses the same
|
|
223
|
+
# bounded cadence, while a live or ambiguously read agent still surfaces once; a
|
|
224
|
+
# secondmate earns the cadence on its declaration alone, because its endpoint
|
|
225
|
+
# liveness is deliberately never read (pause_state_class owns that split).
|
|
226
|
+
# These cases re-surface once for a recheck every PAUSE_RESURFACE_SECS - far
|
|
227
|
+
# longer than the wedge threshold, but finite so a forgotten hold cannot rot invisibly.
|
|
228
|
+
PAUSE_RESURFACE_SECS=${FM_PAUSE_RESURFACE_SECS:-$FM_PAUSE_RESURFACE_SECS_DEFAULT}
|
|
229
|
+
# Consecutive event-path failures (fm_backend_wait_transition returning 2 -
|
|
230
|
+
# connect/subscribe failure) before the push fast-path is disabled for the rest
|
|
231
|
+
# of this watcher process and the loop reverts to pure polling (report section
|
|
232
|
+
# 5c trigger 3: proven-unreliable-at-runtime). A watcher restart re-probes
|
|
233
|
+
# capability, so a transient herdr hiccup self-heals on the next cycle chain.
|
|
234
|
+
EVENT_CAP_FAIL_MAX=${FM_EVENT_CAP_FAIL_MAX:-3}
|
|
235
|
+
# Per-process memo for the push-capability probe (fm_backend_events_capable runs
|
|
236
|
+
# a ~220KB `herdr api schema` read, too heavy to repeat every poll). Keyed by
|
|
237
|
+
# "<backend>:<session>"; re-probed only when that key changes.
|
|
238
|
+
_event_cap_key=""
|
|
239
|
+
_event_cap_ok=0
|
|
240
|
+
_event_cap_fails=0
|
|
241
|
+
|
|
242
|
+
# afk_present: 0 while the away-mode flag exists. When set, the daemon wraps this
|
|
243
|
+
# watcher and owns triage, so the watcher must behave one-shot (enqueue + exit on
|
|
244
|
+
# every wake) and let the daemon classify - never absorb here, or the daemon's
|
|
245
|
+
# digest/injection layer would never see the wake.
|
|
246
|
+
afk_present() { [ -e "$STATE/.afk" ]; }
|
|
247
|
+
|
|
248
|
+
hash_pane() {
|
|
249
|
+
if command -v md5 >/dev/null 2>&1; then md5 -q; else md5sum | cut -d' ' -f1; fi
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
# window_is_busy: 0 (busy) iff the task's harness is PROVABLY working, through
|
|
253
|
+
# the semantic busy-state contract (bin/fm-busy-lib.sh). Only an exact busy
|
|
254
|
+
# verdict returns 0: idle, unknown, and dead all return 1, so a converted
|
|
255
|
+
# adapter whose semantic state is missing, malformed, stale, or unverified is
|
|
256
|
+
# treated as not-provably-working and surfaces rather than being absorbed.
|
|
257
|
+
# <tail40> is the same bounded capture already read for hashing and is
|
|
258
|
+
# consumed only by the Grok-scoped fallback inside the contract.
|
|
259
|
+
window_is_busy() { # <window> <tail40>
|
|
260
|
+
local w=$1 tail40=$2 task meta verdict
|
|
261
|
+
task=$(window_to_task "$w" "$STATE")
|
|
262
|
+
meta="$STATE/$task.meta"
|
|
263
|
+
if [ -n "$task" ] && [ -f "$meta" ]; then
|
|
264
|
+
verdict=$(fm_busy_classify_meta "$meta" "$task" "$STATE" "$tail40")
|
|
265
|
+
else
|
|
266
|
+
verdict=$(fm_busy_classify "$(window_backend "$w")" "$w" "$(window_harness "$w")" \
|
|
267
|
+
"${task:-unknown}" "$STATE" "$tail40")
|
|
268
|
+
fi
|
|
269
|
+
[ "${verdict%% *}" = busy ]
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
window_kind() {
|
|
273
|
+
local w=$1 meta kind
|
|
274
|
+
meta=$(fm_backend_meta_for_window "$w" "$STATE" 2>/dev/null || true)
|
|
275
|
+
if [ -n "$meta" ]; then
|
|
276
|
+
kind=$(grep '^kind=' "$meta" | cut -d= -f2- || true)
|
|
277
|
+
[ -n "$kind" ] || kind=ship
|
|
278
|
+
echo "$kind"
|
|
279
|
+
return 0
|
|
280
|
+
fi
|
|
281
|
+
echo unknown
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
# window_backend: the backend recorded in the meta whose window= matches <w>,
|
|
285
|
+
# defaulting to tmux (absent backend= means tmux; the P1 compatibility
|
|
286
|
+
# contract) when no matching meta carries the field, or none matches at all.
|
|
287
|
+
window_backend() {
|
|
288
|
+
local w=$1 meta backend
|
|
289
|
+
meta=$(fm_backend_meta_for_window "$w" "$STATE" 2>/dev/null || true)
|
|
290
|
+
if [ -n "$meta" ]; then
|
|
291
|
+
backend=$(grep '^backend=' "$meta" | cut -d= -f2- || true)
|
|
292
|
+
[ -n "$backend" ] || backend=tmux
|
|
293
|
+
echo "$backend"
|
|
294
|
+
return 0
|
|
295
|
+
fi
|
|
296
|
+
echo tmux
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
window_harness() {
|
|
300
|
+
local w=$1 meta
|
|
301
|
+
meta=$(fm_backend_meta_for_window "$w" "$STATE" 2>/dev/null || true)
|
|
302
|
+
[ -n "$meta" ] || return 0
|
|
303
|
+
grep '^harness=' "$meta" | cut -d= -f2- || true
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
window_label() {
|
|
307
|
+
local w=$1 task
|
|
308
|
+
task=$(window_to_task "$w" "$STATE")
|
|
309
|
+
[ -n "$task" ] && printf 'fm-%s' "$task"
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
# The ONE derivation of a window's per-window marker key: `:`, `/` and `.` become
|
|
313
|
+
# `_` so a window name is usable as a filename suffix. Every per-window file the
|
|
314
|
+
# watcher keeps is named by it (.hash-, .count-, .stale-, .stale-since-,
|
|
315
|
+
# .wedge-escalations-, .paused-*, .writing-*), and live homes hold those markers on
|
|
316
|
+
# disk under the current format, so the format lives here alone: a second copy is
|
|
317
|
+
# how a future change to it silently orphans a window's markers instead of clearing
|
|
318
|
+
# them. The helpers below take the derived key rather than re-deriving it, so one
|
|
319
|
+
# poll of one window derives it once.
|
|
320
|
+
window_key() { # <window>
|
|
321
|
+
local key=${1//:/_}
|
|
322
|
+
key=${key//\//_}
|
|
323
|
+
printf '%s' "${key//./_}"
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
# Steering-inbox loss detection, one cheap check per recorded window per poll.
|
|
327
|
+
# Quiet when healthy: an absent, empty, or handled inbox costs one directory
|
|
328
|
+
# glob and produces nothing. When the ladder (fm_task_inbox_due_action, the
|
|
329
|
+
# policy owner) reports a due action, a busy pane just waits - the record is
|
|
330
|
+
# durable and the worker will reach a turn boundary - an idle pane gets one
|
|
331
|
+
# delivery attempt, and a spent attempt budget surfaces as an ordinary stale
|
|
332
|
+
# wake for stuck-crewmate-recovery. If the attempt's ladder write fails while
|
|
333
|
+
# its record remains unhandled, that unwritable state surfaces through the same
|
|
334
|
+
# stale path instead of silently re-ringing forever; acknowledgement or teardown
|
|
335
|
+
# still makes the race quiet. The attempt is data-plane typing or a
|
|
336
|
+
# composer-protected skip, never a wake, so normal retries keep the watcher
|
|
337
|
+
# blocking. Runs for secondmates
|
|
338
|
+
# too: their pane-staleness exemption is about quiet panes being healthy,
|
|
339
|
+
# while an unacknowledged instruction past the ladder is a stuck steer.
|
|
340
|
+
inbox_steer_check() { # <window> <task>
|
|
341
|
+
local w=$1 task=$2 action verb rec count tail40 reason ring_rc
|
|
342
|
+
action=$(fm_task_inbox_due_action "$STATE" "$task") || return 0
|
|
343
|
+
verb=${action%% *}
|
|
344
|
+
[ "$verb" != quiet ] || return 0
|
|
345
|
+
rec=${action#* }
|
|
346
|
+
count=
|
|
347
|
+
case "$verb" in
|
|
348
|
+
escalate)
|
|
349
|
+
count=${rec##* }
|
|
350
|
+
rec=${rec% *}
|
|
351
|
+
;;
|
|
352
|
+
esac
|
|
353
|
+
tail40=$(fm_backend_capture "$(window_backend "$w")" "$w" 40 "$(window_label "$w")" 2>/dev/null) || tail40=
|
|
354
|
+
if window_is_busy "$w" "$tail40"; then
|
|
355
|
+
return 0
|
|
356
|
+
fi
|
|
357
|
+
case "$verb" in
|
|
358
|
+
ring)
|
|
359
|
+
ring_rc=0
|
|
360
|
+
fm_task_inbox_ring "$(window_backend "$w")" "$w" "$rec" "$(window_label "$w")" || ring_rc=$?
|
|
361
|
+
if ! fm_task_inbox_record_ring "$STATE" "$task" "$rec"; then
|
|
362
|
+
if [ ! -f "$rec" ]; then
|
|
363
|
+
fm_task_inbox_due_action "$STATE" "$task" >/dev/null || true
|
|
364
|
+
return 0
|
|
365
|
+
fi
|
|
366
|
+
if [ -d "${rec%/*}" ]; then
|
|
367
|
+
reason="stale: $w (steering-inbox ladder bookkeeping unwritable: ${rec%/*}/.ring-state cannot be written while $rec stays unhandled; the doorbell cannot advance toward escalation - inspect the inbox directory)"
|
|
368
|
+
fm_wake_append stale "$w" "$reason" || exit 1
|
|
369
|
+
wake "$reason"
|
|
370
|
+
fi
|
|
371
|
+
fi
|
|
372
|
+
triage_log "steer-inbox delivery attempt: $task ${rec##*/} result=$ring_rc"
|
|
373
|
+
;;
|
|
374
|
+
escalate)
|
|
375
|
+
reason="stale: $w (unread firstmate instruction: $rec still unhandled after $count doorbell delivery attempts with an idle pane; inspect the worker)"
|
|
376
|
+
if [ ! -d "${rec%/*}" ] || [ ! -f "$rec" ]; then
|
|
377
|
+
fm_task_inbox_due_action "$STATE" "$task" >/dev/null || true
|
|
378
|
+
return 0
|
|
379
|
+
fi
|
|
380
|
+
fm_wake_append stale "$w" "$reason" || exit 1
|
|
381
|
+
if ! fm_task_inbox_record_escalated "$STATE" "$task" "$rec"; then
|
|
382
|
+
echo "error: stale wake was queued for $task but its inbox escalation marker could not be written" >&2
|
|
383
|
+
exit 1
|
|
384
|
+
fi
|
|
385
|
+
wake "$reason"
|
|
386
|
+
;;
|
|
387
|
+
esac
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
# 0 (benign/absorb) if EVERY task in a no-verb "signal:" wake has positive work
|
|
391
|
+
# evidence; 1 otherwise. Each task may satisfy the authoritative working proof,
|
|
392
|
+
# or an eligible bare turn-end may use the opt-in pane-churn proof below.
|
|
393
|
+
#
|
|
394
|
+
# OFF unless the home creates config/turnend-churn-absorb. The first two proofs
|
|
395
|
+
# read a verdict the harness itself vouches for; this one infers execution from
|
|
396
|
+
# rendered bytes, which is weaker, so widening the absorb is a home's choice to
|
|
397
|
+
# make rather than a default every fleet inherits. With the flag absent this
|
|
398
|
+
# delegates to the unchanged all-tasks authoritative proof.
|
|
399
|
+
#
|
|
400
|
+
# It exists because the first two are unreachable for a harness whose semantic
|
|
401
|
+
# busy state has no verified source: bin/fm-crew-state.sh can only answer unknown
|
|
402
|
+
# for such an adapter, crew_is_provably_working is therefore never satisfiable,
|
|
403
|
+
# and every worker turn boundary surfaced a wake with nothing to act on - the cost
|
|
404
|
+
# scaling with the number of workers in flight. Pane churn needs no harness
|
|
405
|
+
# cooperation, so it restores the absorb branch for those adapters without
|
|
406
|
+
# fabricating a busy verdict any adapter has not earned.
|
|
407
|
+
#
|
|
408
|
+
# The evidence is the one the pane-staleness backbone below already trusts for
|
|
409
|
+
# liveness: this compares a fresh capture against the .hash- marker that backbone
|
|
410
|
+
# recorded on the previous poll, which is why the derivation lives here with the
|
|
411
|
+
# marker format rather than in the shared classifier. Absorbing here DEFERS a wake
|
|
412
|
+
# rather than swallowing it, and the deferral is BOUNDED: a task's turn-ends may
|
|
413
|
+
# ride churn evidence for at most FM_TURNEND_CHURN_ABSORB_SECS, tracked per window
|
|
414
|
+
# in .churn-since-, after which the wake surfaces and the window restarts. The
|
|
415
|
+
# bound is what keeps churn from muting supervision outright. A pane that renders
|
|
416
|
+
# continuously - a clock, a spinner, a shell heartbeat, a harness that leaves a
|
|
417
|
+
# background renderer alive after its agent yields - never presents the two
|
|
418
|
+
# identical consecutive hashes the staleness backbone needs either, so without the
|
|
419
|
+
# bound a worker that had genuinely stopped behind such a renderer would be
|
|
420
|
+
# deferred here forever with no fallback path left to surface it. Churn and
|
|
421
|
+
# staleness read the same pane, so neither can be the other's only backstop.
|
|
422
|
+
# Within the bound, an ordinary crew that stops renders nothing more, its pane
|
|
423
|
+
# hash stops moving, and the staleness backbone surfaces it within a couple of
|
|
424
|
+
# polls; any captain-relevant status verb still surfaces immediately through
|
|
425
|
+
# signal_files_actionable. That is why this widens the proof instead of
|
|
426
|
+
# bounding the wake rate, which would have suppressed genuinely stopped workers.
|
|
427
|
+
#
|
|
428
|
+
# Every negative outcome returns 1, so absence of evidence surfaces exactly as
|
|
429
|
+
# before: any batch that references a secondmate, an unresolvable task, a task
|
|
430
|
+
# with no uniquely attributable recorded endpoint, no previous hash to compare
|
|
431
|
+
# against (nothing has been polled yet), a capture that fails or comes back empty,
|
|
432
|
+
# an exhausted deferral bound, and of course an unchanged pane. Any .status file
|
|
433
|
+
# also returns 1: an authored append is content the
|
|
434
|
+
# supervisor may need to read, so only the mechanical turn-end marker gets the
|
|
435
|
+
# fallback.
|
|
436
|
+
#
|
|
437
|
+
# NOT a pure read: one bounded pane capture per referenced task that lacks
|
|
438
|
+
# authoritative proof. Once EVERY task passes, each churn-proven pane's prior
|
|
439
|
+
# .stale- classification and wedge-escalation count are cleared because churn
|
|
440
|
+
# begins a new quiet interval; retaining either would make the new interval
|
|
441
|
+
# inherit the prior one. Reached only for a non-afk, no-captain-verb signal, so
|
|
442
|
+
# it never runs on the ordinary per-wake path.
|
|
443
|
+
signal_turnend_panes_churned() { # <file> ...
|
|
444
|
+
[ -e "$CONFIG/turnend-churn-absorb" ] || return 1
|
|
445
|
+
local f base task meta kind w key backend label terminal prev now since now_s absorb_secs marker age
|
|
446
|
+
local rec_task task_index i j count hash_file hash_bytes created
|
|
447
|
+
local max_absorb_secs=9223372036854775807
|
|
448
|
+
local -a signal_tasks=() signal_statuses=() snapshot_tasks=() snapshot_kinds=()
|
|
449
|
+
local -a snapshot_windows=() snapshot_keys=() snapshot_backends=() snapshot_labels=()
|
|
450
|
+
local -a signal_indexes=() churn_indexes=() churned_keys=() missing_keys=() created_keys=()
|
|
451
|
+
[ "$#" -gt 0 ] || return 1
|
|
452
|
+
for f in "$@"; do
|
|
453
|
+
base=${f##*/}
|
|
454
|
+
case "$base" in
|
|
455
|
+
*.status) return 1 ;;
|
|
456
|
+
*.turn-ended) task=${base%.turn-ended}; kind=turn-ended ;;
|
|
457
|
+
*) return 1 ;;
|
|
458
|
+
esac
|
|
459
|
+
[ -n "$task" ] || return 1
|
|
460
|
+
task_index=-1
|
|
461
|
+
for ((i = 0; i < ${#signal_tasks[@]}; i++)); do
|
|
462
|
+
[ "${signal_tasks[$i]}" = "$task" ] && { task_index=$i; break; }
|
|
463
|
+
done
|
|
464
|
+
if [ "$task_index" -lt 0 ]; then
|
|
465
|
+
signal_tasks+=("$task")
|
|
466
|
+
[ "$kind" = status ] && signal_statuses+=(1) || signal_statuses+=(0)
|
|
467
|
+
elif [ "$kind" = status ]; then
|
|
468
|
+
signal_statuses[task_index]=1
|
|
469
|
+
fi
|
|
470
|
+
done
|
|
471
|
+
for meta in "$STATE"/*.meta; do
|
|
472
|
+
[ -e "$meta" ] || continue
|
|
473
|
+
rec_task=${meta##*/}
|
|
474
|
+
rec_task=${rec_task%.meta}
|
|
475
|
+
kind=$(fm_meta_get "$meta" kind)
|
|
476
|
+
backend=$(fm_backend_of_meta "$meta")
|
|
477
|
+
if [ "$backend" = orca ]; then
|
|
478
|
+
terminal=$(fm_meta_get "$meta" terminal)
|
|
479
|
+
w=${terminal:-$(fm_meta_get "$meta" window)}
|
|
480
|
+
else
|
|
481
|
+
w=$(fm_meta_get "$meta" window)
|
|
482
|
+
fi
|
|
483
|
+
key=
|
|
484
|
+
[ -n "$w" ] && key=$(window_key "$w")
|
|
485
|
+
label="fm-$rec_task"
|
|
486
|
+
snapshot_tasks+=("$rec_task")
|
|
487
|
+
snapshot_kinds+=("$kind")
|
|
488
|
+
snapshot_windows+=("$w")
|
|
489
|
+
snapshot_keys+=("$key")
|
|
490
|
+
snapshot_backends+=("$backend")
|
|
491
|
+
snapshot_labels+=("$label")
|
|
492
|
+
done
|
|
493
|
+
# These linear lookups deliberately support stock macOS Bash 3.2.57, enforced
|
|
494
|
+
# by macos-stock-bash, and this repository uses no associative arrays in bin/
|
|
495
|
+
# or tests/. A batch is normally one to three tasks and captures dominate its
|
|
496
|
+
# cost; indexed lookup is the upgrade path if coalesced batches grow large.
|
|
497
|
+
for task in "${signal_tasks[@]}"; do
|
|
498
|
+
task_index=-1
|
|
499
|
+
for ((i = 0; i < ${#snapshot_tasks[@]}; i++)); do
|
|
500
|
+
[ "${snapshot_tasks[$i]}" = "$task" ] && { task_index=$i; break; }
|
|
501
|
+
done
|
|
502
|
+
[ "$task_index" -ge 0 ] || return 1
|
|
503
|
+
w=${snapshot_windows[$task_index]}
|
|
504
|
+
key=${snapshot_keys[$task_index]}
|
|
505
|
+
[ -n "$w" ] && [ -n "$key" ] || return 1
|
|
506
|
+
count=0
|
|
507
|
+
for ((j = 0; j < ${#snapshot_keys[@]}; j++)); do
|
|
508
|
+
[ "${snapshot_keys[$j]}" = "$key" ] && count=$((count + 1))
|
|
509
|
+
done
|
|
510
|
+
[ "$count" -eq 1 ] || return 1
|
|
511
|
+
signal_indexes+=("$task_index")
|
|
512
|
+
done
|
|
513
|
+
for task_index in "${signal_indexes[@]}"; do
|
|
514
|
+
[ "${snapshot_kinds[$task_index]}" != secondmate ] || return 1
|
|
515
|
+
done
|
|
516
|
+
for ((i = 0; i < ${#signal_tasks[@]}; i++)); do
|
|
517
|
+
task=${signal_tasks[$i]}
|
|
518
|
+
crew_is_provably_working "$task" && continue
|
|
519
|
+
task_index=${signal_indexes[$i]}
|
|
520
|
+
churn_indexes+=("$task_index")
|
|
521
|
+
done
|
|
522
|
+
[ "${#churn_indexes[@]}" -gt 0 ] || return 0
|
|
523
|
+
[[ $TURNEND_CHURN_ABSORB_SECS =~ ^[1-9][0-9]*$ ]] || return 1
|
|
524
|
+
if [ "${#TURNEND_CHURN_ABSORB_SECS}" -gt "${#max_absorb_secs}" ] \
|
|
525
|
+
|| { [ "${#TURNEND_CHURN_ABSORB_SECS}" -eq "${#max_absorb_secs}" ] \
|
|
526
|
+
&& [[ $TURNEND_CHURN_ABSORB_SECS -gt $max_absorb_secs ]]; }; then
|
|
527
|
+
return 1
|
|
528
|
+
fi
|
|
529
|
+
absorb_secs=$((10#$TURNEND_CHURN_ABSORB_SECS))
|
|
530
|
+
for task_index in "${churn_indexes[@]}"; do
|
|
531
|
+
w=${snapshot_windows[$task_index]}
|
|
532
|
+
key=${snapshot_keys[$task_index]}
|
|
533
|
+
backend=${snapshot_backends[$task_index]}
|
|
534
|
+
label=${snapshot_labels[$task_index]}
|
|
535
|
+
hash_file="$STATE/.hash-$key"
|
|
536
|
+
hash_bytes=$(LC_ALL=C wc -c 2>/dev/null < "$hash_file") || return 1
|
|
537
|
+
hash_bytes=${hash_bytes//[[:space:]]/}
|
|
538
|
+
[ "$hash_bytes" = 32 ] || return 1
|
|
539
|
+
prev=$(cat "$hash_file" 2>/dev/null) || return 1
|
|
540
|
+
[[ $prev =~ ^[0-9a-f]{32}$ ]] || return 1
|
|
541
|
+
now=$(fm_backend_capture "$backend" "$w" 40 "$label" 2>/dev/null) || return 1
|
|
542
|
+
[ -n "$now" ] || return 1
|
|
543
|
+
[ "$(printf '%s' "$now" | hash_pane)" != "$prev" ] || return 1
|
|
544
|
+
churned_keys+=("$key")
|
|
545
|
+
done
|
|
546
|
+
# Enforce the deferral bound BEFORE any .stale- state is touched, so a wake that
|
|
547
|
+
# surfaces here leaves the staleness backbone's own classification alone.
|
|
548
|
+
now_s=$(date +%s)
|
|
549
|
+
for key in "${churned_keys[@]}"; do
|
|
550
|
+
marker="$STATE/.churn-since-$key"
|
|
551
|
+
if [ ! -e "$marker" ]; then
|
|
552
|
+
[ ! -L "$marker" ] || return 1
|
|
553
|
+
missing_keys+=("$key")
|
|
554
|
+
continue
|
|
555
|
+
fi
|
|
556
|
+
since=$(cat "$marker" 2>/dev/null) || return 1
|
|
557
|
+
[[ $since =~ ^(0|[1-9][0-9]*)$ ]] || return 1
|
|
558
|
+
if [ "${#since}" -gt "${#now_s}" ] \
|
|
559
|
+
|| { [ "${#since}" -eq "${#now_s}" ] && [[ $since > $now_s ]]; }; then
|
|
560
|
+
return 1
|
|
561
|
+
fi
|
|
562
|
+
age=$((10#$now_s - 10#$since))
|
|
563
|
+
if [ "$age" -ge "$absorb_secs" ]; then
|
|
564
|
+
rm -f "$marker"
|
|
565
|
+
return 1
|
|
566
|
+
fi
|
|
567
|
+
done
|
|
568
|
+
for key in "${missing_keys[@]}"; do
|
|
569
|
+
marker="$STATE/.churn-since-$key"
|
|
570
|
+
if (set -C; printf '%s' "$now_s" > "$marker") 2>/dev/null; then
|
|
571
|
+
created_keys+=("$key")
|
|
572
|
+
continue
|
|
573
|
+
fi
|
|
574
|
+
for created in "${created_keys[@]}"; do
|
|
575
|
+
rm -f "$STATE/.churn-since-$created"
|
|
576
|
+
done
|
|
577
|
+
return 1
|
|
578
|
+
done
|
|
579
|
+
for key in "${churned_keys[@]}"; do
|
|
580
|
+
if ! rm -f "$STATE/.stale-$key" "$STATE/.wedge-escalations-$key"; then
|
|
581
|
+
for created in "${created_keys[@]}"; do
|
|
582
|
+
rm -f "$STATE/.churn-since-$created"
|
|
583
|
+
done
|
|
584
|
+
return 1
|
|
585
|
+
fi
|
|
586
|
+
done
|
|
587
|
+
return 0
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
recorded_windows() {
|
|
591
|
+
local meta w seen=
|
|
592
|
+
for meta in "$STATE"/*.meta; do
|
|
593
|
+
[ -e "$meta" ] || continue
|
|
594
|
+
w=$(fm_backend_target_of_meta "$meta")
|
|
595
|
+
[ -n "$w" ] || continue
|
|
596
|
+
case "$seen" in
|
|
597
|
+
*"|$w|"*) continue ;;
|
|
598
|
+
esac
|
|
599
|
+
seen="$seen|$w|"
|
|
600
|
+
printf '%s\n' "$w"
|
|
601
|
+
done
|
|
602
|
+
}
|
|
603
|
+
|
|
604
|
+
# Print the oldest structurally valid row in a local secondmate's foreign queue.
|
|
605
|
+
# This is a read-only observation: the receiving home owns acknowledgement and
|
|
606
|
+
# this parent never changes the row or the foreign queue.
|
|
607
|
+
secondmate_oldest_queue_row() { # <queue-path>
|
|
608
|
+
local queue=$1
|
|
609
|
+
[ -f "$queue" ] && [ ! -L "$queue" ] || return 0
|
|
610
|
+
awk -F '\t' '
|
|
611
|
+
NF >= 5 && $1 ~ /^[0-9]+$/ && $2 ~ /^[0-9]+$/ {
|
|
612
|
+
if (!found || $2 < seq) {
|
|
613
|
+
found = 1
|
|
614
|
+
seq = $2
|
|
615
|
+
row = $0
|
|
616
|
+
}
|
|
617
|
+
}
|
|
618
|
+
END { if (found) print row }
|
|
619
|
+
' "$queue" 2>/dev/null || true
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
# Surface one durable parent check for one unchanged foreign row after its
|
|
623
|
+
# bounded age. The primary marker and queued-key check make repeated watcher
|
|
624
|
+
# cycles converge without a notification storm, while an empty queue removes
|
|
625
|
+
# only this home's marker so a later row can be observed.
|
|
626
|
+
secondmate_wake_stall_tick() {
|
|
627
|
+
local now=$(( $(date +%s) )) threshold=$SECONDMATE_WAKE_STALL_SECS
|
|
628
|
+
local meta task kind remote_host home queue row epoch seq row_key marker receipt receipt_dir notify_key queued age reason
|
|
629
|
+
case "$threshold" in ''|*[!0-9]*|0) threshold=60 ;; esac
|
|
630
|
+
# Endpoint metadata admits this queue-loop check; secondmate-liveness owns registered mates whose endpoint is missing or dead.
|
|
631
|
+
for meta in "$STATE"/*.meta; do
|
|
632
|
+
[ -e "$meta" ] || continue
|
|
633
|
+
kind=$(fm_meta_get "$meta" kind)
|
|
634
|
+
[ "$kind" = secondmate ] || continue
|
|
635
|
+
remote_host=$(fm_meta_get "$meta" remote_host)
|
|
636
|
+
[ -z "$remote_host" ] || continue
|
|
637
|
+
task=${meta##*/}
|
|
638
|
+
task=${task%.meta}
|
|
639
|
+
case "$task" in ''|*[!A-Za-z0-9._-]*) continue ;; esac
|
|
640
|
+
home=$(fm_meta_get "$meta" home)
|
|
641
|
+
[ -n "$home" ] || continue
|
|
642
|
+
[ -f "$home/.fm-secondmate-home" ] && [ ! -L "$home/.fm-secondmate-home" ] || continue
|
|
643
|
+
[ "$(cat "$home/.fm-secondmate-home" 2>/dev/null || true)" = "$task" ] || continue
|
|
644
|
+
queue="$home/state/.wake-queue"
|
|
645
|
+
row=$(secondmate_oldest_queue_row "$queue")
|
|
646
|
+
marker="$STATE/.secondmate-wake-stall-$task"
|
|
647
|
+
receipt_dir="$STATE/.secondmate-wake-stall-receipts/$task"
|
|
648
|
+
if [ -z "$row" ]; then
|
|
649
|
+
rm -f "$marker"
|
|
650
|
+
if [ -e "$receipt_dir" ] || [ -L "$receipt_dir" ]; then
|
|
651
|
+
[ -d "$receipt_dir" ] && [ ! -L "$receipt_dir" ] || return 1
|
|
652
|
+
rm -rf -- "$receipt_dir" || return 1
|
|
653
|
+
fi
|
|
654
|
+
continue
|
|
655
|
+
fi
|
|
656
|
+
IFS=$(printf '\t') read -r epoch seq _row_kind _row_key _row_payload <<EOF
|
|
657
|
+
$row
|
|
658
|
+
EOF
|
|
659
|
+
case "$epoch" in ''|*[!0-9]*) continue ;; esac
|
|
660
|
+
case "$seq" in ''|*[!0-9]*) continue ;; esac
|
|
661
|
+
age=$((now - epoch))
|
|
662
|
+
[ "$age" -ge "$threshold" ] || continue
|
|
663
|
+
row_key="$epoch-$seq"
|
|
664
|
+
receipt="$receipt_dir/$row_key"
|
|
665
|
+
if [ -e "$marker" ] || [ -L "$marker" ]; then
|
|
666
|
+
[ -f "$marker" ] && [ ! -L "$marker" ] || return 1
|
|
667
|
+
fi
|
|
668
|
+
[ "$(cat "$marker" 2>/dev/null || true)" = "$row_key" ] && continue
|
|
669
|
+
[ "$(cat "$receipt" 2>/dev/null || true)" = "$row_key" ] && continue
|
|
670
|
+
notify_key="secondmate-wake-loop-$task-$row_key"
|
|
671
|
+
reason="check: secondmate wake-loop stalled: mate=$task row=$seq age=${age}s"
|
|
672
|
+
queued=$(fm_wake_queued_keys check)
|
|
673
|
+
if ! printf '%s\n' "$queued" | grep -Fx "$notify_key" >/dev/null 2>&1; then
|
|
674
|
+
fm_wake_append check "$notify_key" "$reason" || return 1
|
|
675
|
+
fi
|
|
676
|
+
fm_wake_secondmate_stall_receipt_write "$task" "$row_key" || return 1
|
|
677
|
+
fm_wake_secondmate_stall_marker_write "$task" "$row_key" || return 1
|
|
678
|
+
wake "$reason"
|
|
679
|
+
done
|
|
680
|
+
return 0
|
|
681
|
+
}
|
|
682
|
+
|
|
683
|
+
# Consecutive wedge-escalation count for a window past FM_WEDGE_DEMAND_INSPECT_COUNT
|
|
684
|
+
# (default 3): a pane that keeps re-wedging on the SAME stale hash - each
|
|
685
|
+
# escalation gets absorbed again as "still validating" one poll later, since the
|
|
686
|
+
# hash never changes - can otherwise repeat forever with no signal that this is
|
|
687
|
+
# no longer a one-off. At the threshold, wedge_timer_check appends a
|
|
688
|
+
# "demand-deep-inspection" marker to the wake payload so the wake reason itself
|
|
689
|
+
# (not just repetition the supervisor has to notice on its own) forces a closer
|
|
690
|
+
# look instead of another routine supervision resume. Reset wherever a window's
|
|
691
|
+
# pane/hash state resets to genuinely active (see the two rm-on-reset call sites
|
|
692
|
+
# below).
|
|
693
|
+
FM_WEDGE_DEMAND_INSPECT_COUNT=${FM_WEDGE_DEMAND_INSPECT_COUNT:-3}
|
|
694
|
+
|
|
695
|
+
# One bounded re-surface for a pane the watcher is deliberately absorbing, so no
|
|
696
|
+
# absorb can rot invisibly. <age> is how long the current absorb has held and
|
|
697
|
+
# <throttle> is the per-window marker whose mtime records the last re-surface, so
|
|
698
|
+
# once past PAUSE_RESURFACE_SECS the pane wakes once per window rather than every
|
|
699
|
+
# poll. Shared by the declared-pause absorb and the worktree-write deferral so the
|
|
700
|
+
# two cadences cannot drift apart; each caller owns its own marker and reason.
|
|
701
|
+
# Returns without waking while either the absorb or the throttle is inside the
|
|
702
|
+
# window; wake() itself exits the cycle, exactly as it does inline.
|
|
703
|
+
resurface_absorbed() { # <window> <throttle-marker> <age> <reason>
|
|
704
|
+
local win=$1 throttle=$2 age=$3 reason=$4
|
|
705
|
+
[ "$age" -ge "$PAUSE_RESURFACE_SECS" ] || return 0
|
|
706
|
+
[ "$(age_of "$throttle")" -ge "$PAUSE_RESURFACE_SECS" ] || return 0 # 999999 when no prior re-surface
|
|
707
|
+
fm_wake_append stale "$win" "$reason" || exit 1
|
|
708
|
+
date +%s > "$throttle"
|
|
709
|
+
wake "$reason"
|
|
710
|
+
}
|
|
711
|
+
|
|
712
|
+
# Defer ONE wedge escalation for a pane that went quiet while its own task
|
|
713
|
+
# worktree is demonstrably still being written (crew_worktree_written_since in
|
|
714
|
+
# fm-classify-lib.sh). The pane and the run step both say nothing is happening;
|
|
715
|
+
# the worktree says otherwise, and files appearing in it is the harder signal to
|
|
716
|
+
# fake, so the escalation is deferred rather than fired. Deliberately a DEFERRAL,
|
|
717
|
+
# not a cancellation: the idle timer restarts, so the next window probes again,
|
|
718
|
+
# and a .writing-since-<key> marker ages the whole deferral chain so the pane
|
|
719
|
+
# still re-surfaces once every PAUSE_RESURFACE_SECS through the shared
|
|
720
|
+
# resurface_absorbed above - literally the same bounded cadence a declared pause
|
|
721
|
+
# uses, throttled by its own .writing-resurfaced-<key> marker - and a crew whose
|
|
722
|
+
# worktree churns without real progress cannot stay invisible. The escalation
|
|
723
|
+
# counter is left alone: it is neither advanced (this is not an escalation) nor
|
|
724
|
+
# reset (a later genuine escalation must still carry the demand-deep-inspection
|
|
725
|
+
# history it had already earned).
|
|
726
|
+
wedge_defer_writing() { # <window> <since-file> <triage-label> <idle-age>
|
|
727
|
+
local win=$1 since_file=$2 label=$3 age=$4 key wsf wage
|
|
728
|
+
key=$(window_key "$win")
|
|
729
|
+
wsf="$STATE/.writing-since-$key"
|
|
730
|
+
[ -e "$wsf" ] || date +%s > "$wsf"
|
|
731
|
+
wage=$(age_of "$wsf")
|
|
732
|
+
date +%s > "$since_file"
|
|
733
|
+
resurface_absorbed "$win" "$STATE/.writing-resurfaced-$key" "$wage" \
|
|
734
|
+
"stale: $win (idle ${age}s, writing its worktree for ${wage}s, rechecked on a long cadence not a wedge; confirm the writes are real progress)"
|
|
735
|
+
triage_log "absorbed $label (worktree written since the idle window opened, idle ${age}s): $win"
|
|
736
|
+
}
|
|
737
|
+
|
|
738
|
+
# Drop a window's write-deferral chain wherever its stale bookkeeping resets, so
|
|
739
|
+
# the bounded re-surface cadence is measured from the CURRENT quiet stretch and a
|
|
740
|
+
# long-finished one cannot make the next deferral resurface immediately.
|
|
741
|
+
clear_write_tracking() { # <window-key>
|
|
742
|
+
local key=$1
|
|
743
|
+
rm -f "$STATE/.writing-since-$key" "$STATE/.writing-resurfaced-$key"
|
|
744
|
+
}
|
|
745
|
+
|
|
746
|
+
# Repeat-poll wedge-timer bookkeeping for an already-classified stale hash
|
|
747
|
+
# absorbed as provably-working - repairs a missing/corrupt timer (self-heals a
|
|
748
|
+
# watcher restart between recording the hash and recording the timer), or
|
|
749
|
+
# escalates once STALE_ESCALATE_SECS have elapsed. Never re-reads the crew
|
|
750
|
+
# state (the costly check already ran once, at classification time). Shared by
|
|
751
|
+
# both places a hash can be absorbed this way: the plain non-terminal path,
|
|
752
|
+
# and the stale_is_terminal-overridden path (a captain-relevant status-log
|
|
753
|
+
# line that an active run/busy pane outranked).
|
|
754
|
+
# The worktree write probe runs ONLY here, inside the at-threshold branch that is
|
|
755
|
+
# about to escalate: at most one bounded walk per window per STALE_ESCALATE_SECS,
|
|
756
|
+
# never per poll.
|
|
757
|
+
wedge_timer_check() { # <window> <since-file> <triage-label> <escalation-count-file> <task>
|
|
758
|
+
local win=$1 since_file=$2 label=$3 escalation_file=$4 task=$5 since age n reason
|
|
759
|
+
since=$(cat "$since_file" 2>/dev/null || true)
|
|
760
|
+
case "$since" in
|
|
761
|
+
''|*[!0-9]*)
|
|
762
|
+
# Publish the repaired timer only after its old write-deferral chain is
|
|
763
|
+
# gone, so observers cannot mistake a new idle window for the old chain.
|
|
764
|
+
clear_write_tracking "$(window_key "$win")"
|
|
765
|
+
date +%s > "$since_file"
|
|
766
|
+
triage_log "absorbed $label timer reset: $win"
|
|
767
|
+
;;
|
|
768
|
+
*)
|
|
769
|
+
age=$(( $(date +%s) - since ))
|
|
770
|
+
if [ "$age" -ge "$STALE_ESCALATE_SECS" ]; then
|
|
771
|
+
if crew_worktree_written_since "$task" "$STATE" "$since_file"; then
|
|
772
|
+
wedge_defer_writing "$win" "$since_file" "$label" "$age"
|
|
773
|
+
return 0
|
|
774
|
+
fi
|
|
775
|
+
n=$(( $(cat "$escalation_file" 2>/dev/null || echo 0) + 1 ))
|
|
776
|
+
echo "$n" > "$escalation_file"
|
|
777
|
+
reason="stale: $win (idle ${age}s, possible wedge, escalation $n)"
|
|
778
|
+
if [ "$n" -ge "$FM_WEDGE_DEMAND_INSPECT_COUNT" ]; then
|
|
779
|
+
reason="stale: $win (idle ${age}s, possible wedge, escalation $n, demand-deep-inspection: same pane has wedge-escalated $n times in a row - do not re-absorb on the run-step/pane state alone)"
|
|
780
|
+
fi
|
|
781
|
+
fm_wake_append stale "$win" "$reason" || exit 1
|
|
782
|
+
rm -f "$since_file"
|
|
783
|
+
clear_write_tracking "$(window_key "$win")"
|
|
784
|
+
wake "$reason"
|
|
785
|
+
fi
|
|
786
|
+
;;
|
|
787
|
+
esac
|
|
788
|
+
}
|
|
789
|
+
|
|
790
|
+
# busy_turn_over_age: 0 iff <task>'s latest completed-turn marker is at least
|
|
791
|
+
# BUSY_TURN_MAX_SECS old. Ages the per-task turn-ended marker, the harness-neutral
|
|
792
|
+
# signal every verified harness's turn-end hook touches; before any turn has
|
|
793
|
+
# completed, ages the task's spawn record instead so a fresh task still gets a
|
|
794
|
+
# bound. The caller checks that the pane is busy and routes a crossed bound
|
|
795
|
+
# through busy_turn_bound_check, never anything that touches the worker itself.
|
|
796
|
+
busy_turn_over_age() { # <task>
|
|
797
|
+
local task=$1 f
|
|
798
|
+
f="$STATE/$task.turn-ended"
|
|
799
|
+
[ -e "$f" ] || f="$STATE/$task.meta"
|
|
800
|
+
[ "$(age_of "$f")" -ge "$BUSY_TURN_MAX_SECS" ]
|
|
801
|
+
}
|
|
802
|
+
|
|
803
|
+
# Absorb a stale pane under a declared external-wait pause (paused:) or a
|
|
804
|
+
# dead-agent captain-held transfer, and re-surface it once every
|
|
805
|
+
# PAUSE_RESURFACE_SECS for a recheck so it cannot rot invisibly. Called on any
|
|
806
|
+
# stale poll once pause_state_class permits the bounded cadence, so it must be
|
|
807
|
+
# cheap: it NEVER re-reads crew state. The re-surface age is anchored on the
|
|
808
|
+
# status file mtime, not a per-hash marker, so a churny idle pane (a ticking
|
|
809
|
+
# clock, a token counter) cannot keep resetting the cadence the way a hash-tied
|
|
810
|
+
# timer would. The bounded re-surface itself is the shared resurface_absorbed
|
|
811
|
+
# above, throttled by this window's own .paused-resurfaced-<key> marker. Advances
|
|
812
|
+
# the stale suppressor to <hash> and flags the key paused.
|
|
813
|
+
#
|
|
814
|
+
# The recheck names WHICH human the declared wait is on, because that is the whole
|
|
815
|
+
# point of a recheck the captain reads: an external dependency for paused:, and the
|
|
816
|
+
# captain themself for a verified hold. Only the captain-held verb takes the second
|
|
817
|
+
# wording; a caller that reached the bounded cadence off pause tracking alone, with
|
|
818
|
+
# no declaring verb left on the log, keeps the external-wait wording it always had.
|
|
819
|
+
handle_paused_stale() { # <window> <task> <hash>
|
|
820
|
+
local win=$1 task=$2 h=$3 key statusf mtime age detail reason
|
|
821
|
+
key=$(window_key "$win")
|
|
822
|
+
printf '%s' "$h" > "$STATE/.stale-$key"
|
|
823
|
+
: > "$STATE/.paused-$key"
|
|
824
|
+
rm -f "$STATE/.stale-since-$key" "$STATE/.wedge-escalations-$key"
|
|
825
|
+
clear_write_tracking "$key"
|
|
826
|
+
statusf="$STATE/$task.status"
|
|
827
|
+
mtime=$(stat_mtime "$statusf")
|
|
828
|
+
case "$mtime" in ''|*[!0-9]*) mtime=$(date +%s) ;; esac
|
|
829
|
+
age=$(( $(date +%s) - mtime ))
|
|
830
|
+
if status_is_captain_held "$(last_status_line "$statusf")"; then
|
|
831
|
+
detail="captain-held, awaiting the captain"
|
|
832
|
+
reason="captain-held ${age}s, awaiting the captain - verified hold transfer, rechecked on a long cadence not a wedge; answer the held decision or release the hold"
|
|
833
|
+
else
|
|
834
|
+
detail="paused, awaiting external"
|
|
835
|
+
reason="paused ${age}s, awaiting external - declared pause, rechecked on a long cadence not a wedge; confirm the wait still holds"
|
|
836
|
+
fi
|
|
837
|
+
resurface_absorbed "$win" "$STATE/.paused-resurfaced-$key" "$age" "stale: $win ($reason)"
|
|
838
|
+
triage_log "absorbed stale ($detail, age ${age}s): $win"
|
|
839
|
+
}
|
|
840
|
+
|
|
841
|
+
# Apply the busy-pane completed-turn bound to a window whose bound has already
|
|
842
|
+
# crossed, honoring the worker's OWN declared external wait. Prints/queues
|
|
843
|
+
# nothing itself; it only chooses which absorber owns the crossed bound.
|
|
844
|
+
# 0 when the declared-pause cadence took the pane, 1 when the wedge timer did.
|
|
845
|
+
#
|
|
846
|
+
# A busy pane past BUSY_TURN_MAX_SECS is normally a wedge suspect because a hung
|
|
847
|
+
# foreground call can hide behind a busy signature. A `paused:` declaration or
|
|
848
|
+
# verified captain-held transfer instead identifies that live foreground call as
|
|
849
|
+
# the expected external wait. The caller has already confirmed liveness through
|
|
850
|
+
# the busy verdict, so this exception does not suppress undeclared wedges or
|
|
851
|
+
# alter the separate non-busy classification. handle_paused_stale keeps the
|
|
852
|
+
# exception bounded by re-surfacing it once per PAUSE_RESURFACE_SECS. Away mode
|
|
853
|
+
# remains daemon-owned and receives the undecorated wake identity for its own
|
|
854
|
+
# classification, which is why the declaration is read before the afk branch
|
|
855
|
+
# rather than after it.
|
|
856
|
+
busy_turn_bound_check() { # <window> <task> <hash> <since-file> <escalation-file>
|
|
857
|
+
local win=$1 task=$2 h=$3 since_file=$4 escalation_file=$5 key statusf declared
|
|
858
|
+
statusf="$STATE/$task.status"
|
|
859
|
+
if status_is_paused_or_captain_held "$(last_status_line "$statusf")"; then
|
|
860
|
+
if afk_present; then
|
|
861
|
+
# Away mode is daemon-owned, so this bound hands off the PLAIN wake identity
|
|
862
|
+
# and lets the daemon classify the declaration itself - the undecorated
|
|
863
|
+
# identity the rest of this function's contract promises. Running the wedge
|
|
864
|
+
# timer here instead would decorate the wake as a possible wedge, and that
|
|
865
|
+
# decoration overrides the daemon's own pause verdict for the pane: the
|
|
866
|
+
# ladder then climbs on every re-arm, escalating a crew that declared the
|
|
867
|
+
# wait itself once per FM_STALE_ESCALATE_SECS for as long as the wait lasts.
|
|
868
|
+
# The one-shot is keyed on the DECLARATION (the status log's signature),
|
|
869
|
+
# never on the pane hash: a busy pane's harness footer ticks on every
|
|
870
|
+
# capture, so a hash-keyed one-shot would re-fire on every poll and the
|
|
871
|
+
# daemon, which relaunches the watcher after each handled wake, would be
|
|
872
|
+
# woken in a loop for the whole declared wait. The suppressor therefore
|
|
873
|
+
# advances to the declaration rather than the hash, and the daemon is woken
|
|
874
|
+
# once per distinct declaration. The wedge timer, escalation count and
|
|
875
|
+
# write-deferral chain are cleared exactly as handle_paused_stale clears
|
|
876
|
+
# them, so an undeclared busy phase that had already started the timer does
|
|
877
|
+
# not resume its count the moment the declaration is lifted. Normal-mode
|
|
878
|
+
# pause tracking stays unwritten here, exactly as the idle away-mode handoff
|
|
879
|
+
# leaves it, because the daemon owns that bookkeeping.
|
|
880
|
+
key=$(window_key "$win")
|
|
881
|
+
rm -f "$since_file" "$escalation_file"
|
|
882
|
+
clear_write_tracking "$key"
|
|
883
|
+
declared="declared:$(fm_wake_signal_sig "$statusf" || true)"
|
|
884
|
+
if [ "$(cat "$STATE/.stale-$key" 2>/dev/null || true)" != "$declared" ]; then
|
|
885
|
+
fm_wake_append stale "$win" "stale: $win" || exit 1
|
|
886
|
+
printf '%s' "$declared" > "$STATE/.stale-$key"
|
|
887
|
+
wake "stale: $win"
|
|
888
|
+
fi
|
|
889
|
+
return 0
|
|
890
|
+
fi
|
|
891
|
+
handle_paused_stale "$win" "$task" "$h"
|
|
892
|
+
return 0
|
|
893
|
+
fi
|
|
894
|
+
wedge_timer_check "$win" "$since_file" "busy (no completed turn)" "$escalation_file" "$task"
|
|
895
|
+
return 1
|
|
896
|
+
}
|
|
897
|
+
|
|
898
|
+
clear_pause_state() { # <window-key>
|
|
899
|
+
local key=$1
|
|
900
|
+
rm -f "$STATE/.paused-$key" "$STATE/.paused-rechecked-$key" "$STATE/.paused-resurfaced-$key"
|
|
901
|
+
}
|
|
902
|
+
|
|
903
|
+
clear_pause_tracking() { # <window-key>
|
|
904
|
+
local key=$1
|
|
905
|
+
clear_pause_state "$key"
|
|
906
|
+
clear_write_tracking "$key"
|
|
907
|
+
rm -f "$STATE/.stale-$key" "$STATE/.stale-since-$key" "$STATE/.wedge-escalations-$key"
|
|
908
|
+
}
|
|
909
|
+
|
|
910
|
+
# Reconcile a declared pause or captain-held status with authoritative crew state.
|
|
911
|
+
# After fm-crew-state has fallen back to stopped or unknown, paused classification is
|
|
912
|
+
# recovered only for a confidently dead ordinary crew, or for a secondmate, whose
|
|
913
|
+
# endpoint liveness this function deliberately never reads.
|
|
914
|
+
pause_state_class() { # <window> <task>
|
|
915
|
+
local win=$1 task=$2 key last recheck_file class agent_alive kind
|
|
916
|
+
key=$(window_key "$win")
|
|
917
|
+
last=$(last_status_line "$STATE/$task.status")
|
|
918
|
+
recheck_file="$STATE/.paused-rechecked-$key"
|
|
919
|
+
if ! status_is_paused_or_captain_held "$last"; then
|
|
920
|
+
rm -f "$recheck_file"
|
|
921
|
+
crew_absorb_class "$task"
|
|
922
|
+
return
|
|
923
|
+
fi
|
|
924
|
+
# Read once past the declared-wait gate and reused by both liveness gates below,
|
|
925
|
+
# so a mate's stale poll costs one metadata scan rather than one per gate, and the
|
|
926
|
+
# far more common no-declaration path above still costs none.
|
|
927
|
+
kind=$(window_kind "$win")
|
|
928
|
+
if [ -e "$STATE/.paused-$key" ] && [ "$(age_of "$recheck_file")" -lt "$STALE_ESCALATE_SECS" ]; then
|
|
929
|
+
if [ "$kind" != secondmate ]; then
|
|
930
|
+
agent_alive=$(fm_backend_agent_alive "$(window_backend "$win")" "$win" 2>/dev/null) || agent_alive=unknown
|
|
931
|
+
if [ "$agent_alive" != dead ]; then
|
|
932
|
+
rm -f "$recheck_file"
|
|
933
|
+
printf 'none'
|
|
934
|
+
return
|
|
935
|
+
fi
|
|
936
|
+
fi
|
|
937
|
+
printf 'paused'
|
|
938
|
+
return
|
|
939
|
+
fi
|
|
940
|
+
class=$(crew_absorb_class "$task")
|
|
941
|
+
if [ "$class" = working ]; then
|
|
942
|
+
rm -f "$recheck_file"
|
|
943
|
+
printf 'working'
|
|
944
|
+
return
|
|
945
|
+
fi
|
|
946
|
+
if [ "$kind" != secondmate ]; then
|
|
947
|
+
agent_alive=$(fm_backend_agent_alive "$(window_backend "$win")" "$win" 2>/dev/null) || agent_alive=unknown
|
|
948
|
+
if [ "$agent_alive" != dead ]; then
|
|
949
|
+
rm -f "$recheck_file"
|
|
950
|
+
printf 'none'
|
|
951
|
+
return
|
|
952
|
+
fi
|
|
953
|
+
fi
|
|
954
|
+
# Recover paused classification for a declared wait that authoritative crew state
|
|
955
|
+
# could not name. Reaching here already proves the only two admissible cases: an
|
|
956
|
+
# ordinary crew whose agent the gate above confirmed dead, so no live decision gate
|
|
957
|
+
# is being silenced, or a secondmate, whose endpoint liveness is deliberately never
|
|
958
|
+
# read and so cannot supply that confirmation. Without the mate case a mate's
|
|
959
|
+
# captain hold - which has no current-state mapping and so arrives as `none` -
|
|
960
|
+
# would be silenced by every caller rather than taking the bounded re-surface
|
|
961
|
+
# cadence, and a forgotten hold would rot invisibly.
|
|
962
|
+
[ "$class" = none ] && class=paused
|
|
963
|
+
case "$class" in
|
|
964
|
+
paused) date +%s > "$recheck_file" ;;
|
|
965
|
+
*) rm -f "$recheck_file" ;;
|
|
966
|
+
esac
|
|
967
|
+
printf '%s' "$class"
|
|
968
|
+
}
|
|
969
|
+
|
|
970
|
+
surface_nonterminal_stale() { # <window> <hash>
|
|
971
|
+
local win=$1 h=$2 key task last
|
|
972
|
+
key=$(window_key "$win")
|
|
973
|
+
fm_wake_append stale "$win" "stale: $win" || exit 1
|
|
974
|
+
printf '%s' "$h" > "$STATE/.stale-$key"
|
|
975
|
+
rm -f "$STATE/.stale-since-$key"
|
|
976
|
+
clear_write_tracking "$key"
|
|
977
|
+
task=$(window_to_task "$win" "$STATE")
|
|
978
|
+
last=$(last_status_line "$STATE/$task.status")
|
|
979
|
+
if status_is_paused_or_captain_held "$last"; then
|
|
980
|
+
: > "$STATE/.paused-$key"
|
|
981
|
+
date +%s > "$STATE/.paused-rechecked-$key"
|
|
982
|
+
date +%s > "$STATE/.paused-resurfaced-$key"
|
|
983
|
+
else
|
|
984
|
+
rm -f "$STATE/.paused-$key" "$STATE/.paused-rechecked-$key" "$STATE/.paused-resurfaced-$key"
|
|
985
|
+
fi
|
|
986
|
+
wake "stale: $win"
|
|
987
|
+
}
|
|
988
|
+
|
|
989
|
+
# Check and heartbeat cadence must survive actionable exits and restarts: the
|
|
990
|
+
# watcher may be relaunched before in-memory counters reach their threshold on a
|
|
991
|
+
# busy fleet. Persist the schedule as file mtimes instead.
|
|
992
|
+
age_of() { # seconds since file mtime; "due immediately" if missing
|
|
993
|
+
local f=$1 m now
|
|
994
|
+
m=$(stat_mtime "$f") || { echo 999999; return; }
|
|
995
|
+
now=$(date +%s)
|
|
996
|
+
[ "$m" -le "$now" ] || { echo 999999; return; }
|
|
997
|
+
echo $(( now - m ))
|
|
998
|
+
}
|
|
999
|
+
|
|
1000
|
+
# Layer 2 + 3 signal scan: status files and turn-end markers.
|
|
1001
|
+
# Each file is compared against its persisted reported signature in .seen-* rather
|
|
1002
|
+
# than mtime-vs-a-startup-touch, so signals that land while no watcher is running
|
|
1003
|
+
# are caught by the next one and same-second writes cannot slip through a strict
|
|
1004
|
+
# -nt comparison.
|
|
1005
|
+
# Status signatures include observable file and readability state, while turn-end
|
|
1006
|
+
# markers retain their size-and-mtime signature.
|
|
1007
|
+
# Pure read: prints one "<seen-file>\t<sig>\t<file>" line per changed file.
|
|
1008
|
+
# The caller records reported state only after surfacing or intentional absorption,
|
|
1009
|
+
# and commits a status classification position only after a successful span read.
|
|
1010
|
+
scan_signals() {
|
|
1011
|
+
local f sig sf
|
|
1012
|
+
for f in "$STATE"/*.status "$STATE"/*.turn-ended; do
|
|
1013
|
+
if [ ! -e "$f" ]; then
|
|
1014
|
+
case "$f" in *.status) [ -L "$f" ] || continue ;; *) continue ;; esac
|
|
1015
|
+
fi
|
|
1016
|
+
sig=$(fm_wake_signal_sig "$f") || continue
|
|
1017
|
+
[ -n "$sig" ] || continue
|
|
1018
|
+
sf=$(fm_wake_signal_seen_path "$STATE" "$f")
|
|
1019
|
+
case "$f" in
|
|
1020
|
+
*.status) fm_wake_signal_seen_current "$STATE" "$f" && continue ;;
|
|
1021
|
+
*) [ "$sig" = "$(cat "$sf" 2>/dev/null)" ] && continue ;;
|
|
1022
|
+
esac
|
|
1023
|
+
printf '%s\t%s\t%s\n' "$sf" "$sig" "$f"
|
|
1024
|
+
done
|
|
1025
|
+
return 0
|
|
1026
|
+
}
|
|
1027
|
+
|
|
1028
|
+
# Deliver a durably queued process-event result to firstmate. Publication is
|
|
1029
|
+
# owned by bin/fm-procevent.sh - by the runner at capture time and by reconcile's
|
|
1030
|
+
# re-announcement - so this decides only whether a queued check record has been
|
|
1031
|
+
# surfaced yet, then reports it through the same actionable exit every other wake
|
|
1032
|
+
# uses. Without it a captured result sits on the queue until something else
|
|
1033
|
+
# happens to wake firstmate, which is exactly the missed delivery this repairs.
|
|
1034
|
+
# Dedup uses the same .seen-* discipline as scan_signals: the durable record is
|
|
1035
|
+
# always written before its marker, so nothing is suppressed before it is queued,
|
|
1036
|
+
# and re-announcement, drain-time deduplication, and the handled acknowledgement
|
|
1037
|
+
# keep their existing owners untouched.
|
|
1038
|
+
procevent_surfaced_marker() { # <queue-key>
|
|
1039
|
+
printf '%s/.seen-procevent-%s' "$STATE" "$(printf '%s' "$1" | LC_ALL=C od -An -tx1 | tr -d ' \n')"
|
|
1040
|
+
}
|
|
1041
|
+
|
|
1042
|
+
procevent_surface_after_output() {
|
|
1043
|
+
local output_status=$1 key marker tmp status=0
|
|
1044
|
+
if [ "$output_status" -eq 0 ]; then
|
|
1045
|
+
for key in $PROCEVENT_SURFACED; do
|
|
1046
|
+
marker=$(procevent_surfaced_marker "$key")
|
|
1047
|
+
tmp=$(umask 077; mktemp "$STATE/.seen-procevent.XXXXXX") || { status=1; continue; }
|
|
1048
|
+
if ! mv -f -- "$tmp" "$marker"; then
|
|
1049
|
+
rm -f -- "$tmp"
|
|
1050
|
+
status=1
|
|
1051
|
+
fi
|
|
1052
|
+
done
|
|
1053
|
+
fi
|
|
1054
|
+
fm_lock_release "$FM_WAKE_QUEUE_LOCK"
|
|
1055
|
+
return "$status"
|
|
1056
|
+
}
|
|
1057
|
+
|
|
1058
|
+
procevent_surface_queued() {
|
|
1059
|
+
local key reason
|
|
1060
|
+
PROCEVENT_SURFACED=
|
|
1061
|
+
[ -s "$FM_WAKE_QUEUE" ] || return 0
|
|
1062
|
+
fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK"
|
|
1063
|
+
while IFS= read -r key; do
|
|
1064
|
+
case "$key" in procevent:*) ;; *) continue ;; esac
|
|
1065
|
+
[ -e "$(procevent_surfaced_marker "$key")" ] && continue
|
|
1066
|
+
PROCEVENT_SURFACED="$PROCEVENT_SURFACED $key"
|
|
1067
|
+
done < <(fm_wake_queued_keys_locked check)
|
|
1068
|
+
if [ -z "$PROCEVENT_SURFACED" ]; then
|
|
1069
|
+
fm_lock_release "$FM_WAKE_QUEUE_LOCK"
|
|
1070
|
+
return 0
|
|
1071
|
+
fi
|
|
1072
|
+
reason="check: process-event result captured:$PROCEVENT_SURFACED"
|
|
1073
|
+
# shellcheck disable=SC2034 # Consumed by wake() in the separately linted transition owner.
|
|
1074
|
+
FM_WAKE_POST_OUTPUT_ACTION=procevent_surface_after_output
|
|
1075
|
+
wake "$reason"
|
|
1076
|
+
}
|
|
1077
|
+
|
|
1078
|
+
run_check_process() {
|
|
1079
|
+
local c=$1
|
|
1080
|
+
shift
|
|
1081
|
+
if [ "${FM_CHECK_FORCE_FALLBACK:-0}" != 1 ] && command -v timeout >/dev/null 2>&1; then
|
|
1082
|
+
exec timeout "$CHECK_TIMEOUT" bash "$c" "$@"
|
|
1083
|
+
elif [ "${FM_CHECK_FORCE_FALLBACK:-0}" != 1 ] && command -v gtimeout >/dev/null 2>&1; then
|
|
1084
|
+
exec gtimeout "$CHECK_TIMEOUT" bash "$c" "$@"
|
|
1085
|
+
else
|
|
1086
|
+
# shellcheck disable=SC2016 # single quotes are deliberate: Perl expands its own variables.
|
|
1087
|
+
exec perl -e 'my $t = shift; my $owned = shift; my $pid = fork; die "fork failed" unless defined $pid; if (!$pid) { setpgrp(0, 0) unless $owned; exec @ARGV } my $group = $owned ? getpgrp(0) : $pid; my $stop = sub { $SIG{HUP} = $SIG{INT} = $SIG{TERM} = "IGNORE"; kill "TERM", -$group; select undef, undef, undef, 0.2; kill "KILL", -$group; waitpid $pid, 0; exit 124 }; local $SIG{ALRM} = $stop; local $SIG{HUP} = $stop; local $SIG{INT} = $stop; local $SIG{TERM} = $stop; alarm $t; waitpid $pid, 0; exit($? >> 8)' "$CHECK_TIMEOUT" "${FM_CHECK_OWNED_GROUP:-0}" bash "$c" "$@"
|
|
1088
|
+
fi
|
|
1089
|
+
}
|
|
1090
|
+
|
|
1091
|
+
run_check() {
|
|
1092
|
+
( run_check_process "$@" ) 2>/dev/null || true
|
|
1093
|
+
}
|
|
1094
|
+
|
|
1095
|
+
FM_ACTIVE_CHECK_PID=
|
|
1096
|
+
FM_ACTIVE_CHECK_PGID=
|
|
1097
|
+
FM_CHECK_OUTPUT=
|
|
1098
|
+
FM_CHECK_RESULT=
|
|
1099
|
+
FM_CHECK_SIGNAL_PENDING=
|
|
1100
|
+
|
|
1101
|
+
fm_check_output_cleanup() {
|
|
1102
|
+
[ -z "$FM_CHECK_OUTPUT" ] || rm -f -- "$FM_CHECK_OUTPUT"
|
|
1103
|
+
FM_CHECK_OUTPUT=
|
|
1104
|
+
}
|
|
1105
|
+
|
|
1106
|
+
fm_active_check_stop() {
|
|
1107
|
+
local pid=${FM_ACTIVE_CHECK_PID:-} pgid=${FM_ACTIVE_CHECK_PGID:-} i
|
|
1108
|
+
[ -n "$pid" ] || [ -n "$pgid" ] || return 0
|
|
1109
|
+
[ -z "$pgid" ] || kill -TERM -- "-$pgid" 2>/dev/null || true
|
|
1110
|
+
[ -z "$pid" ] || kill -TERM "$pid" 2>/dev/null || true
|
|
1111
|
+
i=0
|
|
1112
|
+
while [ -n "$pgid" ] && kill -0 -- "-$pgid" 2>/dev/null && [ "$i" -lt 20 ]; do
|
|
1113
|
+
sleep 0.01
|
|
1114
|
+
i=$((i + 1))
|
|
1115
|
+
done
|
|
1116
|
+
[ -z "$pgid" ] || kill -KILL -- "-$pgid" 2>/dev/null || true
|
|
1117
|
+
[ -z "$pid" ] || kill -KILL "$pid" 2>/dev/null || true
|
|
1118
|
+
[ -z "$pid" ] || wait "$pid" 2>/dev/null || true
|
|
1119
|
+
i=0
|
|
1120
|
+
while [ -n "$pgid" ] && kill -0 -- "-$pgid" 2>/dev/null && [ "$i" -lt 100 ]; do
|
|
1121
|
+
sleep 0.01
|
|
1122
|
+
i=$((i + 1))
|
|
1123
|
+
done
|
|
1124
|
+
if [ -n "$pgid" ] && kill -0 -- "-$pgid" 2>/dev/null; then
|
|
1125
|
+
return 1
|
|
1126
|
+
fi
|
|
1127
|
+
FM_ACTIVE_CHECK_PID=
|
|
1128
|
+
FM_ACTIVE_CHECK_PGID=
|
|
1129
|
+
}
|
|
1130
|
+
|
|
1131
|
+
run_check_capture() {
|
|
1132
|
+
local pgid
|
|
1133
|
+
fm_check_output_cleanup
|
|
1134
|
+
FM_CHECK_RESULT=
|
|
1135
|
+
FM_CHECK_OUTPUT=$(mktemp "$STATE/.fm-check-output.XXXXXX") || return 1
|
|
1136
|
+
chmod 0600 "$FM_CHECK_OUTPUT" || { fm_check_output_cleanup; return 1; }
|
|
1137
|
+
FM_CHECK_SIGNAL_PENDING=
|
|
1138
|
+
trap 'FM_CHECK_SIGNAL_PENDING=1' HUP INT TERM
|
|
1139
|
+
set -m
|
|
1140
|
+
( FM_CHECK_OWNED_GROUP=1 run_check_process "$@" ) > "$FM_CHECK_OUTPUT" 2>/dev/null &
|
|
1141
|
+
FM_ACTIVE_CHECK_PID=$!
|
|
1142
|
+
FM_ACTIVE_CHECK_PGID=$FM_ACTIVE_CHECK_PID
|
|
1143
|
+
set +m
|
|
1144
|
+
pgid=$(ps -o pgid= -p "$FM_ACTIVE_CHECK_PID" 2>/dev/null | tr -d '[:space:]')
|
|
1145
|
+
trap 'exit 1' HUP INT TERM
|
|
1146
|
+
if [ -n "$pgid" ] && [ "$pgid" != "$FM_ACTIVE_CHECK_PGID" ]; then
|
|
1147
|
+
fm_active_check_stop || true
|
|
1148
|
+
fm_check_output_cleanup
|
|
1149
|
+
return 1
|
|
1150
|
+
fi
|
|
1151
|
+
[ -z "$FM_CHECK_SIGNAL_PENDING" ] || exit 1
|
|
1152
|
+
wait "$FM_ACTIVE_CHECK_PID" 2>/dev/null || true
|
|
1153
|
+
FM_ACTIVE_CHECK_PID=
|
|
1154
|
+
fm_active_check_stop || return 1
|
|
1155
|
+
FM_CHECK_RESULT=$(cat "$FM_CHECK_OUTPUT" 2>/dev/null || true)
|
|
1156
|
+
fm_check_output_cleanup
|
|
1157
|
+
}
|
|
1158
|
+
|
|
1159
|
+
# 0 when any signaled status file carries a captain-relevant event in the bytes
|
|
1160
|
+
# appended since this watcher last classified it. The start offset is the
|
|
1161
|
+
# classified-position field in that file's .seen-* marker, and fm-classify-lib.sh's
|
|
1162
|
+
# status-span contract owns both that format and what counts as actionable in
|
|
1163
|
+
# the span. Reading the SPAN rather than the last line is what stops a later
|
|
1164
|
+
# routine append - a `working:` note landing inside SIGNAL_GRACE below - from
|
|
1165
|
+
# hiding the `needs-decision`, `blocked`, `failed`, or `done` event that arrived
|
|
1166
|
+
# just before it: the .seen-* marker advances either way, so an event absorbed
|
|
1167
|
+
# here is never re-read. Non-.status arguments (.turn-ended markers, which carry
|
|
1168
|
+
# no verb) are skipped. A 1 here is NOT "benign" on its own: a no-verb signal
|
|
1169
|
+
# still needs the authoritative working proof or the eligible opt-in bare
|
|
1170
|
+
# turn-end pane-churn proof before it is benign.
|
|
1171
|
+
signal_files_actionable() { # <status-file> ...
|
|
1172
|
+
local f task record rest endpoint ident rc found=1
|
|
1173
|
+
FM_SIGNAL_SURFACE_ENDPOINTS=''
|
|
1174
|
+
for f in "$@"; do
|
|
1175
|
+
case "$f" in *.status) ;; *) continue ;; esac
|
|
1176
|
+
[ -e "$f" ] || [ -L "$f" ] || continue
|
|
1177
|
+
task=$(basename "$f"); task="${task%.status}"
|
|
1178
|
+
record=$(status_span_first_actionable_record "$f" \
|
|
1179
|
+
"$(fm_wake_signal_seen_size "$STATE" "$f")")
|
|
1180
|
+
rc=$?
|
|
1181
|
+
[ "$rc" -eq 1 ] && [ -z "$record" ] && continue
|
|
1182
|
+
if [ "$rc" -eq 2 ]; then
|
|
1183
|
+
# Could not classify this log. Surface it rather than absorbing it, and
|
|
1184
|
+
# record NO classified endpoint for it below, so its content is classified
|
|
1185
|
+
# again once it is readable. The wake signature still advances, which is
|
|
1186
|
+
# what bounds this to one report per distinct file state.
|
|
1187
|
+
found=0
|
|
1188
|
+
continue
|
|
1189
|
+
fi
|
|
1190
|
+
endpoint=${record%%$'\t'*}; rest=${record#*$'\t'}; ident=${rest%%$'\t'*}
|
|
1191
|
+
FM_SIGNAL_SURFACE_ENDPOINTS="${FM_SIGNAL_SURFACE_ENDPOINTS}${f}"$'\t'"${endpoint}"$'\t'"${ident}"$'\n'
|
|
1192
|
+
[ "$rc" -eq 0 ] && found=0
|
|
1193
|
+
done
|
|
1194
|
+
return "$found"
|
|
1195
|
+
}
|
|
1196
|
+
|
|
1197
|
+
# Surfaced-marker bookkeeping for the heartbeat backstop is owned by
|
|
1198
|
+
# fm-push-transition-lib.sh because push and poll paths must write one format.
|
|
1199
|
+
# Mark each actionable status log through the endpoint captured by the heartbeat
|
|
1200
|
+
# scan. Called after the backstop enqueues its wake, so the same events are not
|
|
1201
|
+
# re-surfaced by the next heartbeat.
|
|
1202
|
+
mark_all_captain_relevant_surfaced() {
|
|
1203
|
+
local f endpoint ident rc=0
|
|
1204
|
+
while IFS=$(printf '\t') read -r f endpoint ident; do
|
|
1205
|
+
[ -n "$f" ] || continue
|
|
1206
|
+
if [ "$endpoint" = ERROR ]; then
|
|
1207
|
+
mark_surface_reported "$f" "$ident" || rc=1
|
|
1208
|
+
else
|
|
1209
|
+
mark_surfaced "$f" "$endpoint" "$ident" || rc=1
|
|
1210
|
+
fi
|
|
1211
|
+
done <<EOF
|
|
1212
|
+
$FM_HEARTBEAT_SURFACE_ENDPOINTS
|
|
1213
|
+
EOF
|
|
1214
|
+
return "$rc"
|
|
1215
|
+
}
|
|
1216
|
+
|
|
1217
|
+
# Cheap heartbeat fleet-scan (the always-on twin of the daemon's catch-all). 0 if
|
|
1218
|
+
# any status log carries a captain-relevant event past the position already
|
|
1219
|
+
# surfaced to firstmate (.hb-surfaced-<task>). It walks every log rather than only
|
|
1220
|
+
# those whose LAST line looks captain-relevant, because the event this backstop
|
|
1221
|
+
# most needs to catch is precisely one a later routine append has already moved
|
|
1222
|
+
# past. Pure detect, no side effects: the caller enqueues first, then marks
|
|
1223
|
+
# surfaced. Because every captain-relevant signal/stale already marks itself
|
|
1224
|
+
# surfaced when it wakes firstmate, this normally finds nothing and the heartbeat
|
|
1225
|
+
# is absorbed; it surfaces only an event the per-wake path absorbed by mistake -
|
|
1226
|
+
# the fail-safe backstop.
|
|
1227
|
+
heartbeat_scan_finds_actionable() {
|
|
1228
|
+
local f task record rest endpoint ident rc found=1 sig marker
|
|
1229
|
+
FM_HEARTBEAT_SURFACE_ENDPOINTS=''
|
|
1230
|
+
for f in "$STATE"/*.status; do
|
|
1231
|
+
[ -e "$f" ] || [ -L "$f" ] || continue
|
|
1232
|
+
task=$(basename "$f"); task="${task%.status}"
|
|
1233
|
+
record=$(status_span_first_actionable_record "$f" "$(hb_surfaced_offset "$task")")
|
|
1234
|
+
rc=$?
|
|
1235
|
+
[ "$rc" -eq 1 ] && [ -z "$record" ] && continue
|
|
1236
|
+
if [ "$rc" -eq 2 ]; then
|
|
1237
|
+
sig=$(status_observed_signature "$f")
|
|
1238
|
+
marker=$(_hb_surfaced_path "$task")
|
|
1239
|
+
status_presentation_marker_reported_matches "$marker" "$sig" && continue
|
|
1240
|
+
FM_HEARTBEAT_SURFACE_ENDPOINTS="${FM_HEARTBEAT_SURFACE_ENDPOINTS}${f}"$'\t'"ERROR"$'\t'"${sig}"$'\n'
|
|
1241
|
+
found=0
|
|
1242
|
+
continue
|
|
1243
|
+
fi
|
|
1244
|
+
endpoint=${record%%$'\t'*}; rest=${record#*$'\t'}; ident=${rest%%$'\t'*}
|
|
1245
|
+
FM_HEARTBEAT_SURFACE_ENDPOINTS="${FM_HEARTBEAT_SURFACE_ENDPOINTS}${f}"$'\t'"${endpoint}"$'\t'"${ident}"$'\n'
|
|
1246
|
+
[ "$rc" -eq 0 ] && found=0
|
|
1247
|
+
done
|
|
1248
|
+
return "$found"
|
|
1249
|
+
}
|
|
1250
|
+
|
|
1251
|
+
# event_wait_or_sleep: the terminal wait of each supervision cycle. For a home
|
|
1252
|
+
# with push-capable windows (herdr), it replaces the blind `sleep POLL` with a
|
|
1253
|
+
# bounded wait on the backend's native transition stream, so a crew going
|
|
1254
|
+
# `blocked` wakes the supervisor sub-second instead of after the stale-pane
|
|
1255
|
+
# wedge timer. For every other home - no push-capable window, backend not
|
|
1256
|
+
# capable, or the event path proven unreliable this process - it sleeps POLL,
|
|
1257
|
+
# byte-for-byte today's behavior. The poll loop above still runs every cycle, so
|
|
1258
|
+
# this only ever SHORTENS latency; it can never drop an escalation (the poll
|
|
1259
|
+
# loop is the permanent fail-closed backstop). This preserves the single live
|
|
1260
|
+
# supervision cycle: the reader is a short-lived subprocess of THIS watcher, not
|
|
1261
|
+
# a second watcher, so every guard/beacon/arm/turn-end mechanism is unchanged.
|
|
1262
|
+
event_wait_or_sleep() {
|
|
1263
|
+
local w b session first_backend="" first_session="" rec rc
|
|
1264
|
+
local windows=()
|
|
1265
|
+
while IFS= read -r w; do
|
|
1266
|
+
b=$(window_backend "$w")
|
|
1267
|
+
fm_backend_has_push "$b" || continue
|
|
1268
|
+
# Secondmate endpoints are supervised via status writes, not pane/agent
|
|
1269
|
+
# state (an idle or blocked secondmate agent pane is healthy by design), so
|
|
1270
|
+
# they are excluded from the fast escalation exactly as the stale loop skips
|
|
1271
|
+
# them.
|
|
1272
|
+
[ "$(window_kind "$w")" = secondmate ] && continue
|
|
1273
|
+
session=${w%%:*}
|
|
1274
|
+
if [ -z "$first_backend" ]; then first_backend=$b; first_session=$session; fi
|
|
1275
|
+
# One socket connection covers one backend+session; a home normally has a
|
|
1276
|
+
# single herdr session. A window in a different backend/session stays on the
|
|
1277
|
+
# poll path this cycle.
|
|
1278
|
+
if [ "$b" != "$first_backend" ] || [ "$session" != "$first_session" ]; then
|
|
1279
|
+
continue
|
|
1280
|
+
fi
|
|
1281
|
+
windows+=("$w")
|
|
1282
|
+
done < <(recorded_windows)
|
|
1283
|
+
|
|
1284
|
+
if [ "${#windows[@]}" -eq 0 ]; then
|
|
1285
|
+
sleep "$POLL"
|
|
1286
|
+
return
|
|
1287
|
+
fi
|
|
1288
|
+
|
|
1289
|
+
# Memoized capability probe (fm_backend_events_capable runs a heavy schema
|
|
1290
|
+
# read); re-probed only when the backend/session key changes.
|
|
1291
|
+
if [ "$_event_cap_key" != "$first_backend:$first_session" ]; then
|
|
1292
|
+
_event_cap_key="$first_backend:$first_session"
|
|
1293
|
+
if fm_backend_events_capable "$first_backend" "$first_session"; then
|
|
1294
|
+
_event_cap_ok=1
|
|
1295
|
+
else
|
|
1296
|
+
_event_cap_ok=0
|
|
1297
|
+
fi
|
|
1298
|
+
_event_cap_fails=0
|
|
1299
|
+
fi
|
|
1300
|
+
if [ "$_event_cap_ok" != 1 ]; then
|
|
1301
|
+
sleep "$POLL"
|
|
1302
|
+
return
|
|
1303
|
+
fi
|
|
1304
|
+
|
|
1305
|
+
rec=$(FM_BACKEND_EVENTS_CAPABILITY_CONFIRMED=1 fm_backend_wait_transition "$first_backend" "$first_session" "$POLL" "$STATE" "${windows[@]}")
|
|
1306
|
+
rc=$?
|
|
1307
|
+
case "$rc" in
|
|
1308
|
+
0)
|
|
1309
|
+
_event_cap_fails=0
|
|
1310
|
+
handle_push_transition "$first_backend" "$first_session" "$rec"
|
|
1311
|
+
;;
|
|
1312
|
+
2)
|
|
1313
|
+
# Event path unusable this cycle (connect/subscribe failure). Sleep the
|
|
1314
|
+
# budget and count toward the runtime-disable threshold; past it, drop to
|
|
1315
|
+
# pure polling for the rest of this watcher process.
|
|
1316
|
+
_event_cap_fails=$((_event_cap_fails + 1))
|
|
1317
|
+
[ "$_event_cap_fails" -ge "$EVENT_CAP_FAIL_MAX" ] && _event_cap_ok=0
|
|
1318
|
+
sleep "$POLL"
|
|
1319
|
+
;;
|
|
1320
|
+
*)
|
|
1321
|
+
# 1: a clean full-budget wait with no actionable edge - the reader already
|
|
1322
|
+
# blocked ~POLL, so just continue; the next cycle re-scans.
|
|
1323
|
+
_event_cap_fails=0
|
|
1324
|
+
;;
|
|
1325
|
+
esac
|
|
1326
|
+
}
|
|
1327
|
+
|
|
1328
|
+
# --- Main entry: the runtime below runs only when this file is executed as a
|
|
1329
|
+
# script. When sourced (unit tests loading the functions above), return here
|
|
1330
|
+
# before acquiring the singleton lock or entering the blocking loop.
|
|
1331
|
+
if [ "${BASH_SOURCE[0]}" != "$0" ]; then
|
|
1332
|
+
return 0
|
|
1333
|
+
fi
|
|
1334
|
+
|
|
1335
|
+
if ! fm_lock_try_acquire "$WATCH_LOCK"; then
|
|
1336
|
+
BEAT="$STATE/.last-watcher-beat"
|
|
1337
|
+
if [ -n "${FM_LOCK_HELD_PID:-}" ]; then
|
|
1338
|
+
if [ -e "$BEAT" ]; then
|
|
1339
|
+
beat_age=$(fm_path_age "$BEAT")
|
|
1340
|
+
if [ "$beat_age" -ge "$WATCHER_STALE_GRACE" ]; then
|
|
1341
|
+
echo "watcher: lock held by live pid $FM_LOCK_HELD_PID but heartbeat is stale for ${beat_age}s (>${WATCHER_STALE_GRACE}s); inspect or stop that watcher before re-arming." >&2
|
|
1342
|
+
exit 1
|
|
1343
|
+
fi
|
|
1344
|
+
elif [ "$(fm_path_age "$WATCH_LOCK")" -ge "$WATCHER_STALE_GRACE" ]; then
|
|
1345
|
+
echo "watcher: lock held by live pid $FM_LOCK_HELD_PID but no heartbeat exists; inspect or stop that watcher before re-arming." >&2
|
|
1346
|
+
exit 1
|
|
1347
|
+
fi
|
|
1348
|
+
echo "watcher: already running pid $FM_LOCK_HELD_PID"
|
|
1349
|
+
else
|
|
1350
|
+
echo "watcher: already running"
|
|
1351
|
+
fi
|
|
1352
|
+
exit 0
|
|
1353
|
+
fi
|
|
1354
|
+
WATCHER_RECOVERY_PENDING=0
|
|
1355
|
+
if [ -n "${FM_LOCK_RECOVERED_PID:-}" ]; then
|
|
1356
|
+
WATCHER_RECOVERY_PENDING=1
|
|
1357
|
+
fi
|
|
1358
|
+
if [ "${FM_WATCH_HANDLING_SUCCESSOR:-0}" != 1 ]; then
|
|
1359
|
+
if ! fm_recovery_marker_reopen_announced "$WATCHER_DOWNTIME_MARKER"; then
|
|
1360
|
+
echo "watcher: recovery state could not be reopened safely; retaining stale lock evidence" >&2
|
|
1361
|
+
exit 1
|
|
1362
|
+
fi
|
|
1363
|
+
fi
|
|
1364
|
+
if ! fm_recovery_marker_arm_check "$WATCHER_DOWNTIME_MARKER"; then
|
|
1365
|
+
echo "watcher: recovery state could not be consumed safely; retaining stale lock evidence" >&2
|
|
1366
|
+
exit 1
|
|
1367
|
+
fi
|
|
1368
|
+
if [ "${FM_WATCH_HANDLING_SUCCESSOR:-0}" = 1 ]; then
|
|
1369
|
+
WATCHER_RECOVERY_PENDING=0
|
|
1370
|
+
elif [ "$FM_RECOVERY_MARKER_ACTION" = recover ]; then
|
|
1371
|
+
WATCHER_RECOVERY_PENDING=1
|
|
1372
|
+
fi
|
|
1373
|
+
# Side-band ledger publication, detached from the poll loop.
|
|
1374
|
+
#
|
|
1375
|
+
# The poll loop owns the liveness beacon below, and fm-guard.sh reads that
|
|
1376
|
+
# beacon's freshness as proof that supervision is alive. Publication is a
|
|
1377
|
+
# side-band nicety bounded by FM_HOME_SUMMARY_TIMEOUT, but that bound is far
|
|
1378
|
+
# larger than one poll and, in a home whose publication keeps failing, it is
|
|
1379
|
+
# paid on every poll - so running it inline puts up to a full publication
|
|
1380
|
+
# deadline between two beacon touches and can starve the guard's grace. Nothing
|
|
1381
|
+
# in the poll depends on the ledger, so start it and move on: the beacon keeps
|
|
1382
|
+
# advancing no matter how slow the publication is.
|
|
1383
|
+
#
|
|
1384
|
+
# The trade the detachment makes: a reader can briefly see a ledger that
|
|
1385
|
+
# predates the event this poll just surfaced, where the inline call published
|
|
1386
|
+
# first. Publication is eventually consistent by design and every reader
|
|
1387
|
+
# re-derives current state from the owning home anyway, while beacon freshness
|
|
1388
|
+
# is what the whole supervision chain rests on.
|
|
1389
|
+
HOME_SUMMARY_PID=
|
|
1390
|
+
home_summary_refresh_detached() {
|
|
1391
|
+
if [ -n "$HOME_SUMMARY_PID" ]; then
|
|
1392
|
+
if kill -0 "$HOME_SUMMARY_PID" 2>/dev/null; then
|
|
1393
|
+
return 0
|
|
1394
|
+
fi
|
|
1395
|
+
wait "$HOME_SUMMARY_PID" 2>/dev/null || true
|
|
1396
|
+
HOME_SUMMARY_PID=
|
|
1397
|
+
fi
|
|
1398
|
+
FM_HOME_SUMMARY_IF_IDLE=1 \
|
|
1399
|
+
"$SCRIPT_DIR/fm-home-summary-refresh.sh" --best-effort </dev/null >/dev/null 2>&1 &
|
|
1400
|
+
HOME_SUMMARY_PID=$!
|
|
1401
|
+
}
|
|
1402
|
+
|
|
1403
|
+
watcher_cleanup() {
|
|
1404
|
+
local cleanup_status=0 owns_lock=0 transition=release-lock
|
|
1405
|
+
if [ "$(cat "$WATCH_LOCK/pid" 2>/dev/null || true)" = "${WATCHER_PID:-}" ]; then
|
|
1406
|
+
owns_lock=1
|
|
1407
|
+
if [ "${WATCHER_RECOVERY_PENDING:-0}" -eq 1 ] \
|
|
1408
|
+
&& [ "${FM_WATCH_DELIVERED_REASON:-}" = "check: rearm-resurface" ]; then
|
|
1409
|
+
transition=release-lock-existing
|
|
1410
|
+
fi
|
|
1411
|
+
fi
|
|
1412
|
+
fm_active_check_stop || cleanup_status=1
|
|
1413
|
+
fm_check_output_cleanup
|
|
1414
|
+
fm_custom_check_snapshot_cleanup
|
|
1415
|
+
if [ "$owns_lock" -eq 1 ] \
|
|
1416
|
+
&& ! fm_recovery_transition "$WATCHER_DOWNTIME_MARKER" "$transition" "$WATCH_LOCK" downtime; then
|
|
1417
|
+
echo "watcher: recovery state could not be persisted; retaining stale lock evidence" >&2
|
|
1418
|
+
cleanup_status=1
|
|
1419
|
+
fi
|
|
1420
|
+
return "$cleanup_status"
|
|
1421
|
+
}
|
|
1422
|
+
trap watcher_cleanup EXIT
|
|
1423
|
+
trap 'exit 1' HUP INT TERM
|
|
1424
|
+
# This watcher's own pid, as recorded in the lock by fm_lock_claim (which writes
|
|
1425
|
+
# ${BASHPID:-$$} from this same main shell). Read directly, never via a command
|
|
1426
|
+
# substitution, so it matches the stored holder pid for the self-eviction check.
|
|
1427
|
+
WATCHER_PID=${BASHPID:-$$}
|
|
1428
|
+
printf '%s\n' "$FM_HOME" > "$WATCH_LOCK/fm-home" || true
|
|
1429
|
+
printf '%s\n' "$WATCH_PATH" > "$WATCH_LOCK/watcher-path" || true
|
|
1430
|
+
# shellcheck disable=SC2034 # Consumed by wake() in the separately linted transition owner.
|
|
1431
|
+
FM_WATCH_DELIVERY_PID=$WATCHER_PID
|
|
1432
|
+
FM_WATCH_DELIVERY_IDENTITY=$(fm_pid_identity "$WATCHER_PID" 2>/dev/null || true)
|
|
1433
|
+
printf '%s\n' "$FM_WATCH_DELIVERY_IDENTITY" > "$WATCH_LOCK/pid-identity" 2>/dev/null || true
|
|
1434
|
+
|
|
1435
|
+
[ -e "$STATE/.last-heartbeat" ] || touch "$STATE/.last-heartbeat"
|
|
1436
|
+
|
|
1437
|
+
# A merged poll may have queued its terminal wake and then lost the process
|
|
1438
|
+
# between receipt publication and fixed-path removal.
|
|
1439
|
+
# Finish only identity-bound retirement receipts before any check can run.
|
|
1440
|
+
if ! fm_pr_poll_retirement_recover_all "$STATE" "$SCRIPT_DIR/fm-pr-poll.sh"; then
|
|
1441
|
+
reason="check: rejected unauthenticated PR poll retirement receipts:$FM_PR_POLL_RETIREMENT_REJECTED"
|
|
1442
|
+
fm_wake_append check pr-poll-retirement "$reason" || exit 1
|
|
1443
|
+
touch "$STATE/.last-check"
|
|
1444
|
+
wake "$reason"
|
|
1445
|
+
fi
|
|
1446
|
+
|
|
1447
|
+
# Shared by both the first-notification and already-notified paths below so
|
|
1448
|
+
# the retirement sequence (bin/fm-pr-lib.sh) is stated once.
|
|
1449
|
+
retire_merged_pr_poll() { # <id>
|
|
1450
|
+
local id=$1
|
|
1451
|
+
if fm_pr_poll_retirement_publish "$STATE" "$id" "$SCRIPT_DIR/fm-pr-poll.sh" merged; then
|
|
1452
|
+
fm_pr_poll_retirement_recover_one "$STATE" "$id" "$SCRIPT_DIR/fm-pr-poll.sh" \
|
|
1453
|
+
|| triage_log "merged PR poll retirement remains recoverable for $id"
|
|
1454
|
+
else
|
|
1455
|
+
triage_log "merged PR poll retirement deferred because its canonical snapshot changed for $id"
|
|
1456
|
+
fi
|
|
1457
|
+
}
|
|
1458
|
+
|
|
1459
|
+
resurface_after_downtime() {
|
|
1460
|
+
# Handling successors already have a predecessor-delivered wake on the way.
|
|
1461
|
+
# Re-announcing from this cycle is what turned a lost handshake into an
|
|
1462
|
+
# unbounded recovery loop; stay in the poll loop and supervise instead.
|
|
1463
|
+
if [ "${FM_WATCH_HANDLING_SUCCESSOR:-0}" = 1 ]; then
|
|
1464
|
+
return 0
|
|
1465
|
+
fi
|
|
1466
|
+
if [ "$WATCHER_RECOVERY_PENDING" -ne 1 ]; then
|
|
1467
|
+
if ! fm_recovery_marker_arm_check "$WATCHER_DOWNTIME_MARKER"; then
|
|
1468
|
+
echo "watcher: recovery state could not be consumed safely" >&2
|
|
1469
|
+
exit 1
|
|
1470
|
+
fi
|
|
1471
|
+
[ "$FM_RECOVERY_MARKER_ACTION" = recover ] || return 0
|
|
1472
|
+
fi
|
|
1473
|
+
wake "check: rearm-resurface"
|
|
1474
|
+
}
|
|
1475
|
+
|
|
1476
|
+
while :; do
|
|
1477
|
+
# Self-eviction: if the singleton lock no longer names this process, a second
|
|
1478
|
+
# watcher has taken over (e.g. a transient duplicate from a racy arm). Stand
|
|
1479
|
+
# down so the rightful singleton continues alone. The EXIT trap's release
|
|
1480
|
+
# no-ops because the lock pid is not ours, so the survivor's lock is untouched.
|
|
1481
|
+
# This makes any duplicate self-resolve within one poll instead of persisting
|
|
1482
|
+
# and doubling every wake.
|
|
1483
|
+
if [ "$(cat "$WATCH_LOCK/pid" 2>/dev/null || true)" != "$WATCHER_PID" ]; then
|
|
1484
|
+
exit 0
|
|
1485
|
+
fi
|
|
1486
|
+
|
|
1487
|
+
# Liveness beacon for fm-guard.sh: a fresh mtime here means a watcher is
|
|
1488
|
+
# alive. Supervision scripts warn when this goes stale with tasks in flight.
|
|
1489
|
+
touch "$STATE/.last-watcher-beat"
|
|
1490
|
+
|
|
1491
|
+
if [ "$(age_of "$STATE/home-summary.json")" -ge "$HOME_SUMMARY_INTERVAL" ]; then
|
|
1492
|
+
home_summary_refresh_detached
|
|
1493
|
+
fi
|
|
1494
|
+
|
|
1495
|
+
# Parent-owned secondmate pending-reply reconciliation: resolve correlated
|
|
1496
|
+
# parent reports, observe backend busy/idle turn completion, send one recovery
|
|
1497
|
+
# repost after grace, and escalate once if the recovery turn is also missed.
|
|
1498
|
+
# No conversation scraping; unresolved records are never silently expired.
|
|
1499
|
+
fm_pending_reply_tick "$STATE" || true
|
|
1500
|
+
|
|
1501
|
+
# A live secondmate endpoint does not prove that its own wake loop is alive.
|
|
1502
|
+
# Observe the foreign queue before the rest of this cycle so an aged row wakes
|
|
1503
|
+
# the parent without consuming or rewriting the receiving home's record.
|
|
1504
|
+
secondmate_wake_stall_tick || {
|
|
1505
|
+
echo "watcher: secondmate wake-loop observation failed" >&2
|
|
1506
|
+
exit 1
|
|
1507
|
+
}
|
|
1508
|
+
|
|
1509
|
+
# Process-to-event liveness repair. This never discovers a result by polling:
|
|
1510
|
+
# each registered source has its own child blocking on that source, and this
|
|
1511
|
+
# only republishes results already captured durably and restarts a source
|
|
1512
|
+
# whose owner is gone. It is a no-op with nothing registered.
|
|
1513
|
+
if [ -d "$STATE/procevent" ]; then
|
|
1514
|
+
FM_HOME="$FM_HOME" "$SCRIPT_DIR/fm-procevent.sh" reconcile >/dev/null 2>&1 || true
|
|
1515
|
+
fi
|
|
1516
|
+
# Then deliver any queued-but-unsurfaced result, including one a runner
|
|
1517
|
+
# published while this watcher was between cycles.
|
|
1518
|
+
procevent_surface_queued
|
|
1519
|
+
|
|
1520
|
+
# A process-event result carries richer adapter-owned wake context than the
|
|
1521
|
+
# generic recovery reason, so give that owner first refusal.
|
|
1522
|
+
resurface_after_downtime
|
|
1523
|
+
|
|
1524
|
+
# The existing poll loop also owns the bounded inactive-outcome cadence.
|
|
1525
|
+
# This is mechanical and silent unless a durable terminal-outcome obligation
|
|
1526
|
+
# was created, so quiet cycles never wake firstmate or consume model tokens.
|
|
1527
|
+
inactive_out=
|
|
1528
|
+
if inactive_out=$(FM_HOME="$FM_HOME" FM_STATE_OVERRIDE="$STATE" \
|
|
1529
|
+
"$SCRIPT_DIR/fm-inactive-reconcile.sh" scan 2>/dev/null); then
|
|
1530
|
+
if [ -n "$inactive_out" ]; then
|
|
1531
|
+
wake "check: inactive-outcome"
|
|
1532
|
+
fi
|
|
1533
|
+
else
|
|
1534
|
+
triage_log "inactive-outcome reconciliation unavailable"
|
|
1535
|
+
fi
|
|
1536
|
+
|
|
1537
|
+
# Slow per-task checks (firstmate writes these, e.g. a merged-PR poll).
|
|
1538
|
+
# Time-based via .last-check mtime so the cadence survives watcher restarts.
|
|
1539
|
+
# Evaluated BEFORE the signal scan: wake() exits the cycle, so a check placed
|
|
1540
|
+
# after the signal scan would be starved whenever a chatty sibling crewmate
|
|
1541
|
+
# keeps producing signals - the slow poll (e.g. merge detection) would then
|
|
1542
|
+
# never run until the fleet went quiet. Checks are due only every
|
|
1543
|
+
# CHECK_INTERVAL, so most cycles skip this block and fall straight through.
|
|
1544
|
+
if [ "$(age_of "$STATE/.last-check")" -ge "$CHECK_INTERVAL" ]; then
|
|
1545
|
+
rejected_checks=
|
|
1546
|
+
for c in "$STATE"/*.check.sh; do
|
|
1547
|
+
[ -e "$c" ] || continue
|
|
1548
|
+
is_pr_poll=0
|
|
1549
|
+
if [ "$(basename "$c")" = x-watch.check.sh ]; then
|
|
1550
|
+
if fmx_poll_shim_valid "$c" "$FM_HOME" "$FM_ROOT" \
|
|
1551
|
+
&& [ -f "$FM_ROOT/bin/fm-x-poll.sh" ] && [ ! -L "$FM_ROOT/bin/fm-x-poll.sh" ]; then
|
|
1552
|
+
FM_HOME="$FM_HOME" run_check_capture "$FM_ROOT/bin/fm-x-poll.sh" || exit 1
|
|
1553
|
+
out=$FM_CHECK_RESULT
|
|
1554
|
+
else
|
|
1555
|
+
rejected_checks="$rejected_checks $c"
|
|
1556
|
+
continue
|
|
1557
|
+
fi
|
|
1558
|
+
else
|
|
1559
|
+
id=$(basename "$c" .check.sh)
|
|
1560
|
+
if fm_pr_poll_snapshot_capture "$STATE" "$id" "$SCRIPT_DIR/fm-pr-poll.sh"; then
|
|
1561
|
+
is_pr_poll=1
|
|
1562
|
+
provider=$FM_PR_POLL_SNAPSHOT_PROVIDER
|
|
1563
|
+
url=$FM_PR_POLL_SNAPSHOT_URL
|
|
1564
|
+
host=$FM_PR_POLL_SNAPSHOT_HOST
|
|
1565
|
+
path=$FM_PR_POLL_SNAPSHOT_PATH
|
|
1566
|
+
number=$FM_PR_POLL_SNAPSHOT_NUMBER
|
|
1567
|
+
run_check_capture "$SCRIPT_DIR/fm-pr-poll.sh" --validated \
|
|
1568
|
+
"$provider" "$url" "$host" "$path" "$number" || exit 1
|
|
1569
|
+
out=$FM_CHECK_RESULT
|
|
1570
|
+
elif fm_custom_check_snapshot_prepare "$STATE" "$id"; then
|
|
1571
|
+
custom_snapshot=$FM_CUSTOM_CHECK_SNAPSHOT
|
|
1572
|
+
run_check_capture "$custom_snapshot" || exit 1
|
|
1573
|
+
out=$FM_CHECK_RESULT
|
|
1574
|
+
fm_custom_check_snapshot_cleanup
|
|
1575
|
+
else
|
|
1576
|
+
fm_custom_check_snapshot_cleanup
|
|
1577
|
+
rejected_checks="$rejected_checks $c"
|
|
1578
|
+
continue
|
|
1579
|
+
fi
|
|
1580
|
+
fi
|
|
1581
|
+
if [ -n "$out" ]; then
|
|
1582
|
+
reason="check: $c: $out"
|
|
1583
|
+
if [ "$is_pr_poll" -eq 1 ] && [ "$out" = merged ]; then
|
|
1584
|
+
merge_outcome_rc=0
|
|
1585
|
+
fm_merge_outcome_report "$FM_HOME" "$STATE" "$id" "$url" poll \
|
|
1586
|
+
|| merge_outcome_rc=$?
|
|
1587
|
+
if [ "$merge_outcome_rc" -ne 0 ]; then
|
|
1588
|
+
triage_log "merge outcome for $id could not be recorded (rc=$merge_outcome_rc)"
|
|
1589
|
+
exit 1
|
|
1590
|
+
fi
|
|
1591
|
+
retire_merged_pr_poll "$id"
|
|
1592
|
+
touch "$STATE/.last-check"
|
|
1593
|
+
if [ "$FM_MERGE_OUTCOME_ALREADY_RECORDED" = true ]; then
|
|
1594
|
+
triage_log "absorbed duplicate merged PR poll result for $id"
|
|
1595
|
+
continue
|
|
1596
|
+
fi
|
|
1597
|
+
wake "$reason"
|
|
1598
|
+
fi
|
|
1599
|
+
fm_wake_append check "$c" "$reason" || exit 1
|
|
1600
|
+
touch "$STATE/.last-check"
|
|
1601
|
+
wake "$reason"
|
|
1602
|
+
fi
|
|
1603
|
+
done
|
|
1604
|
+
if [ -n "$rejected_checks" ]; then
|
|
1605
|
+
reason="check: rejected unauthenticated state checks:$rejected_checks"
|
|
1606
|
+
fm_wake_append check unauthenticated-state-checks "$reason" || exit 1
|
|
1607
|
+
touch "$STATE/.last-check"
|
|
1608
|
+
wake "$reason"
|
|
1609
|
+
fi
|
|
1610
|
+
touch "$STATE/.last-check"
|
|
1611
|
+
fi
|
|
1612
|
+
|
|
1613
|
+
# On the first changed signal, linger one grace period and re-scan before
|
|
1614
|
+
# classifying: a crewmate's final status write and the same turn's turn-end
|
|
1615
|
+
# hook land seconds apart, and reporting them as separate actionable wakes
|
|
1616
|
+
# costs a full firstmate turn each. The re-scan also picks up a newer
|
|
1617
|
+
# signature for an already-pending file (last write wins below).
|
|
1618
|
+
pending=$(scan_signals)
|
|
1619
|
+
if [ -n "$pending" ]; then
|
|
1620
|
+
sleep "$SIGNAL_GRACE"
|
|
1621
|
+
pending=$(printf '%s\n%s' "$pending" "$(scan_signals)")
|
|
1622
|
+
# The final coalesced signal set is the watcher-carried status-change
|
|
1623
|
+
# trigger for this home's published summary. Start it before either
|
|
1624
|
+
# surfacing or absorbing the signal, but never wait on it: see
|
|
1625
|
+
# home_summary_refresh_detached for why publication stays off the beacon's
|
|
1626
|
+
# path. Publication failure stays side-band.
|
|
1627
|
+
home_summary_refresh_detached
|
|
1628
|
+
files=""
|
|
1629
|
+
while IFS=$(printf '\t') read -r sf sig f; do
|
|
1630
|
+
[ -n "$sf" ] || continue
|
|
1631
|
+
case " $files " in *" $f "*) ;; *) files="$files $f" ;; esac
|
|
1632
|
+
done <<EOF
|
|
1633
|
+
$pending
|
|
1634
|
+
EOF
|
|
1635
|
+
reason="signal:$files"
|
|
1636
|
+
# Triage: a signal is ACTIONABLE when any of these holds (cheapest first):
|
|
1637
|
+
# - the away-mode daemon owns triage (afk) and wants every wake;
|
|
1638
|
+
# - any status file gained a captain-relevant event since it was last
|
|
1639
|
+
# classified (its whole new span, not merely its last line);
|
|
1640
|
+
# - or it is a no-verb wake (a bare turn-end, a working: note) with no
|
|
1641
|
+
# positive evidence the crew is still executing - the crew stopped its turn
|
|
1642
|
+
# with no actively-running pipeline and no busy pane, so it may be done
|
|
1643
|
+
# (even via an interactive menu that wrote no done: status), waiting on a
|
|
1644
|
+
# decision, or wedged. Absorbing such a turn-end is exactly the
|
|
1645
|
+
# swallowed-finish this change guards against.
|
|
1646
|
+
# Positive evidence is either an authoritative provably-working verdict or, in a
|
|
1647
|
+
# home that opts in with config/turnend-churn-absorb and for a BARE turn-end
|
|
1648
|
+
# alone, a pane that rendered something since the previous poll
|
|
1649
|
+
# (signal_turnend_panes_churned) - the only proof available to a harness whose
|
|
1650
|
+
# busy state has no verified semantic source, bounded so it cannot defer that
|
|
1651
|
+
# task's turn-ends forever. Absorb stays evidence-driven: with neither proof the
|
|
1652
|
+
# wake surfaces exactly as before.
|
|
1653
|
+
# Actionable -> enqueue, advance .seen-* markers, exit. Benign (a no-verb wake
|
|
1654
|
+
# whose crew is still executing) in always-on mode -> advance the markers so it
|
|
1655
|
+
# will not re-fire, log, and keep blocking without enqueuing. Both evidence
|
|
1656
|
+
# checks are costly (a bounded no-mistakes call, then a pane capture), so the ||
|
|
1657
|
+
# ordering evaluates them ONLY for a non-afk signal with no captain-relevant
|
|
1658
|
+
# status span, and the capture only once the authoritative verdict comes up short.
|
|
1659
|
+
FM_SIGNAL_SURFACE_ENDPOINTS=''
|
|
1660
|
+
# shellcheck disable=SC2086 # $files is a space-separated status-path list (ids carry no spaces)
|
|
1661
|
+
signal_files_actionable $files
|
|
1662
|
+
signal_actionable=$?
|
|
1663
|
+
# shellcheck disable=SC2086 # same space-separated status-path list
|
|
1664
|
+
if afk_present || [ "$signal_actionable" -eq 0 ] \
|
|
1665
|
+
|| { ! signal_crew_provably_working $files && ! signal_turnend_panes_churned $files; }; then
|
|
1666
|
+
while IFS=$(printf '\t') read -r sf sig f; do
|
|
1667
|
+
[ -n "$sf" ] || continue
|
|
1668
|
+
fm_wake_append signal "$(basename "$f")" "$reason" || exit 1
|
|
1669
|
+
done <<EOF
|
|
1670
|
+
$pending
|
|
1671
|
+
EOF
|
|
1672
|
+
# The wake signature advances for every file in this batch, including one
|
|
1673
|
+
# whose span could not be classified: it has now been reported, and this is
|
|
1674
|
+
# what bounds an unreadable log to one report per distinct file state. Only
|
|
1675
|
+
# a SUCCESSFULLY classified log commits a classification position below, so
|
|
1676
|
+
# an unreadable log's content is still classified once it becomes readable.
|
|
1677
|
+
while IFS=$(printf '\t') read -r sf sig f; do
|
|
1678
|
+
[ -n "$sf" ] || continue
|
|
1679
|
+
case "$f" in
|
|
1680
|
+
*.status)
|
|
1681
|
+
fm_wake_status_reported_commit "$STATE" "$f" "$sig" || true
|
|
1682
|
+
mark_surface_reported "$f" "$sig" || true
|
|
1683
|
+
;;
|
|
1684
|
+
*) printf '%s' "$sig" > "$sf" ;;
|
|
1685
|
+
esac
|
|
1686
|
+
done <<EOF
|
|
1687
|
+
$pending
|
|
1688
|
+
EOF
|
|
1689
|
+
while IFS=$(printf '\t') read -r f surface_end surface_ident; do
|
|
1690
|
+
[ -n "$f" ] || continue
|
|
1691
|
+
fm_wake_status_seen_commit "$STATE" "$f" "$surface_end" "$surface_ident" || true
|
|
1692
|
+
mark_surfaced "$f" "$surface_end" "$surface_ident"
|
|
1693
|
+
done <<EOF
|
|
1694
|
+
$FM_SIGNAL_SURFACE_ENDPOINTS
|
|
1695
|
+
EOF
|
|
1696
|
+
wake "$reason"
|
|
1697
|
+
else
|
|
1698
|
+
while IFS=$(printf '\t') read -r sf sig f; do
|
|
1699
|
+
[ -n "$sf" ] || continue
|
|
1700
|
+
case "$f" in *.status) ;; *) printf '%s' "$sig" > "$sf" ;; esac
|
|
1701
|
+
done <<EOF
|
|
1702
|
+
$pending
|
|
1703
|
+
EOF
|
|
1704
|
+
signal_commit_error=0
|
|
1705
|
+
while IFS=$(printf '\t') read -r f surface_end surface_ident; do
|
|
1706
|
+
[ -n "$f" ] || continue
|
|
1707
|
+
fm_wake_status_seen_commit "$STATE" "$f" "$surface_end" "$surface_ident" \
|
|
1708
|
+
|| signal_commit_error=1
|
|
1709
|
+
done <<EOF
|
|
1710
|
+
$FM_SIGNAL_SURFACE_ENDPOINTS
|
|
1711
|
+
EOF
|
|
1712
|
+
if [ "$signal_commit_error" -ne 0 ]; then
|
|
1713
|
+
while IFS=$(printf '\t') read -r sf sig f; do
|
|
1714
|
+
[ -n "$sf" ] || continue
|
|
1715
|
+
fm_wake_append signal "$(basename "$f")" "$reason" || exit 1
|
|
1716
|
+
done <<EOF
|
|
1717
|
+
$pending
|
|
1718
|
+
EOF
|
|
1719
|
+
wake "$reason"
|
|
1720
|
+
fi
|
|
1721
|
+
triage_log "absorbed benign $reason"
|
|
1722
|
+
fi
|
|
1723
|
+
fi
|
|
1724
|
+
|
|
1725
|
+
# Layer 1 backbone: pane staleness. Two consecutive identical hashes with no busy
|
|
1726
|
+
# signature means the crewmate finished, is waiting, or is wedged. Each distinct
|
|
1727
|
+
# stale hash is surfaced, absorbed, or timed toward escalation once (.stale-*
|
|
1728
|
+
# remembers the hash already classified, or the declaration a busy pane's
|
|
1729
|
+
# crossed turn bound already handed to the away-mode daemon).
|
|
1730
|
+
while IFS= read -r w; do
|
|
1731
|
+
kind=$(window_kind "$w")
|
|
1732
|
+
task=$(window_to_task "$w" "$STATE")
|
|
1733
|
+
# Steering-inbox loss detection runs before the secondmate stale
|
|
1734
|
+
# exemption below, because a mate's steers land in an inbox too.
|
|
1735
|
+
[ -z "$task" ] || inbox_steer_check "$w" "$task"
|
|
1736
|
+
key=$(window_key "$w")
|
|
1737
|
+
last=$(last_status_line "$STATE/$task.status")
|
|
1738
|
+
if ! status_is_paused_or_captain_held "$last" && [ -e "$STATE/.paused-$key" ]; then
|
|
1739
|
+
clear_pause_tracking "$key"
|
|
1740
|
+
fi
|
|
1741
|
+
# An idle secondmate endpoint is healthy by design, so a mate is admitted to
|
|
1742
|
+
# the pane-stale path ONLY to serve a declared wait's bounded re-surface -
|
|
1743
|
+
# the same declarations pause_state_class reconciles below, which is why this
|
|
1744
|
+
# gate reads the shared predicate rather than the pause verb alone. Narrowing
|
|
1745
|
+
# it to `paused` would leave a mate's captain hold rotting invisibly: the
|
|
1746
|
+
# clear above already spares its pause tracking, but nothing would ever
|
|
1747
|
+
# re-surface it.
|
|
1748
|
+
if [ "$kind" = secondmate ] && ! status_is_paused_or_captain_held "$last"; then
|
|
1749
|
+
continue
|
|
1750
|
+
fi
|
|
1751
|
+
tail40=$(fm_backend_capture "$(window_backend "$w")" "$w" 40 "$(window_label "$w")" 2>/dev/null) || continue
|
|
1752
|
+
h=$(printf '%s' "$tail40" | hash_pane)
|
|
1753
|
+
hf="$STATE/.hash-$key"
|
|
1754
|
+
cf="$STATE/.count-$key"
|
|
1755
|
+
sf="$STATE/.stale-$key"
|
|
1756
|
+
ssf="$STATE/.stale-since-$key"
|
|
1757
|
+
ewf="$STATE/.wedge-escalations-$key"
|
|
1758
|
+
pf="$STATE/.paused-$key" # flag: this key's stale is using the bounded pause cadence
|
|
1759
|
+
prev=$(cat "$hf" 2>/dev/null || true)
|
|
1760
|
+
# Busy match: a backend's native semantic state when available (herdr), else
|
|
1761
|
+
# the last 6 non-blank lines only (the TUI footer area, where every verified
|
|
1762
|
+
# harness renders its busy indicator) so busy-looking strings in displayed
|
|
1763
|
+
# content cannot suppress stale detection. Read once per window per poll and
|
|
1764
|
+
# reused below so a busy verdict is consistent within one cycle.
|
|
1765
|
+
if window_is_busy "$w" "$tail40"; then busy_now=0; else busy_now=1; fi
|
|
1766
|
+
if [ "$h" = "$prev" ]; then
|
|
1767
|
+
n=$(( $(cat "$cf" 2>/dev/null || echo 0) + 1 ))
|
|
1768
|
+
echo "$n" > "$cf"
|
|
1769
|
+
if [ "$n" -ge 2 ] && [ "$busy_now" -ne 0 ]; then
|
|
1770
|
+
# The pane is idle/stale at hash $h. Triage decides whether this wakes
|
|
1771
|
+
# firstmate. Detection itself is unchanged from above.
|
|
1772
|
+
if [ "$kind" = secondmate ]; then
|
|
1773
|
+
case "$(pause_state_class "$w" "$task")" in
|
|
1774
|
+
paused) handle_paused_stale "$w" "$task" "$h" ;;
|
|
1775
|
+
*) clear_pause_tracking "$key" ;;
|
|
1776
|
+
esac
|
|
1777
|
+
elif afk_present; then
|
|
1778
|
+
# Daemon owns triage: one-shot per distinct stale hash, as before.
|
|
1779
|
+
if [ "$(cat "$sf" 2>/dev/null || true)" != "$h" ]; then
|
|
1780
|
+
fm_wake_append stale "$w" "stale: $w" || exit 1
|
|
1781
|
+
printf '%s' "$h" > "$sf"
|
|
1782
|
+
wake "stale: $w"
|
|
1783
|
+
fi
|
|
1784
|
+
elif stale_is_terminal "$w" "$STATE"; then
|
|
1785
|
+
# The log's last line is captain-relevant - but that alone is not
|
|
1786
|
+
# proof the crew is actually done: a crew's own status log gets no
|
|
1787
|
+
# new entry once firstmate hands it to a no-mistakes validation
|
|
1788
|
+
# (AGENTS.md's sparse status-reporting contract), so the log can
|
|
1789
|
+
# keep showing a "done:"/needs-decision/blocked leftover from
|
|
1790
|
+
# BEFORE that validation started for the run's entire (possibly
|
|
1791
|
+
# many-minutes) duration, while stale_is_terminal - which has no
|
|
1792
|
+
# run-step awareness - keeps reporting it as still-current on every
|
|
1793
|
+
# poll. Root cause of the 2026-07 herdr false-surface incidents: a
|
|
1794
|
+
# validating crew was surfaced as stale every few minutes despite an
|
|
1795
|
+
# actively-running pipeline, purely because of this stale leftover
|
|
1796
|
+
# line. On a NEW hash, give an active run/busy pane (the same
|
|
1797
|
+
# authoritative source fm-crew-state.sh itself already prioritizes
|
|
1798
|
+
# over the log) a chance to override before trusting the log.
|
|
1799
|
+
if [ "$(cat "$sf" 2>/dev/null || true)" != "$h" ]; then
|
|
1800
|
+
if crew_is_provably_working "$(window_to_task "$w" "$STATE")"; then
|
|
1801
|
+
printf '%s' "$h" > "$sf"
|
|
1802
|
+
date +%s > "$ssf"
|
|
1803
|
+
clear_write_tracking "$key"
|
|
1804
|
+
triage_log "absorbed stale (provably working, overriding a stale captain-relevant status): $w"
|
|
1805
|
+
else
|
|
1806
|
+
fm_wake_append stale "$w" "stale: $w" || exit 1
|
|
1807
|
+
printf '%s' "$h" > "$sf"
|
|
1808
|
+
rm -f "$ssf"
|
|
1809
|
+
clear_write_tracking "$key"
|
|
1810
|
+
stale_status="$STATE/$(window_to_task "$w" "$STATE").status"
|
|
1811
|
+
stale_record=$(status_span_first_actionable_record "$stale_status" 0)
|
|
1812
|
+
case $? in
|
|
1813
|
+
0|1) stale_end=${stale_record%%$'\t'*}; stale_rest=${stale_record#*$'\t'}; stale_ident=${stale_rest%%$'\t'*} ;;
|
|
1814
|
+
*) stale_end=''; stale_ident='' ;;
|
|
1815
|
+
esac
|
|
1816
|
+
mark_surfaced "$stale_status" "$stale_end" "$stale_ident"
|
|
1817
|
+
wake "stale: $w"
|
|
1818
|
+
fi
|
|
1819
|
+
elif [ -e "$ssf" ]; then
|
|
1820
|
+
# This exact hash was already overridden as provably-working (a
|
|
1821
|
+
# wedge timer is running for it) - keep treating it that way
|
|
1822
|
+
# without re-reading the crew state every poll, and without
|
|
1823
|
+
# letting the still-captain-relevant log line re-surface it.
|
|
1824
|
+
wedge_timer_check "$w" "$ssf" "stale (overridden terminal status)" "$ewf" "$task"
|
|
1825
|
+
fi
|
|
1826
|
+
# else: already surfaced as genuinely terminal on a prior poll of
|
|
1827
|
+
# this same hash - nothing left to do (matches the original,
|
|
1828
|
+
# unmodified terminal-status behavior).
|
|
1829
|
+
else
|
|
1830
|
+
# Non-terminal stale: a crew gone quiet without a captain-relevant status.
|
|
1831
|
+
# Decided once per distinct stale hash (the costly state reads run only
|
|
1832
|
+
# on first sight, never every poll) via pause_state_class, which returns:
|
|
1833
|
+
# - working: an actively-running pipeline legitimately sits on a static
|
|
1834
|
+
# pane (e.g. waiting on CI), so absorb and start the wedge timer so a
|
|
1835
|
+
# genuinely frozen run still escalates past STALE_ESCALATE_SECS;
|
|
1836
|
+
# - paused: a declared wait pause_state_class admits (its header owns which
|
|
1837
|
+
# liveness evidence each kind of crew must supply), so absorb on the long
|
|
1838
|
+
# PAUSE_RESURFACE_SECS cadence instead of wedge-escalating;
|
|
1839
|
+
# - none: no running pipeline, no exact busy verdict, no admitted declared wait.
|
|
1840
|
+
# Surface immediately so firstmate inspects the inconclusive state
|
|
1841
|
+
# (it may be done via an interactive menu that wrote no done: status,
|
|
1842
|
+
# waiting on a decision, or wedged) instead of leaving the finish to
|
|
1843
|
+
# wait out the timer.
|
|
1844
|
+
if [ "$(cat "$sf" 2>/dev/null || true)" != "$h" ]; then
|
|
1845
|
+
task=$(window_to_task "$w" "$STATE")
|
|
1846
|
+
case "$(pause_state_class "$w" "$task")" in
|
|
1847
|
+
working)
|
|
1848
|
+
clear_pause_tracking "$key"
|
|
1849
|
+
printf '%s' "$h" > "$sf"
|
|
1850
|
+
date +%s > "$ssf"
|
|
1851
|
+
triage_log "absorbed non-terminal stale (provably working): $w"
|
|
1852
|
+
;;
|
|
1853
|
+
paused)
|
|
1854
|
+
handle_paused_stale "$w" "$task" "$h"
|
|
1855
|
+
;;
|
|
1856
|
+
*)
|
|
1857
|
+
surface_nonterminal_stale "$w" "$h"
|
|
1858
|
+
;;
|
|
1859
|
+
esac
|
|
1860
|
+
else
|
|
1861
|
+
task=$(window_to_task "$w" "$STATE")
|
|
1862
|
+
if [ -e "$pf" ] || status_is_paused_or_captain_held "$(last_status_line "$STATE/$task.status")"; then
|
|
1863
|
+
case "$(pause_state_class "$w" "$task")" in
|
|
1864
|
+
paused) handle_paused_stale "$w" "$task" "$h" ;;
|
|
1865
|
+
working) clear_pause_state "$key"
|
|
1866
|
+
printf '%s' "$h" > "$sf"
|
|
1867
|
+
wedge_timer_check "$w" "$ssf" "non-terminal stale (provably working after a declared pause)" "$ewf" "$task"
|
|
1868
|
+
triage_log "absorbed non-terminal stale (provably working): $w" ;;
|
|
1869
|
+
*) handle_paused_stale "$w" "$task" "$h" ;;
|
|
1870
|
+
esac
|
|
1871
|
+
else
|
|
1872
|
+
wedge_timer_check "$w" "$ssf" "non-terminal stale" "$ewf" "$task"
|
|
1873
|
+
fi
|
|
1874
|
+
fi
|
|
1875
|
+
fi
|
|
1876
|
+
else
|
|
1877
|
+
# Pane busy or not yet stably stale: reset pending escalation bookkeeping,
|
|
1878
|
+
# unless a genuinely busy pane has gone too long with no completed turn -
|
|
1879
|
+
# then route it through busy_turn_bound_check, which hands the crossed
|
|
1880
|
+
# bound to the same wedge timer unless the crew declared the wait itself.
|
|
1881
|
+
paused_bound=1
|
|
1882
|
+
if [ "$busy_now" -eq 0 ] && busy_turn_over_age "$task"; then
|
|
1883
|
+
busy_turn_bound_check "$w" "$task" "$h" "$ssf" "$ewf" && paused_bound=0
|
|
1884
|
+
else
|
|
1885
|
+
rm -f "$ssf" "$ewf"
|
|
1886
|
+
clear_write_tracking "$key"
|
|
1887
|
+
fi
|
|
1888
|
+
# A busy pane normally means real work resumed, so stale pause bookkeeping
|
|
1889
|
+
# is cleared - but not in the same poll the declared-pause cadence just
|
|
1890
|
+
# recorded it, or the re-surface throttle it depends on would be erased and
|
|
1891
|
+
# the pause would re-surface every poll instead of once per long cadence.
|
|
1892
|
+
if [ "$paused_bound" -ne 0 ] && [ -e "$pf" ] && { [ "$n" -ge 2 ] || ! status_is_paused_or_captain_held "$(last_status_line "$STATE/$(window_to_task "$w" "$STATE").status")"; }; then
|
|
1893
|
+
clear_pause_tracking "$key"
|
|
1894
|
+
fi
|
|
1895
|
+
fi
|
|
1896
|
+
else
|
|
1897
|
+
printf '%s' "$h" > "$hf"
|
|
1898
|
+
echo 0 > "$cf"
|
|
1899
|
+
paused_bound=1
|
|
1900
|
+
if [ "$busy_now" -eq 0 ] && busy_turn_over_age "$task"; then
|
|
1901
|
+
busy_turn_bound_check "$w" "$task" "$h" "$ssf" "$ewf" && paused_bound=0
|
|
1902
|
+
else
|
|
1903
|
+
rm -f "$ssf" "$ewf"
|
|
1904
|
+
clear_write_tracking "$key"
|
|
1905
|
+
fi
|
|
1906
|
+
task=$(window_to_task "$w" "$STATE")
|
|
1907
|
+
if ! afk_present && status_is_paused_or_captain_held "$(last_status_line "$STATE/$task.status")" && [ "$busy_now" -ne 0 ]; then
|
|
1908
|
+
case "$(pause_state_class "$w" "$task")" in
|
|
1909
|
+
paused) handle_paused_stale "$w" "$task" "$h" ;;
|
|
1910
|
+
*) clear_pause_tracking "$key" ;;
|
|
1911
|
+
esac
|
|
1912
|
+
elif [ "$paused_bound" -ne 0 ] && [ -e "$pf" ]; then
|
|
1913
|
+
# Same rule as the stable-hash branch: never clear pause bookkeeping the
|
|
1914
|
+
# declared-pause cadence recorded on this very poll.
|
|
1915
|
+
clear_pause_tracking "$key"
|
|
1916
|
+
fi
|
|
1917
|
+
fi
|
|
1918
|
+
done < <(recorded_windows)
|
|
1919
|
+
|
|
1920
|
+
# Heartbeat: the watcher runs a cheap fleet-scan at a regular cadence no matter
|
|
1921
|
+
# what. Time-based via .last-heartbeat mtime; interval doubles per consecutive
|
|
1922
|
+
# no-change heartbeat (idle fleet) up to HEARTBEAT_MAX, and resets on any
|
|
1923
|
+
# surfaced non-heartbeat wake.
|
|
1924
|
+
streak=$(cat "$STATE/.heartbeat-streak" 2>/dev/null || echo 0)
|
|
1925
|
+
[ "$streak" -gt 12 ] && streak=12
|
|
1926
|
+
hb=$(( HEARTBEAT * (1 << streak) ))
|
|
1927
|
+
[ "$hb" -gt "$HEARTBEAT_MAX" ] && hb=$HEARTBEAT_MAX
|
|
1928
|
+
if [ "$(age_of "$STATE/.last-heartbeat")" -ge "$hb" ]; then
|
|
1929
|
+
# Triage: in always-on mode a heartbeat is benign unless the cheap fleet-scan
|
|
1930
|
+
# turns up a captain-relevant status the per-wake path missed. Absorb the
|
|
1931
|
+
# no-change case (advance the schedule and back off exactly as wake() would,
|
|
1932
|
+
# without exiting); the away-mode daemon, when present, owns triage and wants
|
|
1933
|
+
# every heartbeat.
|
|
1934
|
+
if afk_present; then
|
|
1935
|
+
fm_wake_append heartbeat heartbeat heartbeat || exit 1
|
|
1936
|
+
touch "$STATE/.last-heartbeat"
|
|
1937
|
+
wake "heartbeat"
|
|
1938
|
+
elif heartbeat_scan_finds_actionable; then
|
|
1939
|
+
# Backstop: a captain-relevant event the per-wake path absorbed by mistake.
|
|
1940
|
+
# Enqueue first, then record every status log surfaced through its end so the
|
|
1941
|
+
# next heartbeat does not re-fire it (enqueue-before-suppress preserved);
|
|
1942
|
+
# this wake sends firstmate to the whole fleet, so every log is read.
|
|
1943
|
+
fm_wake_append heartbeat heartbeat heartbeat || exit 1
|
|
1944
|
+
touch "$STATE/.last-heartbeat"
|
|
1945
|
+
mark_all_captain_relevant_surfaced || true
|
|
1946
|
+
wake "heartbeat"
|
|
1947
|
+
else
|
|
1948
|
+
if ! mark_all_captain_relevant_surfaced; then
|
|
1949
|
+
fm_wake_append heartbeat heartbeat heartbeat || exit 1
|
|
1950
|
+
touch "$STATE/.last-heartbeat"
|
|
1951
|
+
wake "heartbeat"
|
|
1952
|
+
fi
|
|
1953
|
+
touch "$STATE/.last-heartbeat"
|
|
1954
|
+
echo $(( $(cat "$STATE/.heartbeat-streak" 2>/dev/null || echo 0) + 1 )) > "$STATE/.heartbeat-streak"
|
|
1955
|
+
triage_log "absorbed heartbeat (no captain-relevant change)"
|
|
1956
|
+
fi
|
|
1957
|
+
fi
|
|
1958
|
+
|
|
1959
|
+
# Terminal wait: a bounded native-event wait for push-capable homes (herdr),
|
|
1960
|
+
# else the blind poll sleep. See event_wait_or_sleep.
|
|
1961
|
+
event_wait_or_sleep
|
|
1962
|
+
done
|