@zerwiz/ymir 0.1.4 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/README.md +291 -0
- package/.agents/agents/bragi-marketer.md +111 -0
- package/.agents/agents/brokk.md +352 -0
- package/.agents/agents/forseti-reviewer.md +93 -0
- package/.agents/agents/frigg-consent.md +77 -0
- package/.agents/agents/groa-update.md +77 -0
- package/.agents/agents/hnoss-designer.md +69 -0
- package/.agents/agents/huginn-researcher.md +99 -0
- package/.agents/agents/jord-projects.md +78 -0
- package/.agents/agents/kvasir-scout.md +91 -0
- package/.agents/agents/mimir-planner.md +92 -0
- package/.agents/agents/muninn-stow.md +77 -0
- package/.agents/agents/saga-bearings.md +78 -0
- package/.agents/agents/sindri-developer.md +114 -0
- package/.agents/agents/snotra-documenter.md +105 -0
- package/.agents/agents/syn-recovery.md +78 -0
- package/.agents/agents/tyr-check.md +78 -0
- package/.agents/agents/urdh-hold.md +77 -0
- package/.agents/agents/volundr-smith.md +80 -0
- package/.agents/agents/vor-diagnostics.md +77 -0
- package/.agents/assets/agents/naming.md +50 -0
- package/.agents/assets/agents/registry.md +93 -0
- package/.agents/assets/agents/runtime.md +36 -0
- package/.agents/assets/agents/toon-tasks-cli.md +42 -0
- package/.agents/assets/templates/company_entity.template.md +37 -0
- package/.agents/backend/README.md +74 -0
- package/.agents/backend/backends/cmux.sh +649 -0
- package/.agents/backend/backends/herdr-eventwait.py +157 -0
- package/.agents/backend/backends/herdr-workspace-move.py +114 -0
- package/.agents/backend/backends/herdr.sh +3386 -0
- package/.agents/backend/backends/orca.sh +290 -0
- package/.agents/backend/backends/tmux.sh +357 -0
- package/.agents/backend/backends/zellij.sh +700 -0
- package/.agents/backend/fm-afk-launch.sh +654 -0
- package/.agents/backend/fm-afk-return.sh +241 -0
- package/.agents/backend/fm-afk-start.sh +172 -0
- package/.agents/backend/fm-arm-command-policy.mjs +960 -0
- package/.agents/backend/fm-arm-pretool-check.sh +200 -0
- package/.agents/backend/fm-backend-hometag-lib.sh +52 -0
- package/.agents/backend/fm-backend.sh +981 -0
- package/.agents/backend/fm-backlog-handoff.sh +909 -0
- package/.agents/backend/fm-backlog-receive.sh +187 -0
- package/.agents/backend/fm-backlog-transition-lib.sh +779 -0
- package/.agents/backend/fm-bearings-board.sh +204 -0
- package/.agents/backend/fm-bearings-snapshot.sh +546 -0
- package/.agents/backend/fm-bootstrap.sh +1568 -0
- package/.agents/backend/fm-branch-outcome.sh +203 -0
- package/.agents/backend/fm-branch-prompt.sh +101 -0
- package/.agents/backend/fm-brief.sh +452 -0
- package/.agents/backend/fm-busy-event.sh +234 -0
- package/.agents/backend/fm-busy-lib.sh +986 -0
- package/.agents/backend/fm-captain-hold.sh +1001 -0
- package/.agents/backend/fm-cd-command-policy.mjs +153 -0
- package/.agents/backend/fm-cd-pretool-check.sh +190 -0
- package/.agents/backend/fm-check-lib.sh +74 -0
- package/.agents/backend/fm-check-register.sh +42 -0
- package/.agents/backend/fm-check-unregister.sh +52 -0
- package/.agents/backend/fm-classify-lib.sh +1770 -0
- package/.agents/backend/fm-claude-stop-autoarm.sh +334 -0
- package/.agents/backend/fm-composer-lib.sh +1416 -0
- package/.agents/backend/fm-config-inherit-lib.sh +1202 -0
- package/.agents/backend/fm-config-push.sh +254 -0
- package/.agents/backend/fm-control-lib.sh +251 -0
- package/.agents/backend/fm-control.sh +878 -0
- package/.agents/backend/fm-crew-state.sh +625 -0
- package/.agents/backend/fm-cursor-lib.sh +243 -0
- package/.agents/backend/fm-decision-hold.sh +232 -0
- package/.agents/backend/fm-doc-audience-check.sh +269 -0
- package/.agents/backend/fm-dod-lib.sh +67 -0
- package/.agents/backend/fm-ensure-agents-md.sh +253 -0
- package/.agents/backend/fm-extension-launch-barrier.mjs +129 -0
- package/.agents/backend/fm-extension.mjs +2577 -0
- package/.agents/backend/fm-extension.sh +16 -0
- package/.agents/backend/fm-ff-lib.sh +420 -0
- package/.agents/backend/fm-fleet-snapshot.sh +1463 -0
- package/.agents/backend/fm-fleet-sync.sh +458 -0
- package/.agents/backend/fm-fleet-view.sh +96 -0
- package/.agents/backend/fm-gate-refuse-lib.sh +102 -0
- package/.agents/backend/fm-guard.sh +242 -0
- package/.agents/backend/fm-harness.sh +195 -0
- package/.agents/backend/fm-herdr-ci-cleanup.sh +115 -0
- package/.agents/backend/fm-herdr-lab.sh +346 -0
- package/.agents/backend/fm-herdr-session-cleanup.sh +337 -0
- package/.agents/backend/fm-home-seed.sh +984 -0
- package/.agents/backend/fm-home-summary-refresh.sh +255 -0
- package/.agents/backend/fm-hook-host-lib.sh +36 -0
- package/.agents/backend/fm-inactive-reconcile.sh +531 -0
- package/.agents/backend/fm-inbox.sh +399 -0
- package/.agents/backend/fm-install-actionlint.sh +84 -0
- package/.agents/backend/fm-install-herdr.sh +96 -0
- package/.agents/backend/fm-install-shellcheck.sh +84 -0
- package/.agents/backend/fm-install-treehouse.sh +95 -0
- package/.agents/backend/fm-kimi-turnend-hook.sh +276 -0
- package/.agents/backend/fm-lease-lib.sh +218 -0
- package/.agents/backend/fm-lease.sh +189 -0
- package/.agents/backend/fm-line-cap-lib.sh +51 -0
- package/.agents/backend/fm-lint-workflows.sh +137 -0
- package/.agents/backend/fm-lint.sh +591 -0
- package/.agents/backend/fm-lock-lib.sh +104 -0
- package/.agents/backend/fm-lock.sh +107 -0
- package/.agents/backend/fm-marker-lib.sh +12 -0
- package/.agents/backend/fm-merge-local.sh +74 -0
- package/.agents/backend/fm-merge-outcome-lib.sh +138 -0
- package/.agents/backend/fm-nm-run-lib.sh +124 -0
- package/.agents/backend/fm-on.sh +125 -0
- package/.agents/backend/fm-operational-input.sh +253 -0
- package/.agents/backend/fm-peek.sh +46 -0
- package/.agents/backend/fm-pending-reply-lib.sh +1409 -0
- package/.agents/backend/fm-pr-check.sh +135 -0
- package/.agents/backend/fm-pr-lib.sh +1016 -0
- package/.agents/backend/fm-pr-merge.sh +707 -0
- package/.agents/backend/fm-pr-poll.sh +110 -0
- package/.agents/backend/fm-primary-scope-lib.sh +33 -0
- package/.agents/backend/fm-procevent-extension-capture.pl +259 -0
- package/.agents/backend/fm-procevent-lavish.sh +628 -0
- package/.agents/backend/fm-procevent-lib.sh +892 -0
- package/.agents/backend/fm-procevent-quota.sh +290 -0
- package/.agents/backend/fm-procevent-remote-reply.sh +577 -0
- package/.agents/backend/fm-procevent-when.sh +504 -0
- package/.agents/backend/fm-procevent.sh +1466 -0
- package/.agents/backend/fm-project-mode.sh +92 -0
- package/.agents/backend/fm-project-origin-lib.sh +180 -0
- package/.agents/backend/fm-promote.sh +282 -0
- package/.agents/backend/fm-public-followup-emit.sh +260 -0
- package/.agents/backend/fm-public-followup-lib.sh +398 -0
- package/.agents/backend/fm-public-followup.sh +1330 -0
- package/.agents/backend/fm-push-transition-lib.sh +168 -0
- package/.agents/backend/fm-quota-axi-lib.sh +93 -0
- package/.agents/backend/fm-quota-choose.sh +384 -0
- package/.agents/backend/fm-remote-delta-read.sh +189 -0
- package/.agents/backend/fm-remote-doctor.sh +800 -0
- package/.agents/backend/fm-remote-entrypoint.sh +193 -0
- package/.agents/backend/fm-remote-file.sh +234 -0
- package/.agents/backend/fm-remote-home-provision.sh +259 -0
- package/.agents/backend/fm-remote-home-seed.sh +259 -0
- package/.agents/backend/fm-remote-inherit-push.sh +90 -0
- package/.agents/backend/fm-remote-inherit.sh +179 -0
- package/.agents/backend/fm-remote-job-lib.sh +1241 -0
- package/.agents/backend/fm-remote-job-reap-orphans.sh +137 -0
- package/.agents/backend/fm-remote-job-worker.sh +1124 -0
- package/.agents/backend/fm-remote-readiness-lib.sh +44 -0
- package/.agents/backend/fm-remote-secondmate-control.sh +340 -0
- package/.agents/backend/fm-review-diff.sh +158 -0
- package/.agents/backend/fm-secondmate-charter-lib.sh +44 -0
- package/.agents/backend/fm-secondmate-nudge-lib.sh +68 -0
- package/.agents/backend/fm-secondmate-parent-lib.sh +70 -0
- package/.agents/backend/fm-secondmate-reconcile.sh +371 -0
- package/.agents/backend/fm-secondmate-registry-lib.sh +311 -0
- package/.agents/backend/fm-secondmate-report.sh +88 -0
- package/.agents/backend/fm-send.sh +1059 -0
- package/.agents/backend/fm-session-lock-lib.sh +176 -0
- package/.agents/backend/fm-session-start.sh +970 -0
- package/.agents/backend/fm-sessionstart-cursor.sh +40 -0
- package/.agents/backend/fm-sessionstart-nudge.sh +45 -0
- package/.agents/backend/fm-sessionstart-run.sh +145 -0
- package/.agents/backend/fm-spawn.sh +3153 -0
- package/.agents/backend/fm-startup-memory-budget-lib.sh +224 -0
- package/.agents/backend/fm-startup-memory-budget.sh +94 -0
- package/.agents/backend/fm-startup-network.sh +642 -0
- package/.agents/backend/fm-stow-cascade.sh +251 -0
- package/.agents/backend/fm-subagent-pretool-check.sh +207 -0
- package/.agents/backend/fm-supervise-daemon.sh +1732 -0
- package/.agents/backend/fm-supervision-instructions.sh +215 -0
- package/.agents/backend/fm-supervision-lib.sh +90 -0
- package/.agents/backend/fm-supervisor-target-lib.sh +78 -0
- package/.agents/backend/fm-tangle-lib.sh +53 -0
- package/.agents/backend/fm-task-inbox-lib.sh +402 -0
- package/.agents/backend/fm-tasks-axi-lib.sh +123 -0
- package/.agents/backend/fm-teardown.sh +2922 -0
- package/.agents/backend/fm-test-isolation-proof.sh +588 -0
- package/.agents/backend/fm-test-run.sh +2191 -0
- package/.agents/backend/fm-timeout-lib.sh +141 -0
- package/.agents/backend/fm-timing-lib.sh +179 -0
- package/.agents/backend/fm-tmux-lib.sh +291 -0
- package/.agents/backend/fm-tool-update-check.sh +898 -0
- package/.agents/backend/fm-trace-context-lib.sh +227 -0
- package/.agents/backend/fm-transition-lib.sh +103 -0
- package/.agents/backend/fm-turnend-guard-cursor.sh +391 -0
- package/.agents/backend/fm-turnend-guard-grok.sh +90 -0
- package/.agents/backend/fm-turnend-guard.sh +432 -0
- package/.agents/backend/fm-update.sh +110 -0
- package/.agents/backend/fm-vendor-auth-probe.sh +190 -0
- package/.agents/backend/fm-voice-client.py +1373 -0
- package/.agents/backend/fm-voice-relay.py +1256 -0
- package/.agents/backend/fm-wake-drain.sh +592 -0
- package/.agents/backend/fm-wake-grant.sh +127 -0
- package/.agents/backend/fm-wake-lib.sh +1849 -0
- package/.agents/backend/fm-watch-arm.sh +602 -0
- package/.agents/backend/fm-watch-checkpoint.sh +109 -0
- package/.agents/backend/fm-watch.sh +1962 -0
- package/.agents/backend/fm-x-dismiss.sh +116 -0
- package/.agents/backend/fm-x-followup.sh +286 -0
- package/.agents/backend/fm-x-lib.sh +1000 -0
- package/.agents/backend/fm-x-link.sh +234 -0
- package/.agents/backend/fm-x-poll.sh +185 -0
- package/.agents/backend/fm-x-reply.sh +385 -0
- package/.agents/backend/fm_voice_frame.py +166 -0
- package/.agents/backend/fm_voice_records.py +574 -0
- package/.agents/backend/model-bridge.py +360 -0
- package/.agents/backend/opencode-go-bridge.py +30 -0
- package/.agents/bus/README.md +12 -0
- package/.agents/bus/messages.json +1 -0
- package/.agents/config/agents.machine.example.yaml +16 -0
- package/.agents/config/agents.yaml.example +72 -0
- package/.agents/config/crew-dispatch.json +22 -0
- package/.agents/config/cron.yaml +11 -0
- package/.agents/config/eindri-dispatch.json +67 -0
- package/.agents/config/eindri-harness +1 -0
- package/.agents/config/startup-memory-budget +1 -0
- package/.agents/filebrowser/.filebrowser.json +6 -0
- package/.agents/filebrowser/README.md +11 -0
- package/.agents/gateway/README.md +12 -0
- package/.agents/github/README.md +15 -0
- package/.agents/github/webhooks/.gitkeep +0 -0
- package/.agents/github/workflows/.gitkeep +0 -0
- package/.agents/harness/opencode/plugins/lib/rodd-operational-input.js +37 -0
- package/.agents/harness/opencode/plugins/package.json +4 -0
- package/.agents/harness/opencode/plugins/saga-sessionstart.js +77 -0
- package/.agents/harness/opencode/plugins/syn-cd-check.js +61 -0
- package/.agents/harness/opencode/plugins/syn-pretool-check.js +68 -0
- package/.agents/harness/opencode/plugins/syn-turnend-guard.js +105 -0
- package/.agents/harness/opencode/plugins/syn-watch-arm.js +521 -0
- package/.agents/memory/README.md +46 -0
- package/.agents/memory/runes_audit.md +5 -0
- package/.agents/migrations/0001-hodd-layout.sh +38 -0
- package/.agents/migrations/0002-a2a-mcp.sh +27 -0
- package/.agents/migrations/0003-private-data-separation.sh +121 -0
- package/.agents/migrations/0004-hoard-and-realms.sh +104 -0
- package/.agents/migrations/0005-roots-out-of-tree.sh +158 -0
- package/.agents/sandbox/Dockerfile.utgard +19 -0
- package/.agents/sandbox/README.md +14 -0
- package/.agents/sandbox/utgard.config.json +14 -0
- package/.agents/skills/README.md +59 -0
- package/.agents/skills/bragi-marketing/README.md +3 -0
- package/.agents/skills/bragi-marketing/SKILL.md +82 -0
- package/.agents/skills/eindri-homes/README.md +3 -0
- package/.agents/skills/eindri-homes/SKILL.md +291 -0
- package/.agents/skills/eindri-homes/assets/control-plane.md +122 -0
- package/.agents/skills/eindri-homes/assets/dispatch.md +80 -0
- package/.agents/skills/frigg-consent/README.md +3 -0
- package/.agents/skills/frigg-consent/SKILL.md +56 -0
- package/.agents/skills/galdr-ymirsystem/README.md +3 -0
- package/.agents/skills/galdr-ymirsystem/SKILL.md +159 -0
- package/.agents/skills/galdr-ymirsystem/assets/README.md +74 -0
- package/.agents/skills/galdr-ymirsystem/assets/brokk-distro-runtime.md +390 -0
- package/.agents/skills/galdr-ymirsystem/assets/build-method.md +113 -0
- package/.agents/skills/galdr-ymirsystem/assets/build-tool-categories.md +178 -0
- package/.agents/skills/galdr-ymirsystem/assets/eindri-orchestration.md +518 -0
- package/.agents/skills/galdr-ymirsystem/assets/eindri-profiles.md +136 -0
- package/.agents/skills/galdr-ymirsystem/assets/harness-integration/README.md +736 -0
- package/.agents/skills/galdr-ymirsystem/assets/harness-integration/claude-code.md +197 -0
- package/.agents/skills/galdr-ymirsystem/assets/harness-integration/codex.md +191 -0
- package/.agents/skills/galdr-ymirsystem/assets/harness-integration/cursor.md +193 -0
- package/.agents/skills/galdr-ymirsystem/assets/harness-integration/opencode.md +262 -0
- package/.agents/skills/galdr-ymirsystem/assets/harness-integration/pi.md +295 -0
- package/.agents/skills/galdr-ymirsystem/assets/hlidskjalf-ui.md +539 -0
- package/.agents/skills/galdr-ymirsystem/assets/installation.md +763 -0
- package/.agents/skills/galdr-ymirsystem/assets/local-models/HOST-DATA.example.md +72 -0
- package/.agents/skills/galdr-ymirsystem/assets/local-models.md +274 -0
- package/.agents/skills/galdr-ymirsystem/assets/memory-well.md +163 -0
- package/.agents/skills/galdr-ymirsystem/assets/nornir-jobs.md +415 -0
- package/.agents/skills/galdr-ymirsystem/assets/norse-naming.md +325 -0
- package/.agents/skills/galdr-ymirsystem/assets/odrerir-hall.md +133 -0
- package/.agents/skills/galdr-ymirsystem/assets/pi-boot/einherjar-spawn.schema.json +81 -0
- package/.agents/skills/galdr-ymirsystem/assets/pi-boot/herdr-profile.toml +75 -0
- package/.agents/skills/galdr-ymirsystem/assets/pi-boot/pi-profile.yml +101 -0
- package/.agents/skills/galdr-ymirsystem/assets/pi-boot/supervision-tree.yml +163 -0
- package/.agents/skills/galdr-ymirsystem/assets/pi-boot-guide.md +58 -0
- package/.agents/skills/galdr-ymirsystem/assets/porting-upstream-to-norse.md +301 -0
- package/.agents/skills/galdr-ymirsystem/assets/principles.md +385 -0
- package/.agents/skills/galdr-ymirsystem/assets/reference-adoption.md +99 -0
- package/.agents/skills/galdr-ymirsystem/assets/registry.md +379 -0
- package/.agents/skills/galdr-ymirsystem/assets/runtime-compliance.md +463 -0
- package/.agents/skills/galdr-ymirsystem/assets/runtime-components.md +553 -0
- package/.agents/skills/galdr-ymirsystem/assets/smidja.md +180 -0
- package/.agents/skills/galdr-ymirsystem/schemas/toon-schemas.md +190 -0
- package/.agents/skills/galdr-ymirsystem/scripts/README.md +45 -0
- package/.agents/skills/galdr-ymirsystem/scripts/bench-one.sh +248 -0
- package/.agents/skills/galdr-ymirsystem/scripts/compliance-check.sh +503 -0
- package/.agents/skills/galdr-ymirsystem/scripts/gpu-sample.sh +43 -0
- package/.agents/skills/galdr-ymirsystem/scripts/stop-all.sh +34 -0
- package/.agents/skills/galdr-ymirsystem/scripts/toon-check.py +188 -0
- package/.agents/skills/gjallarhorn-relay/README.md +3 -0
- package/.agents/skills/gjallarhorn-relay/SKILL.md +283 -0
- package/.agents/skills/groa-update/SKILL.md +90 -0
- package/.agents/skills/hamr-adapters/README.md +3 -0
- package/.agents/skills/hamr-adapters/SKILL.md +123 -0
- package/.agents/skills/hamr-adapters/references/common/control-and-recovery.md +37 -0
- package/.agents/skills/hamr-adapters/references/common/dispatch.md +32 -0
- package/.agents/skills/hamr-adapters/references/common/model-and-effort.md +42 -0
- package/.agents/skills/hamr-adapters/references/common/primary-hooks.md +40 -0
- package/.agents/skills/hamr-adapters/references/harness/claude.md +55 -0
- package/.agents/skills/hamr-adapters/references/harness/codex.md +43 -0
- package/.agents/skills/hamr-adapters/references/harness/cursor.md +75 -0
- package/.agents/skills/hamr-adapters/references/harness/grok.md +69 -0
- package/.agents/skills/hamr-adapters/references/harness/kimi.md +51 -0
- package/.agents/skills/hamr-adapters/references/harness/muse.md +70 -0
- package/.agents/skills/hamr-adapters/references/harness/opencode.md +42 -0
- package/.agents/skills/hamr-adapters/references/harness/pi.md +56 -0
- package/.agents/skills/herdr-panes/README.md +3 -0
- package/.agents/skills/herdr-panes/SKILL.md +92 -0
- package/.agents/skills/herdr-panes/assets/herdr-backend.md +346 -0
- package/.agents/skills/herdr-panes/assets/tmux-backend.md +115 -0
- package/.agents/skills/hnoss-design/DESIGN.md +19 -0
- package/.agents/skills/hnoss-design/README.md +3 -0
- package/.agents/skills/hnoss-design/SKILL.md +83 -0
- package/.agents/skills/hvild-afk/README.md +3 -0
- package/.agents/skills/hvild-afk/SKILL.md +241 -0
- package/.agents/skills/jord-projects/README.md +3 -0
- package/.agents/skills/jord-projects/SKILL.md +92 -0
- package/.agents/skills/lifecycle/SKILL.md +56 -0
- package/.agents/skills/lifecycle/smoke_test.sh +57 -0
- package/.agents/skills/lifecycle/start.sh +21 -0
- package/.agents/skills/lifecycle/status.sh +35 -0
- package/.agents/skills/lifecycle/stop.sh +16 -0
- package/.agents/skills/muninn-stow/README.md +3 -0
- package/.agents/skills/muninn-stow/SKILL.md +309 -0
- package/.agents/skills/no-mistakes/SKILL.md +379 -0
- package/.agents/skills/nornir-schedule/README.md +3 -0
- package/.agents/skills/nornir-schedule/SKILL.md +25 -0
- package/.agents/skills/nornir-schedule/assets/events.md +137 -0
- package/.agents/skills/nornir-schedule/assets/quota.md +131 -0
- package/.agents/skills/nsr-compliance/README.md +3 -0
- package/.agents/skills/nsr-compliance/SKILL.md +29 -0
- package/.agents/skills/nsr-compliance/assets/nsr/README.md +40 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/RULES/README.md +15 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/RULES/git-workflow.md +18 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/RULES/security.md +22 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/RULES/tech-stack.md +8 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/SKILL.md +71 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/config/core_four.yaml +16 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/gates/check_danger.sh +102 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/gates/check_wiring.sh +69 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/gates/validate_code.sh +29 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/gates/verify_docs.py +66 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/generate.sh +109 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/harness/envelopes/result_envelope.json +14 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/harness/envelopes/task_envelope.json +20 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/harness/runner.py +303 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/telemetry/logger.py +112 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/templates/compliance/README.md +18 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/templates/compliance/config/README.md +10 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/templates/compliance/gates/README.md +12 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/templates/compliance/gates/check_env.sh +23 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/templates/compliance/gates/check_paths.sh +20 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/templates/compliance/gates/check_platform.sh +23 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/templates/compliance/harness/README.md +11 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/templates/compliance/harness/envelopes/README.md +10 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/templates/compliance/installer/README.md +10 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/templates/compliance/installer/install.sh +125 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/templates/compliance/telemetry/README.md +11 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/README.md +20 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/features/README.md +36 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/features/SKILL-template.md +14 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/git_ops/README.md +25 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/git_ops/SKILL.md +13 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/git_ops/create_branch.sh +6 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/git_ops/safe_commit.sh +9 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/git_ops/sync_upstream.sh +6 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/lifecycle/README.md +27 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/lifecycle/SKILL.md +15 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/lifecycle/smoke_test.sh +12 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/lifecycle/start.sh +19 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/lifecycle/status.sh +12 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/lifecycle/stop.sh +15 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/README.md +18 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/config/README.md +10 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/config/core_four.yaml +13 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/gates/README.md +12 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/gates/check_danger.sh +102 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/gates/check_env.sh +23 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/gates/check_paths.sh +20 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/gates/check_platform.sh +23 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/gates/check_wiring.sh +69 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/harness/README.md +11 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/harness/envelopes/README.md +10 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/harness/envelopes/result_envelope.json +14 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/harness/envelopes/task_envelope.json +20 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/harness/runner.py +303 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/installer/README.md +10 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/installer/install.sh +125 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/telemetry/README.md +11 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/telemetry/logger.py +112 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/BEST_PRACTICES/README.md +14 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/BEST_PRACTICES/api.md +17 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/BEST_PRACTICES/compliance-wiring.md +155 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/BEST_PRACTICES/frontend.md +15 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/CI_CD/README.md +11 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/CI_CD/deployment/README.md +23 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/CI_CD/deployment/clients/README.md +16 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/CI_CD/deployment/clients/template-README.md +11 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/CI_CD/deployment/deploy.sh +49 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/CI_CD/deployment/envs/README.md +12 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/CI_CD/deployment/envs/development.env.example +17 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/CI_CD/deployment/envs/production.env.example +17 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/CI_CD/deployment/envs/staging.env.example +17 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/CI_CD/deployment/tenants/README.md +11 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/CI_CD/deployment/tenants/template.env.example +8 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/DEVELOPER_SETUP/README.md +26 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/DEVELOPER_SETUP/agent.md +7 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/DEVELOPER_SETUP/developers/dev-1.md +20 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/DEVELOPER_SETUP/linux.md +9 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/DEVELOPER_SETUP/macos.md +10 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/DEVELOPER_SETUP/shared.md +19 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/DEVELOPER_SETUP/windows.md +14 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/HOSTING/README.md +25 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/HOSTING/_template.md +41 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/HOSTING/hosting-1.md +41 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/README.md +21 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/RUNBOOK/README.md +15 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/RUNBOOK/deployment.md +19 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/RUNBOOK/disaster-recovery.md +15 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/RUNBOOK/ops-tasks.md +14 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/features/README.md +16 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/project-skills.md +146 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/research/README.md +11 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/research/software-compliance-research.md +269 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/working-with-agents.md +260 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/root/AGENTS.md +38 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/root/ARCHITECTURE.md +36 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/root/BEST_PRACTICES.md +27 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/root/CI_CD.md +23 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/root/FEATURES.md +15 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/root/README.md +22 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/root/RULES.md +37 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/root/STRUCTURE.md +45 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/root/TECH_STACK.md +28 -0
- package/.agents/skills/nsr-compliance/assets/nsr/scaffold-spec.md +63 -0
- package/.agents/skills/nsr-compliance/assets/nsr/scripts/scaffold.sh +103 -0
- package/.agents/skills/nsr-compliance/assets/nsr/scripts/validate.sh +52 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/SKILL.md +71 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/config/core_four.yaml +16 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/gates/check_danger.sh +102 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/gates/check_wiring.sh +69 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/gates/validate_code.sh +29 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/gates/verify_docs.py +66 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/generate.sh +109 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/harness/envelopes/result_envelope.json +14 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/harness/envelopes/task_envelope.json +20 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/harness/runner.py +303 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/telemetry/logger.py +112 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/templates/compliance/README.md +18 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/templates/compliance/config/README.md +10 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/templates/compliance/gates/README.md +12 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/templates/compliance/gates/check_env.sh +23 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/templates/compliance/gates/check_paths.sh +20 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/templates/compliance/gates/check_platform.sh +23 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/templates/compliance/harness/README.md +11 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/templates/compliance/harness/envelopes/README.md +10 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/templates/compliance/installer/README.md +10 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/templates/compliance/installer/install.sh +125 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/templates/compliance/telemetry/README.md +11 -0
- package/.agents/skills/pr-ops/README.md +3 -0
- package/.agents/skills/pr-ops/SKILL.md +109 -0
- package/.agents/skills/ratatoskr-a2a/README.md +3 -0
- package/.agents/skills/ratatoskr-a2a/SKILL.md +148 -0
- package/.agents/skills/rules-check-drift/SKILL.md +81 -0
- package/.agents/skills/saga-bearings/README.md +3 -0
- package/.agents/skills/saga-bearings/SKILL.md +26 -0
- package/.agents/skills/saga-bearings/assets/bearings.md +161 -0
- package/.agents/skills/saga-bearings/assets/board-template.html +731 -0
- package/.agents/skills/saga-bearings/assets/recap.md +56 -0
- package/.agents/skills/syn-recovery/README.md +3 -0
- package/.agents/skills/syn-recovery/SKILL.md +54 -0
- package/.agents/skills/tyr-check/README.md +3 -0
- package/.agents/skills/tyr-check/SKILL.md +218 -0
- package/.agents/skills/urdh-hold/README.md +3 -0
- package/.agents/skills/urdh-hold/SKILL.md +25 -0
- package/.agents/skills/urdh-hold/assets/decisions.md +15 -0
- package/.agents/skills/urdh-hold/assets/hold.md +54 -0
- package/.agents/skills/vor-diagnostics/README.md +3 -0
- package/.agents/skills/vor-diagnostics/SKILL.md +68 -0
- package/.agents/skills/ymir-host/README.md +3 -0
- package/.agents/skills/ymir-host/SKILL.md +35 -0
- package/.agents/skills/ymir-host/assets/desktop.md +36 -0
- package/.agents/skills/ymir-host/assets/install.md +69 -0
- package/.agents/skills/ymir-host/assets/omarchy.md +297 -0
- package/.agents/skills/ymir-host/assets/runbooks.md +15 -0
- package/.agents/skills/ymir-host/assets/thjazi.md +220 -0
- package/.agents/skills/ymir-host/assets/update.md +64 -0
- package/.agents/tests/assets/board-render-harness.mjs +117 -0
- package/.agents/tests/cmux-test-safety.sh +51 -0
- package/.agents/tests/fixtures.sh +291 -0
- package/.agents/tests/fm-afk-inject-e2e.test.sh +428 -0
- package/.agents/tests/fm-afk-inject-herdr-e2e.test.sh +536 -0
- package/.agents/tests/fm-afk-launch.test.sh +961 -0
- package/.agents/tests/fm-afk-pi-herdr-return-e2e.test.sh +297 -0
- package/.agents/tests/fm-afk-return.test.sh +283 -0
- package/.agents/tests/fm-arm-pretool-check.test.sh +482 -0
- package/.agents/tests/fm-ask-user-authority.test.sh +42 -0
- package/.agents/tests/fm-backend-autodetect-smoke.test.sh +176 -0
- package/.agents/tests/fm-backend-cmux-smoke.test.sh +188 -0
- package/.agents/tests/fm-backend-cmux.test.sh +1166 -0
- package/.agents/tests/fm-backend-herdr-eventwait-smoke.test.sh +136 -0
- package/.agents/tests/fm-backend-herdr-eventwait.test.py +106 -0
- package/.agents/tests/fm-backend-herdr-focus-flash-e2e.test.sh +409 -0
- package/.agents/tests/fm-backend-herdr-launcher-workspace-e2e.test.sh +437 -0
- package/.agents/tests/fm-backend-herdr-presentation-e2e.test.sh +1424 -0
- package/.agents/tests/fm-backend-herdr-prune-safety-e2e.test.sh +180 -0
- package/.agents/tests/fm-backend-herdr-respawn-idem-e2e.test.sh +182 -0
- package/.agents/tests/fm-backend-herdr-smoke.test.sh +355 -0
- package/.agents/tests/fm-backend-herdr-workspace-per-home-e2e.test.sh +251 -0
- package/.agents/tests/fm-backend-herdr.test.sh +4606 -0
- package/.agents/tests/fm-backend-orca.test.sh +1357 -0
- package/.agents/tests/fm-backend-tmux-smoke.test.sh +173 -0
- package/.agents/tests/fm-backend-zellij-smoke.test.sh +205 -0
- package/.agents/tests/fm-backend-zellij.test.sh +1360 -0
- package/.agents/tests/fm-backend.test.sh +1142 -0
- package/.agents/tests/fm-backlog-atomicity.test.sh +2302 -0
- package/.agents/tests/fm-backlog-handoff.test.sh +1360 -0
- package/.agents/tests/fm-bearings-board-render.test.sh +135 -0
- package/.agents/tests/fm-bearings-board.test.sh +414 -0
- package/.agents/tests/fm-bearings-snapshot.test.sh +1990 -0
- package/.agents/tests/fm-bootstrap-network-parallel.test.sh +316 -0
- package/.agents/tests/fm-bootstrap.test.sh +1178 -0
- package/.agents/tests/fm-branch-supervision.test.sh +552 -0
- package/.agents/tests/fm-brief.test.sh +785 -0
- package/.agents/tests/fm-busy-adapter-wiring.test.sh +324 -0
- package/.agents/tests/fm-busy-state.test.sh +465 -0
- package/.agents/tests/fm-calm-pi-extension.test.sh +3979 -0
- package/.agents/tests/fm-captain-hold-lifecycle.test.sh +1187 -0
- package/.agents/tests/fm-cd-pretool-check.test.sh +400 -0
- package/.agents/tests/fm-check-unregister.test.sh +198 -0
- package/.agents/tests/fm-classify-corr-token.test.sh +543 -0
- package/.agents/tests/fm-classify-decision-key.test.sh +340 -0
- package/.agents/tests/fm-claude-stop-autoarm-live-e2e.test.sh +164 -0
- package/.agents/tests/fm-claude-stop-autoarm.test.sh +1190 -0
- package/.agents/tests/fm-cmux-claude-composer-live-e2e.test.sh +110 -0
- package/.agents/tests/fm-codex-continuity-live-e2e.test.sh +55 -0
- package/.agents/tests/fm-composer-ghost.test.sh +716 -0
- package/.agents/tests/fm-composer-lib.test.sh +666 -0
- package/.agents/tests/fm-composer-matrix-live-e2e.test.sh +220 -0
- package/.agents/tests/fm-control-herdr-smoke.test.sh +149 -0
- package/.agents/tests/fm-control-relaunch.test.sh +1530 -0
- package/.agents/tests/fm-control.test.sh +910 -0
- package/.agents/tests/fm-crew-state.test.sh +1610 -0
- package/.agents/tests/fm-cursor-harness.test.sh +404 -0
- package/.agents/tests/fm-cursor-primary-live-e2e.test.sh +217 -0
- package/.agents/tests/fm-cursor-primary.test.sh +708 -0
- package/.agents/tests/fm-daemon.test.sh +2740 -0
- package/.agents/tests/fm-documentation-audiences.test.sh +141 -0
- package/.agents/tests/fm-ensure-agents-md.test.sh +371 -0
- package/.agents/tests/fm-extension-binding.test.sh +2187 -0
- package/.agents/tests/fm-fleet-snapshot-view.test.sh +816 -0
- package/.agents/tests/fm-fleet-sync.test.sh +721 -0
- package/.agents/tests/fm-gate-refuse.test.sh +356 -0
- package/.agents/tests/fm-gitignore-config.test.sh +89 -0
- package/.agents/tests/fm-gotmp.test.sh +230 -0
- package/.agents/tests/fm-grok-continuity-live-e2e.test.sh +112 -0
- package/.agents/tests/fm-grok-harness.test.sh +117 -0
- package/.agents/tests/fm-grok-stop-live-e2e.test.sh +219 -0
- package/.agents/tests/fm-guard-stale-banner.test.sh +709 -0
- package/.agents/tests/fm-harness-adapter-instructions-live-e2e.test.sh +134 -0
- package/.agents/tests/fm-harness-adapter-references.test.sh +30 -0
- package/.agents/tests/fm-harness-liveness-drift-live-e2e.test.sh +151 -0
- package/.agents/tests/fm-herdr-lab.test.sh +243 -0
- package/.agents/tests/fm-herdr-session-cleanup-e2e.test.sh +146 -0
- package/.agents/tests/fm-herdr-session-cleanup.test.sh +329 -0
- package/.agents/tests/fm-herdr-submit-confirm-live-e2e.test.sh +125 -0
- package/.agents/tests/fm-herdr-version-floor-live-e2e.test.sh +144 -0
- package/.agents/tests/fm-home-summary-refresh.test.sh +971 -0
- package/.agents/tests/fm-inactive-reconcile.test.sh +485 -0
- package/.agents/tests/fm-kimi-harness.test.sh +687 -0
- package/.agents/tests/fm-lint-workflows.test.sh +532 -0
- package/.agents/tests/fm-lint.test.sh +1023 -0
- package/.agents/tests/fm-muse-harness.test.sh +932 -0
- package/.agents/tests/fm-muse-signals-live-e2e.test.sh +205 -0
- package/.agents/tests/fm-no-mistakes-required.test.sh +72 -0
- package/.agents/tests/fm-on.test.sh +507 -0
- package/.agents/tests/fm-opencode-primary-live-e2e.test.sh +357 -0
- package/.agents/tests/fm-operational-input.test.sh +160 -0
- package/.agents/tests/fm-peek-remote.test.sh +110 -0
- package/.agents/tests/fm-pending-reply.test.sh +1288 -0
- package/.agents/tests/fm-pi-branch-extension.test.sh +3182 -0
- package/.agents/tests/fm-pi-branch-live-e2e.test.sh +527 -0
- package/.agents/tests/fm-pi-primary-live-e2e.test.sh +345 -0
- package/.agents/tests/fm-pi-primary-types.test.sh +68 -0
- package/.agents/tests/fm-pi-watch-extension.test.sh +2843 -0
- package/.agents/tests/fm-pr-check-security.test.sh +2140 -0
- package/.agents/tests/fm-pr-merge.test.sh +2135 -0
- package/.agents/tests/fm-procevent-quota.test.sh +225 -0
- package/.agents/tests/fm-procevent-when.test.sh +406 -0
- package/.agents/tests/fm-procevent.test.sh +1745 -0
- package/.agents/tests/fm-project-origin.test.sh +110 -0
- package/.agents/tests/fm-public-followup.test.sh +2272 -0
- package/.agents/tests/fm-quota-array-dispatch-live-e2e.test.sh +526 -0
- package/.agents/tests/fm-quota-choose.test.sh +626 -0
- package/.agents/tests/fm-remote-backlog-handoff.test.sh +453 -0
- package/.agents/tests/fm-remote-doctor.test.sh +625 -0
- package/.agents/tests/fm-remote-entrypoint.test.sh +59 -0
- package/.agents/tests/fm-remote-job-orphan-reap.test.sh +205 -0
- package/.agents/tests/fm-remote-job.test.sh +768 -0
- package/.agents/tests/fm-remote-reply.test.sh +520 -0
- package/.agents/tests/fm-remote-secondmate-lifecycle-e2e.test.sh +1256 -0
- package/.agents/tests/fm-remote-secondmate-parent-binding.test.sh +326 -0
- package/.agents/tests/fm-remote-secondmate-trace-context.test.sh +310 -0
- package/.agents/tests/fm-remote-transport-lanes.test.sh +431 -0
- package/.agents/tests/fm-review-diff.test.sh +176 -0
- package/.agents/tests/fm-secondmate-harness.test.sh +2600 -0
- package/.agents/tests/fm-secondmate-lifecycle-e2e.test.sh +259 -0
- package/.agents/tests/fm-secondmate-liveness.test.sh +559 -0
- package/.agents/tests/fm-secondmate-reconcile.test.sh +732 -0
- package/.agents/tests/fm-secondmate-safety.test.sh +2972 -0
- package/.agents/tests/fm-secondmate-sync.test.sh +913 -0
- package/.agents/tests/fm-send-inbox-doorbell-live-e2e.test.sh +206 -0
- package/.agents/tests/fm-send-inbox.test.sh +352 -0
- package/.agents/tests/fm-send-popup-settle.test.sh +167 -0
- package/.agents/tests/fm-send-remote-delivery.test.sh +796 -0
- package/.agents/tests/fm-send-resolve-key.test.sh +551 -0
- package/.agents/tests/fm-send-secondmate-marker-herdr-e2e.test.sh +187 -0
- package/.agents/tests/fm-send-secondmate-marker.test.sh +278 -0
- package/.agents/tests/fm-send-settle.test.sh +147 -0
- package/.agents/tests/fm-send-strict.test.sh +241 -0
- package/.agents/tests/fm-session-lock-ancestry.test.sh +365 -0
- package/.agents/tests/fm-session-start.test.sh +2515 -0
- package/.agents/tests/fm-sessionstart-hook-live-e2e.test.sh +631 -0
- package/.agents/tests/fm-sessionstart-instruction-refresh-live-e2e.test.sh +230 -0
- package/.agents/tests/fm-sessionstart-nudge.test.sh +1039 -0
- package/.agents/tests/fm-shared-captain-inheritance.test.sh +404 -0
- package/.agents/tests/fm-spawn-batch.test.sh +148 -0
- package/.agents/tests/fm-spawn-dispatch-profile.test.sh +827 -0
- package/.agents/tests/fm-spawn-pool-base-freshen.test.sh +440 -0
- package/.agents/tests/fm-spawn-worktree-settle.test.sh +147 -0
- package/.agents/tests/fm-startup-memory-budget.test.sh +333 -0
- package/.agents/tests/fm-startup-network.test.sh +712 -0
- package/.agents/tests/fm-stow-cascade.test.sh +370 -0
- package/.agents/tests/fm-subagent-pretool-check.test.sh +291 -0
- package/.agents/tests/fm-supervision-events.test.sh +156 -0
- package/.agents/tests/fm-supervision-instructions.test.sh +189 -0
- package/.agents/tests/fm-tangle-guard.test.sh +277 -0
- package/.agents/tests/fm-task-delivery.test.sh +420 -0
- package/.agents/tests/fm-task-inbox.test.sh +513 -0
- package/.agents/tests/fm-teardown-endpoint-safety.test.sh +374 -0
- package/.agents/tests/fm-teardown.test.sh +2665 -0
- package/.agents/tests/fm-test-fixture-cleanup.test.sh +172 -0
- package/.agents/tests/fm-test-fixtures.test.sh +132 -0
- package/.agents/tests/fm-test-isolation-proof.test.sh +278 -0
- package/.agents/tests/fm-test-run.test.sh +1196 -0
- package/.agents/tests/fm-tmux-agent-liveness.test.sh +354 -0
- package/.agents/tests/fm-tmux-submit-busy.test.sh +340 -0
- package/.agents/tests/fm-tool-update-check.test.sh +1044 -0
- package/.agents/tests/fm-trace-context-lib.test.sh +253 -0
- package/.agents/tests/fm-trace-context-spawn.test.sh +581 -0
- package/.agents/tests/fm-transition-lib.test.sh +54 -0
- package/.agents/tests/fm-turnend-guard.test.sh +1822 -0
- package/.agents/tests/fm-update.test.sh +304 -0
- package/.agents/tests/fm-vendor-auth-probe.test.sh +395 -0
- package/.agents/tests/fm-voice-relay.test.sh +4741 -0
- package/.agents/tests/fm-wake-daemon-lifecycle-e2e.test.sh +171 -0
- package/.agents/tests/fm-wake-drain-open-decisions-cursor.test.sh +356 -0
- package/.agents/tests/fm-wake-drain-open-decisions.test.sh +226 -0
- package/.agents/tests/fm-wake-drain-unread-status.test.sh +379 -0
- package/.agents/tests/fm-wake-queue.test.sh +1224 -0
- package/.agents/tests/fm-watch-arm.test.sh +821 -0
- package/.agents/tests/fm-watch-checkpoint.test.sh +87 -0
- package/.agents/tests/fm-watch-recovery-loop.test.sh +223 -0
- package/.agents/tests/fm-watch-triage.test.sh +3889 -0
- package/.agents/tests/fm-watcher-lock.test.sh +1133 -0
- package/.agents/tests/fm-x-mode.test.sh +3042 -0
- package/.agents/tests/gleipnir-guard-pretool-check.test.sh +51 -0
- package/.agents/tests/gleipnir-machine-lock.test.sh +130 -0
- package/.agents/tests/herdr-test-safety.sh +42 -0
- package/.agents/tests/lib.sh +315 -0
- package/.agents/tests/remote-herdr-fixture.sh +125 -0
- package/.agents/tests/secondmate-helpers.sh +193 -0
- package/.agents/tests/smoke.test.sh +93 -0
- package/.agents/tests/wake-helpers.sh +338 -0
- package/.agents/tests/zellij-test-safety.sh +60 -0
- package/.agents/tools/README.md +10 -0
- package/.agents/tools/bin/tasks-cli +267 -0
- package/.agents/tools/tasks-cli.ts +216 -0
- package/AGENTS.md +498 -0
- package/CHANGELOG.md +1913 -0
- package/LICENSE +1 -1
- package/NOTICE +63 -0
- package/README.md +114 -60
- package/RULES/01-domains.md +51 -0
- package/RULES/02-agents.md +78 -0
- package/RULES/03-houses.md +61 -0
- package/RULES/04-hoard.md +97 -0
- package/RULES/05-platforms.md +67 -0
- package/RULES/06-append-only.md +108 -0
- package/RULES/07-config.md +101 -0
- package/RULES/08-delivery-gate.md +75 -0
- package/RULES/09-electron.md +58 -0
- package/RULES/README.md +22 -0
- package/bin/README.md +107 -0
- package/bin/a2a-mcp.sh +133 -0
- package/bin/a2a-serve.py +103 -0
- package/bin/a2a-serve.sh +8 -0
- package/bin/a2a-talk.sh +50 -0
- package/bin/a2abridge-ensure.sh +108 -0
- package/bin/agent-run.sh +43 -0
- package/bin/agents-config.sh +256 -0
- package/bin/apodex-smoke-test.sh +105 -0
- package/bin/bifrost-bridge.sh +131 -0
- package/bin/bifrost-ingress.sh +77 -0
- package/bin/bootstrap-macos.sh +112 -0
- package/bin/bootstrap-windows.ps1 +112 -0
- package/bin/branch-guard.sh +64 -0
- package/bin/brokk +73 -0
- package/bin/brokk-classify-lib.sh +1770 -0
- package/bin/brokk-lease-lib.sh +222 -0
- package/bin/brokk-lease.sh +193 -0
- package/bin/brokk-lint.sh +65 -0
- package/bin/brokk-send.sh +51 -0
- package/bin/brokk-update.sh +9 -0
- package/bin/brokk-wake-grant.sh +127 -0
- package/bin/brokk-wake-lib.sh +1849 -0
- package/bin/changelog-assemble.sh +154 -0
- package/bin/changelog-guard.sh +197 -0
- package/bin/crash-sense.sh +173 -0
- package/bin/design-check.sh +80 -0
- package/bin/design-icon.sh +153 -0
- package/bin/desktop-place.sh +224 -0
- package/bin/docs-guard.sh +39 -0
- package/bin/editor-place.sh +225 -0
- package/bin/eindri-acclaim.sh +47 -0
- package/bin/eindri-control.sh +28 -0
- package/bin/eindri-role.sh +83 -0
- package/bin/eindri-seat.sh +83 -0
- package/bin/eindri-seen.sh +54 -0
- package/bin/eindri-send.sh +15 -0
- package/bin/eindri-start.sh +98 -0
- package/bin/eindri-watch.sh +57 -0
- package/bin/einherjar-spawn.sh +576 -0
- package/bin/eir-doctor.sh +158 -0
- package/bin/erindi-brief.sh +271 -0
- package/bin/feature-inventory.sh +73 -0
- package/bin/fleet-apply.sh +81 -0
- package/bin/github-deploy.sh +94 -0
- package/bin/gjallarhorn-expose.sh +136 -0
- package/bin/gjallarhorn-notify.sh +71 -0
- package/bin/gjallarhorn-purge.sh +67 -0
- package/bin/gjallarhorn-tunnel.sh +85 -0
- package/bin/gleipnir-lock-lib.sh +284 -0
- package/bin/groa-update.sh +130 -0
- package/bin/hall-snapshot.sh +111 -0
- package/bin/hamr-harness.sh +246 -0
- package/bin/heimdall-ssh-keys.sh +374 -0
- package/bin/herdr-agents.py +44 -0
- package/bin/herdr-ensure.sh +104 -0
- package/bin/herdr-run.sh +452 -0
- package/bin/hermes-ensure.sh +88 -0
- package/bin/hlidskjalf-agents.sh +172 -0
- package/bin/hlidskjalf-usage.sh +196 -0
- package/bin/hoard-guard.sh +135 -0
- package/bin/hoard-lib.sh +105 -0
- package/bin/hodd.sh +102 -0
- package/bin/host-sense.sh +199 -0
- package/bin/huginn-research-worker.sh +121 -0
- package/bin/local-model-lock.sh +70 -0
- package/bin/mimir-bridge.py +259 -0
- package/bin/mimir-bridge.sh +100 -0
- package/bin/mimir-ingest.sh +131 -0
- package/bin/mimir-reflect.sh +27 -0
- package/bin/mimir.sh +113 -0
- package/bin/mjollnir-webhook.sh +72 -0
- package/bin/mjollnir.sh +104 -0
- package/bin/model-hardware.sh +96 -0
- package/bin/model-resolve.sh +160 -0
- package/bin/models-detect.sh +178 -0
- package/bin/nornir-cron-start.sh +162 -0
- package/bin/nornir-job-daily-briefing.sh +191 -0
- package/bin/nornir-job-git-sync.sh +152 -0
- package/bin/nornir-job-hall-snapshot.sh +28 -0
- package/bin/nornir-job-memory-housekeeping.sh +114 -0
- package/bin/nornir-job-nsr-compliance.sh +54 -0
- package/bin/nornir-job-observer.sh +166 -0
- package/bin/nornir-job-skillopt-sleep.sh +76 -0
- package/bin/npm-publish.sh +136 -0
- package/bin/omarchy-hook-install.sh +60 -0
- package/bin/omarchy-install.sh +161 -0
- package/bin/omarchy-plugins.sh +187 -0
- package/bin/omarchy-sense.sh +162 -0
- package/bin/perm-guard.sh +30 -0
- package/bin/pi-agent.sh +68 -0
- package/bin/pi-ensure.sh +77 -0
- package/bin/pi-local.sh +37 -0
- package/bin/pi-model-agnostic.sh +71 -0
- package/bin/pi-seat.sh +63 -0
- package/bin/prereq-ensure.sh +166 -0
- package/bin/private-guard.sh +58 -0
- package/bin/project-git.sh +69 -0
- package/bin/public-guard.sh +71 -0
- package/bin/ratatoskr.sh +49 -0
- package/bin/realm-lib.sh +32 -0
- package/bin/repo-scrub.sh +62 -0
- package/bin/rodd-operational-input.sh +250 -0
- package/bin/runes-append.sh +198 -0
- package/bin/saga-session-start.sh +172 -0
- package/bin/saga-sessionstart-run.sh +79 -0
- package/bin/saga-wake-drain.sh +41 -0
- package/bin/secret-guard.sh +63 -0
- package/bin/sessrumnir-ensure.sh +100 -0
- package/bin/sessrumnir-sync.sh +237 -0
- package/bin/sessrumnir.sh +104 -0
- package/bin/skillopt-setup.sh +76 -0
- package/bin/skuld-branch-outcome.sh +203 -0
- package/bin/skuld-branch-prompt.sh +101 -0
- package/bin/smidja-bootstrap.sh +94 -0
- package/bin/smidja-observe.sh +86 -0
- package/bin/syn-arm-pretool-check.sh +57 -0
- package/bin/syn-asset-pretool-check.sh +89 -0
- package/bin/syn-cd-pretool-check.sh +28 -0
- package/bin/syn-guard-pretool-check.sh +83 -0
- package/bin/syn-spawn-pretool-check.sh +58 -0
- package/bin/syn-turnend-guard.sh +36 -0
- package/bin/syn-watch-arm.sh +99 -0
- package/bin/tailscale-sync.sh +203 -0
- package/bin/telegram-bot.sh +82 -0
- package/bin/toolchain.sh +74 -0
- package/bin/ui-truth-check.sh +75 -0
- package/bin/utgard.sh +111 -0
- package/bin/valhalla.sh +115 -0
- package/bin/valknut-load.sh +298 -0
- package/bin/vor-crew-state.sh +124 -0
- package/bin/wedge-notify.sh +60 -0
- package/bin/workspace-provision.sh +80 -0
- package/bin/workspace-rag.sh +163 -0
- package/bin/wyrd-db.sh +47 -0
- package/bin/yggdrasil.sh +152 -0
- package/bin/ymir-edit.sh +145 -0
- package/bin/ymir-install.sh +806 -0
- package/bin/ymir-invite.sh +93 -0
- package/bin/ymir-isolation.sh +88 -0
- package/bin/ymir-migrate.sh +61 -0
- package/bin/ymir-plan.sh +356 -0
- package/bin/ymir-platform.sh +348 -0
- package/bin/ymir-say.sh +110 -0
- package/bin/ymir-setup-auth.sh +119 -0
- package/bin/ymir-validate.sh +193 -0
- package/bin/ymir.js +74 -16
- package/docs/Architecture.md +339 -0
- package/docs/design.md +351 -0
- package/docs/installations/README.md +63 -0
- package/docs/installations/ymir-home.md +93 -0
- package/docs/integration.md +105 -0
- package/docs/lore.md +1060 -0
- package/docs/research/agent-harnesses.md +290 -0
- package/docs/runbooks/README.md +19 -0
- package/docs/runbooks/a2a.md +91 -0
- package/docs/runbooks/agent-permissions.md +99 -0
- package/docs/runbooks/agents.md +57 -0
- package/docs/runbooks/models.md +69 -0
- package/docs/runbooks/realm-onboarding.md +60 -0
- package/docs/runbooks/secrets-and-hoard.md +60 -0
- package/docs/runbooks/updates-and-migrations.md +110 -0
- package/docs/runbooks/worktrees-isolation.md +72 -0
- package/docs/session-start.md +105 -0
- package/docs/skillopt-integration.md +321 -0
- package/docs/sync-system.md +66 -0
- package/docs/workspaces.md +104 -0
- package/docs/ymir-rut.md +260 -0
- package/package.json +54 -14
- package/packaging/build.sh +74 -0
- package/packaging/macos/Ymir Installer.command +43 -0
- package/packaging/windows/ymir-setup.nsi +74 -0
- package/scripts/README.md +22 -0
- package/scripts/electron.sh +269 -0
- package/scripts/lower.sh +34 -0
- package/scripts/raise.sh +42 -0
- package/scripts/start.sh +217 -0
- package/scripts/stop.sh +113 -0
- package/install.sh +0 -35
|
@@ -0,0 +1,4741 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# tests/fm-voice-relay.test.sh - the spoken interface's wire format, read scope and handover.
|
|
3
|
+
#
|
|
4
|
+
# Every case here runs offline. The three things worth protecting in this feature
|
|
5
|
+
# are all offline properties: the frame format the laptop and the desktop agree
|
|
6
|
+
# on, WHAT a status answer is allowed to contain, and the fact that real work is
|
|
7
|
+
# handed to firstmate rather than done by the voice agent. The latency work that
|
|
8
|
+
# motivated the build is a measurement, not an assertion, so it is not here; the
|
|
9
|
+
# numbers and the method live in docs/voice-relay.md.
|
|
10
|
+
#
|
|
11
|
+
# THE CASE THAT MATTERS MOST is the confidentiality boundary. The captain granted
|
|
12
|
+
# the voice agent full read access to their records, so the reader defaults to the
|
|
13
|
+
# wider scope. What keeps that safe is structural: finished work and free-form
|
|
14
|
+
# note bodies are never assembled at all, and those are exactly where commercial
|
|
15
|
+
# detail accumulates. This suite plants a marker in both places and fails if it
|
|
16
|
+
# ever reaches an answer, so widening the reader later breaks a test instead of
|
|
17
|
+
# quietly widening what is sent to a model in another region.
|
|
18
|
+
#
|
|
19
|
+
# The markers below are invented for this fixture. Real customer names are not
|
|
20
|
+
# committed to a test file.
|
|
21
|
+
set -u
|
|
22
|
+
|
|
23
|
+
# shellcheck source=tests/lib.sh
|
|
24
|
+
. "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
|
|
25
|
+
|
|
26
|
+
command -v python3 >/dev/null 2>&1 || { echo "skip: python3 not found"; exit 0; }
|
|
27
|
+
|
|
28
|
+
TMP_ROOT=$(fm_test_tmproot fm-voice-relay)
|
|
29
|
+
HOME_FIXTURE="$TMP_ROOT/home"
|
|
30
|
+
|
|
31
|
+
# NEVER_TOKEN sits in finished work and in a note body: both are excluded by
|
|
32
|
+
# construction, so it must never appear at any scope.
|
|
33
|
+
NEVER_TOKEN=NEVERLEAVESTHISHOST
|
|
34
|
+
# DENY_TOKEN sits in the title of open in-flight work, which the wide scope does
|
|
35
|
+
# report. It proves the deny list suppresses something that genuinely would have
|
|
36
|
+
# been sent, rather than passing vacuously against text no answer contains.
|
|
37
|
+
DENY_TOKEN=DENYMEPLEASE
|
|
38
|
+
|
|
39
|
+
seed_home() {
|
|
40
|
+
mkdir -p "$HOME_FIXTURE/data" "$HOME_FIXTURE/state" "$HOME_FIXTURE/config"
|
|
41
|
+
cat > "$HOME_FIXTURE/data/backlog.md" <<EOF
|
|
42
|
+
# Backlog
|
|
43
|
+
|
|
44
|
+
## In flight
|
|
45
|
+
- [ ] alpha-one - Fix the sign-in redirect (repo: alpha) (kind: ship) (priority: 0) (since 2026-08-01)
|
|
46
|
+
Long note body written for someone with the whole file open, mentioning
|
|
47
|
+
$NEVER_TOKEN and the rate we agreed.
|
|
48
|
+
- [ ] beta-two - Decide the storage shape (repo: beta) (kind: captain) (priority: 1)
|
|
49
|
+
- [ ] gamma-three - Migrate the $DENY_TOKEN account onto the new plan (repo: gamma) (kind: ship)
|
|
50
|
+
|
|
51
|
+
## Queued
|
|
52
|
+
- [ ] delta-four - Add the retry (repo: delta) (kind: ship) (hold-kind: captain) (hold: waiting on the captain)
|
|
53
|
+
- [ ] epsilon-five - Tidy the logs (repo: epsilon) (kind: ship)
|
|
54
|
+
|
|
55
|
+
## Done
|
|
56
|
+
- [x] old-six - Shipped the $NEVER_TOKEN integration (repo: alpha) (done 2026-07-01)
|
|
57
|
+
# An unticked line under Done, held for the captain. Two separate mechanisms keep
|
|
58
|
+
# finished work out of an answer: the section is never parsed, and a ticked box is
|
|
59
|
+
# dropped. A ticked line is blocked by both, so it cannot tell which one broke.
|
|
60
|
+
# This line is blocked by the section rule alone, and the list of what waits on
|
|
61
|
+
# the captain is assembled with no section filter at all, so it is the one place
|
|
62
|
+
# where losing that rule would put finished work into a spoken answer.
|
|
63
|
+
- [ ] old-seven - Decide the $NEVER_TOKEN renewal (repo: alpha) (kind: captain)
|
|
64
|
+
EOF
|
|
65
|
+
|
|
66
|
+
fm_write_meta "$HOME_FIXTURE/state/alpha-one.meta" \
|
|
67
|
+
kind=ship mode=no-mistakes window=firstmate:fm-alpha-one \
|
|
68
|
+
pr=https://github.com/example/alpha/pull/7
|
|
69
|
+
fm_write_meta "$HOME_FIXTURE/state/gamma-three.meta" kind=ship mode=direct-PR
|
|
70
|
+
printf 'working: reading the failing test\n' > "$HOME_FIXTURE/state/alpha-one.status"
|
|
71
|
+
# The bracketed shape, which is what bin/fm-secondmate-report.sh writes and
|
|
72
|
+
# what a keyed decision line looks like. Status metadata sits between the verb
|
|
73
|
+
# and the colon, so a reader that only cuts at the colon reads no verb here.
|
|
74
|
+
printf 'blocked [key=api-shape]: needs a credential (via-helper)\n' \
|
|
75
|
+
> "$HOME_FIXTURE/state/gamma-three.status"
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
records_status() {
|
|
79
|
+
python3 "$ROOT/bin/fm_voice_records.py" status --home "$HOME_FIXTURE" "$@"
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
seed_home
|
|
83
|
+
|
|
84
|
+
# --- the wire format --------------------------------------------------------
|
|
85
|
+
#
|
|
86
|
+
# A desynchronised stream must be a loud error rather than audio interpreted as
|
|
87
|
+
# a frame header. The laptop copy of this module is the only other place these
|
|
88
|
+
# rules exist, so they are pinned here.
|
|
89
|
+
|
|
90
|
+
python3 - "$ROOT/bin" <<'PY' || fail "frame round trip"
|
|
91
|
+
import os, io, sys
|
|
92
|
+
sys.path.insert(0, sys.argv[1])
|
|
93
|
+
import fm_voice_frame as frame
|
|
94
|
+
|
|
95
|
+
def check(cond, label):
|
|
96
|
+
if not cond:
|
|
97
|
+
sys.exit("frame: " + label)
|
|
98
|
+
|
|
99
|
+
# Round trip of every kind, including an empty payload and a large one.
|
|
100
|
+
buf = io.BytesIO()
|
|
101
|
+
w = frame.Writer(buf)
|
|
102
|
+
w.send(frame.TALK_START)
|
|
103
|
+
w.send(frame.AUDIO, b"\x01\x02" * 1600)
|
|
104
|
+
w.send_json(frame.TEXT, {"role": "USER", "text": "how is the fleet"})
|
|
105
|
+
w.send(frame.TALK_END)
|
|
106
|
+
buf.seek(0)
|
|
107
|
+
r = frame.Reader(buf)
|
|
108
|
+
got = []
|
|
109
|
+
while True:
|
|
110
|
+
item = r.read()
|
|
111
|
+
if item is None:
|
|
112
|
+
break
|
|
113
|
+
got.append(item)
|
|
114
|
+
check([k for k, _ in got] == [frame.TALK_START, frame.AUDIO, frame.TEXT,
|
|
115
|
+
frame.TALK_END], "kinds did not round trip")
|
|
116
|
+
check(got[1][1] == b"\x01\x02" * 1600, "audio payload did not round trip")
|
|
117
|
+
check(frame.decode_json(got[2][1])["text"] == "how is the fleet",
|
|
118
|
+
"json payload did not round trip")
|
|
119
|
+
|
|
120
|
+
# A clean close between frames is end of input, not an error.
|
|
121
|
+
check(frame.Reader(io.BytesIO(b"")).read() is None, "clean EOF should be None")
|
|
122
|
+
|
|
123
|
+
# A stream cut inside a payload is a dropped connection and must say so.
|
|
124
|
+
try:
|
|
125
|
+
frame.Reader(io.BytesIO(frame.encode(frame.AUDIO, b"12345")[:-2])).read()
|
|
126
|
+
sys.exit("frame: truncated payload was accepted")
|
|
127
|
+
except frame.FrameError:
|
|
128
|
+
pass
|
|
129
|
+
|
|
130
|
+
# A payload that never starts at all is the same fault, one byte earlier.
|
|
131
|
+
try:
|
|
132
|
+
frame.Reader(io.BytesIO(frame.HEADER.pack(frame.AUDIO, 5))).read()
|
|
133
|
+
sys.exit("frame: a header with no payload behind it was accepted")
|
|
134
|
+
except frame.FrameError:
|
|
135
|
+
pass
|
|
136
|
+
|
|
137
|
+
# A stream cut inside the HEADER is a dropped connection too, and must NOT come
|
|
138
|
+
# back as the clean close checked above. A lost SSH connection does not politely
|
|
139
|
+
# end on a frame boundary, and a partial header read as end of input records the
|
|
140
|
+
# turn as unanswered with no error, which puts a transport failure into a results
|
|
141
|
+
# file as an ordinary turn the model did not answer.
|
|
142
|
+
for cut in range(1, frame.HEADER.size):
|
|
143
|
+
try:
|
|
144
|
+
frame.Reader(io.BytesIO(frame.encode(frame.BYE)[:cut])).read()
|
|
145
|
+
sys.exit("frame: %d header bytes then EOF was read as a clean close" % cut)
|
|
146
|
+
except frame.FrameError as exc:
|
|
147
|
+
check("header" in str(exc),
|
|
148
|
+
"a truncated header should name itself: %s" % exc)
|
|
149
|
+
|
|
150
|
+
# Audio bytes that happen to look like a header must not be trusted.
|
|
151
|
+
for bad in (b"\xffZZZZ", frame.HEADER.pack(frame.AUDIO, frame.MAX_PAYLOAD + 1)):
|
|
152
|
+
try:
|
|
153
|
+
frame.Reader(io.BytesIO(bad)).read()
|
|
154
|
+
sys.exit("frame: accepted a bad header: %r" % bad)
|
|
155
|
+
except frame.FrameError:
|
|
156
|
+
pass
|
|
157
|
+
|
|
158
|
+
try:
|
|
159
|
+
frame.encode(b"?")
|
|
160
|
+
sys.exit("frame: encoded an unknown kind")
|
|
161
|
+
except frame.FrameError:
|
|
162
|
+
pass
|
|
163
|
+
|
|
164
|
+
try:
|
|
165
|
+
frame.encode(frame.AUDIO, b"x" * (frame.MAX_PAYLOAD + 1))
|
|
166
|
+
sys.exit("frame: encoded an oversized payload")
|
|
167
|
+
except frame.FrameError:
|
|
168
|
+
pass
|
|
169
|
+
|
|
170
|
+
# The relay's uplink decodes headers itself, on an asynchronous stream Reader
|
|
171
|
+
# cannot drive, and calls this to decide whether to read the payload at all. A
|
|
172
|
+
# bogus length has to be refused BEFORE the read, or the relay waits for up to
|
|
173
|
+
# four gigabytes while the captain waits for an answer.
|
|
174
|
+
for kind, length in ((b"\xff", 0), (frame.AUDIO, frame.MAX_PAYLOAD + 1)):
|
|
175
|
+
try:
|
|
176
|
+
frame.check_header(kind, length)
|
|
177
|
+
sys.exit("frame: check_header accepted %r/%d" % (kind, length))
|
|
178
|
+
except frame.FrameError:
|
|
179
|
+
pass
|
|
180
|
+
frame.check_header(frame.AUDIO, frame.MAX_PAYLOAD)
|
|
181
|
+
PY
|
|
182
|
+
pass "wire format round trips and rejects a desynchronised stream"
|
|
183
|
+
|
|
184
|
+
# --- the relay's uplink ------------------------------------------------------
|
|
185
|
+
#
|
|
186
|
+
# The relay decodes the captain's frames on an asyncio stream, which frame.Reader
|
|
187
|
+
# cannot drive, so the rule above has to be exercised on that path as well. The
|
|
188
|
+
# failure mode it prevents is not a wrong answer, it is no answer: a relay that
|
|
189
|
+
# reads the payload before it checks the length waits inside readexactly for up
|
|
190
|
+
# to four gigabytes that will never arrive, while the captain sits in front of a
|
|
191
|
+
# client that never replies. The timeout below is what tells those two apart.
|
|
192
|
+
|
|
193
|
+
python3 - "$ROOT/bin" <<'PY' || fail "relay uplink"
|
|
194
|
+
import asyncio, sys
|
|
195
|
+
sys.path.insert(0, sys.argv[1])
|
|
196
|
+
import importlib.util, pathlib
|
|
197
|
+
spec = importlib.util.spec_from_file_location(
|
|
198
|
+
"relay", str(pathlib.Path(sys.argv[1]) / "fm-voice-relay.py"))
|
|
199
|
+
relay = importlib.util.module_from_spec(spec)
|
|
200
|
+
spec.loader.exec_module(relay)
|
|
201
|
+
import fm_voice_frame as frame
|
|
202
|
+
|
|
203
|
+
def check(cond, label):
|
|
204
|
+
if not cond:
|
|
205
|
+
sys.exit("uplink: " + label)
|
|
206
|
+
|
|
207
|
+
async def read(raw):
|
|
208
|
+
reader = asyncio.StreamReader()
|
|
209
|
+
reader.feed_data(raw)
|
|
210
|
+
return await asyncio.wait_for(relay.read_uplink_frame(reader), timeout=5)
|
|
211
|
+
|
|
212
|
+
audio = b"\x01\x02" * 8
|
|
213
|
+
check(asyncio.run(read(frame.encode(frame.AUDIO, audio))) == (frame.AUDIO, audio),
|
|
214
|
+
"a frame carrying audio did not survive the uplink")
|
|
215
|
+
check(asyncio.run(read(frame.encode(frame.TALK_END))) == (frame.TALK_END, b""),
|
|
216
|
+
"an empty control frame did not survive the uplink")
|
|
217
|
+
|
|
218
|
+
# A header with nothing behind it. Refused on the header, this raises at once;
|
|
219
|
+
# read first and checked later, it hangs, so a timeout here is the regression.
|
|
220
|
+
for bad in (frame.HEADER.pack(frame.AUDIO, frame.MAX_PAYLOAD + 1),
|
|
221
|
+
b"\xff\x00\x00\x10\x00"):
|
|
222
|
+
try:
|
|
223
|
+
asyncio.run(read(bad))
|
|
224
|
+
sys.exit("uplink: accepted a bad header: %r" % bad)
|
|
225
|
+
except frame.FrameError:
|
|
226
|
+
pass
|
|
227
|
+
except (asyncio.TimeoutError, TimeoutError):
|
|
228
|
+
sys.exit("uplink: waited for the payload of a bad header instead of "
|
|
229
|
+
"refusing it: %r" % bad)
|
|
230
|
+
PY
|
|
231
|
+
pass "the relay refuses a desynchronised uplink header instead of waiting for its payload"
|
|
232
|
+
|
|
233
|
+
# --- whose account, whose model ---------------------------------------------
|
|
234
|
+
#
|
|
235
|
+
# A region, a model id and an AWS profile name somebody's account and somebody's
|
|
236
|
+
# choices, so nothing here ships one. A home that has configured none of them
|
|
237
|
+
# cannot start the relay at all, and it is told which file to write rather than
|
|
238
|
+
# quietly reaching an API in somebody else's account. That configuration IS the
|
|
239
|
+
# opt-in, so this case is what keeps the feature off by default.
|
|
240
|
+
|
|
241
|
+
CONFIG_HOME="$TMP_ROOT/unconfigured"
|
|
242
|
+
mkdir -p "$CONFIG_HOME/config"
|
|
243
|
+
|
|
244
|
+
python3 - "$ROOT/bin" "$CONFIG_HOME" <<'PY' || fail "relay configuration"
|
|
245
|
+
import sys
|
|
246
|
+
sys.path.insert(0, sys.argv[1])
|
|
247
|
+
import importlib.util, os, pathlib
|
|
248
|
+
spec = importlib.util.spec_from_file_location(
|
|
249
|
+
"relay", str(pathlib.Path(sys.argv[1]) / "fm-voice-relay.py"))
|
|
250
|
+
relay = importlib.util.module_from_spec(spec)
|
|
251
|
+
spec.loader.exec_module(relay)
|
|
252
|
+
import fm_voice_records as records
|
|
253
|
+
|
|
254
|
+
home = sys.argv[2]
|
|
255
|
+
for name in ("FM_VOICE_REGION", "FM_VOICE_MODEL", "FM_VOICE_PROFILE", "FM_VOICE_ID",
|
|
256
|
+
"FM_CONFIG_OVERRIDE"):
|
|
257
|
+
os.environ.pop(name, None)
|
|
258
|
+
|
|
259
|
+
def check(cond, label):
|
|
260
|
+
if not cond:
|
|
261
|
+
sys.exit("configuration: " + label)
|
|
262
|
+
|
|
263
|
+
# An unconfigured home refuses, and the refusal is the path to write.
|
|
264
|
+
try:
|
|
265
|
+
relay.resolve_settings(relay.parse_args(["--serve", "--home", home]))
|
|
266
|
+
sys.exit("configuration: an unconfigured home started the relay")
|
|
267
|
+
except records.RecordError as exc:
|
|
268
|
+
check("voice-region" in str(exc),
|
|
269
|
+
"the refusal should name the file to write: %s" % exc)
|
|
270
|
+
check(home in str(exc), "and it should be this home's path: %s" % exc)
|
|
271
|
+
|
|
272
|
+
# One file at a time: the region alone is not enough to reach a model.
|
|
273
|
+
with open(os.path.join(home, "config", "voice-region"), "w") as handle:
|
|
274
|
+
handle.write("# the region this home talks to\neu-somewhere-1\n")
|
|
275
|
+
try:
|
|
276
|
+
relay.resolve_settings(relay.parse_args(["--serve", "--home", home]))
|
|
277
|
+
sys.exit("configuration: a home with no model id started the relay")
|
|
278
|
+
except records.RecordError as exc:
|
|
279
|
+
check("voice-model" in str(exc),
|
|
280
|
+
"the refusal should name the missing model file: %s" % exc)
|
|
281
|
+
|
|
282
|
+
with open(os.path.join(home, "config", "voice-model"), "w") as handle:
|
|
283
|
+
handle.write("some.model-v1:0\n")
|
|
284
|
+
options = relay.resolve_settings(relay.parse_args(["--serve", "--home", home]))
|
|
285
|
+
check(options.region == "eu-somewhere-1",
|
|
286
|
+
"the configured region should be used, comment and all: %r" % options.region)
|
|
287
|
+
check(options.model == "some.model-v1:0",
|
|
288
|
+
"the configured model should be used: %r" % options.model)
|
|
289
|
+
# No profile configured means ambient credentials only, which is a real choice
|
|
290
|
+
# rather than a missing one, so it is not a refusal.
|
|
291
|
+
check(options.profile == "", "an absent profile should stay empty: %r" % options.profile)
|
|
292
|
+
check(options.voice == relay.VOICE,
|
|
293
|
+
"an absent voice should fall back to the shipped one: %r" % options.voice)
|
|
294
|
+
|
|
295
|
+
# The environment overrides a file for a single run.
|
|
296
|
+
os.environ["FM_VOICE_REGION"] = "eu-elsewhere-2"
|
|
297
|
+
os.environ["FM_VOICE_ID"] = "amy"
|
|
298
|
+
options = relay.resolve_settings(relay.parse_args(["--serve", "--home", home]))
|
|
299
|
+
check(options.region == "eu-elsewhere-2",
|
|
300
|
+
"the environment should override the file: %r" % options.region)
|
|
301
|
+
check(options.voice == "amy", "the voice should be overridable: %r" % options.voice)
|
|
302
|
+
|
|
303
|
+
# And an explicit flag overrides both.
|
|
304
|
+
options = relay.resolve_settings(
|
|
305
|
+
relay.parse_args(["--serve", "--home", home, "--region", "eu-flag-3"]))
|
|
306
|
+
check(options.region == "eu-flag-3", "a flag should win: %r" % options.region)
|
|
307
|
+
|
|
308
|
+
os.environ.pop("FM_VOICE_REGION", None)
|
|
309
|
+
os.environ.pop("FM_VOICE_ID", None)
|
|
310
|
+
|
|
311
|
+
# THE PROFILE IS READ BY PRESENCE, NOT BY TRUTHINESS. An empty FM_VOICE_PROFILE is
|
|
312
|
+
# the captain saying "use the credentials I already have", so it must not fall
|
|
313
|
+
# through to a configured profile and spend a second exporting from an account
|
|
314
|
+
# they just opted out of. fm-inbox.sh reads its own equivalent that way and
|
|
315
|
+
# docs/configuration.md promises it for both.
|
|
316
|
+
with open(os.path.join(home, "config", "voice-profile"), "w") as handle:
|
|
317
|
+
handle.write("a-configured-profile\n")
|
|
318
|
+
options = relay.resolve_settings(relay.parse_args(["--serve", "--home", home]))
|
|
319
|
+
check(options.profile == "a-configured-profile",
|
|
320
|
+
"a configured profile should be used: %r" % options.profile)
|
|
321
|
+
|
|
322
|
+
os.environ["FM_VOICE_PROFILE"] = ""
|
|
323
|
+
options = relay.resolve_settings(relay.parse_args(["--serve", "--home", home]))
|
|
324
|
+
check(options.profile == "",
|
|
325
|
+
"an empty FM_VOICE_PROFILE must force ambient credentials: %r" % options.profile)
|
|
326
|
+
|
|
327
|
+
os.environ["FM_VOICE_PROFILE"] = "another-profile"
|
|
328
|
+
options = relay.resolve_settings(relay.parse_args(["--serve", "--home", home]))
|
|
329
|
+
check(options.profile == "another-profile",
|
|
330
|
+
"a set FM_VOICE_PROFILE should override the file: %r" % options.profile)
|
|
331
|
+
os.environ.pop("FM_VOICE_PROFILE", None)
|
|
332
|
+
|
|
333
|
+
# An empty region, by contrast, is not a choice about anything, so it still falls
|
|
334
|
+
# through to the file rather than refusing.
|
|
335
|
+
os.environ["FM_VOICE_REGION"] = ""
|
|
336
|
+
options = relay.resolve_settings(relay.parse_args(["--serve", "--home", home]))
|
|
337
|
+
check(options.region == "eu-somewhere-1",
|
|
338
|
+
"an empty region variable should fall through to the file: %r" % options.region)
|
|
339
|
+
os.environ.pop("FM_VOICE_REGION", None)
|
|
340
|
+
|
|
341
|
+
# --help must work in a home that has configured nothing, or the captain cannot
|
|
342
|
+
# read how to configure it.
|
|
343
|
+
PY
|
|
344
|
+
pass "the relay reads whose account to use from this home and refuses to guess"
|
|
345
|
+
|
|
346
|
+
set +e
|
|
347
|
+
help_out=$(python3 "$ROOT/bin/fm-voice-relay.py" --help 2>&1)
|
|
348
|
+
help_code=$?
|
|
349
|
+
set -e
|
|
350
|
+
expect_code 0 "$help_code" "--help must work with no configuration: $help_out"
|
|
351
|
+
assert_contains "$help_out" 'voice-region' \
|
|
352
|
+
"--help should name the files a home has to write"
|
|
353
|
+
pass "an unconfigured home can still read how to configure the relay"
|
|
354
|
+
|
|
355
|
+
# The captain inbox is the same rule with a different consequence: note, status,
|
|
356
|
+
# list and drain make no model call, so they must keep working unconfigured. The
|
|
357
|
+
# voice handover depends on note, so that is not a nicety.
|
|
358
|
+
#
|
|
359
|
+
# EVERY FM_INBOX_ VARIABLE IS NEUTRALIZED HERE, at the harness rather than in each
|
|
360
|
+
# case, and the list is read out of the environment rather than written down, so a
|
|
361
|
+
# knob added later cannot quietly survive into a refusal case. A shell that
|
|
362
|
+
# exports a region and a model id would otherwise walk these cases straight past
|
|
363
|
+
# the refusal they assert and into a real model call: an offline suite that can
|
|
364
|
+
# spend the operator's credentials is worse than a failing one.
|
|
365
|
+
inbox_env=()
|
|
366
|
+
while IFS= read -r inbox_knob; do
|
|
367
|
+
[ -n "$inbox_knob" ] || continue
|
|
368
|
+
inbox_env+=(-u "$inbox_knob")
|
|
369
|
+
done < <(env | sed -n 's/^\(FM_INBOX_[A-Za-z0-9_]*\)=.*/\1/p' | sort -u)
|
|
370
|
+
inbox_env+=(FM_HOME="$CONFIG_HOME" FM_STATE_OVERRIDE="$CONFIG_HOME/state"
|
|
371
|
+
FM_CONFIG_OVERRIDE="$CONFIG_HOME/config")
|
|
372
|
+
|
|
373
|
+
# And a stub that records any attempt, so "no model call" is a checked fact rather
|
|
374
|
+
# than a claim about control flow. The real aws would need credentials; this one
|
|
375
|
+
# leaves evidence and exits non-zero.
|
|
376
|
+
INBOX_FAKEBIN=$(fm_fakebin "$TMP_ROOT/inbox-fake")
|
|
377
|
+
AWS_CALLED="$TMP_ROOT/aws-was-called"
|
|
378
|
+
cat > "$INBOX_FAKEBIN/aws" <<SH
|
|
379
|
+
#!/usr/bin/env bash
|
|
380
|
+
printf 'aws %s\n' "\$*" >> "$AWS_CALLED"
|
|
381
|
+
exit 9
|
|
382
|
+
SH
|
|
383
|
+
chmod +x "$INBOX_FAKEBIN/aws"
|
|
384
|
+
inbox_env+=(PATH="$INBOX_FAKEBIN:$PATH")
|
|
385
|
+
|
|
386
|
+
set +e
|
|
387
|
+
ask_out=$(env "${inbox_env[@]}" "$ROOT/bin/fm-inbox.sh" ask "how is the fleet" 2>&1)
|
|
388
|
+
ask_code=$?
|
|
389
|
+
set -e
|
|
390
|
+
[ "$ask_code" -ne 0 ] || fail "ask ran with nothing configured"
|
|
391
|
+
assert_contains "$ask_out" 'inbox-region' \
|
|
392
|
+
"the first refusal should name the region file: $ask_out"
|
|
393
|
+
|
|
394
|
+
# One file at a time, so each refusal names one thing to do.
|
|
395
|
+
printf 'eu-somewhere-1\n' > "$CONFIG_HOME/config/inbox-region"
|
|
396
|
+
set +e
|
|
397
|
+
ask_out=$(env "${inbox_env[@]}" "$ROOT/bin/fm-inbox.sh" ask "how is the fleet" 2>&1)
|
|
398
|
+
ask_code=$?
|
|
399
|
+
set -e
|
|
400
|
+
[ "$ask_code" -ne 0 ] || fail "ask ran without a configured model"
|
|
401
|
+
assert_contains "$ask_out" 'inbox-ask-model' \
|
|
402
|
+
"the refusal should name the model file to write: $ask_out"
|
|
403
|
+
|
|
404
|
+
set +e
|
|
405
|
+
say_out=$(printf '' | env "${inbox_env[@]}" "$ROOT/bin/fm-inbox.sh" say 2>&1)
|
|
406
|
+
say_code=$?
|
|
407
|
+
set -e
|
|
408
|
+
[ "$say_code" -ne 0 ] || fail "say ran without a configured model"
|
|
409
|
+
assert_contains "$say_out" 'inbox-stt-model' \
|
|
410
|
+
"the refusal should name the model file to write: $say_out"
|
|
411
|
+
|
|
412
|
+
rm -f "$CONFIG_HOME/config/inbox-region"
|
|
413
|
+
unconfigured_note=$(env "${inbox_env[@]}" \
|
|
414
|
+
"$ROOT/bin/fm-inbox.sh" note "the handover must work with no configuration") \
|
|
415
|
+
|| fail "note should not need any configuration"
|
|
416
|
+
assert_contains "$unconfigured_note" 'queued ' "note should still queue a record"
|
|
417
|
+
assert_absent "$AWS_CALLED" \
|
|
418
|
+
"no case above may reach a model: the aws stub recorded an attempt"
|
|
419
|
+
pass "the model-backed subcommands refuse by name while note keeps working"
|
|
420
|
+
|
|
421
|
+
# --help prints the whole header block, and finds where that block ends rather
|
|
422
|
+
# than counting lines to it, so growing the header cannot silently truncate the
|
|
423
|
+
# help again. The PRIVACY paragraph is the part that matters: it is the only place
|
|
424
|
+
# a new operator is told which subcommands send audio or text off this host, and a
|
|
425
|
+
# fixed line range had already dropped it.
|
|
426
|
+
inbox_help=$("$ROOT/bin/fm-inbox.sh" --help) || fail "fm-inbox.sh --help failed"
|
|
427
|
+
assert_contains "$inbox_help" 'PRIVACY:' \
|
|
428
|
+
"the help must say which subcommands send anything to a model"
|
|
429
|
+
assert_contains "$inbox_help" 'make no network call at all' \
|
|
430
|
+
"the help must name the subcommands that stay on this host"
|
|
431
|
+
assert_contains "$inbox_help" 'FM_HOME' \
|
|
432
|
+
"the help must keep its environment section"
|
|
433
|
+
assert_contains "$inbox_help" 'inbox-ask-model' \
|
|
434
|
+
"the help must name the files a home has to write"
|
|
435
|
+
assert_contains "$inbox_help" 'fm-inbox.sh note' \
|
|
436
|
+
"the help must still open with the usage it always had"
|
|
437
|
+
pass "fm-inbox.sh --help prints its whole header, privacy paragraph included"
|
|
438
|
+
|
|
439
|
+
# --- the tool surface the two sides share -----------------------------------
|
|
440
|
+
#
|
|
441
|
+
# The relay declares the tools and fm_voice_records implements them. Renaming one
|
|
442
|
+
# side only would leave the agent unable to answer or unable to hand over, and
|
|
443
|
+
# the failure would look like a confused model rather than a typo.
|
|
444
|
+
|
|
445
|
+
python3 - "$ROOT/bin" <<'PY' || fail "tool surface"
|
|
446
|
+
import sys
|
|
447
|
+
sys.path.insert(0, sys.argv[1])
|
|
448
|
+
import importlib.util, pathlib
|
|
449
|
+
spec = importlib.util.spec_from_file_location(
|
|
450
|
+
"relay", str(pathlib.Path(sys.argv[1]) / "fm-voice-relay.py"))
|
|
451
|
+
relay = importlib.util.module_from_spec(spec)
|
|
452
|
+
spec.loader.exec_module(relay)
|
|
453
|
+
|
|
454
|
+
names = sorted(t["toolSpec"]["name"] for t in relay.TOOLS["tools"])
|
|
455
|
+
if names != ["get_fleet_status", "hand_over_to_firstmate"]:
|
|
456
|
+
sys.exit("relay declares unexpected tools: %s" % names)
|
|
457
|
+
|
|
458
|
+
# The handover tool has to take the request text, or the agent can announce a
|
|
459
|
+
# handover it never performed.
|
|
460
|
+
handover = [t["toolSpec"] for t in relay.TOOLS["tools"]
|
|
461
|
+
if t["toolSpec"]["name"] == "hand_over_to_firstmate"][0]
|
|
462
|
+
import json
|
|
463
|
+
schema = json.loads(handover["inputSchema"]["json"])
|
|
464
|
+
if schema.get("required") != ["request"]:
|
|
465
|
+
sys.exit("hand_over_to_firstmate must require the request text")
|
|
466
|
+
|
|
467
|
+
# Push to talk is the default for this build and is meant to be one setting.
|
|
468
|
+
options = relay.parse_args(["--self-test", "x.pcm"])
|
|
469
|
+
if options.tail_ms <= 0:
|
|
470
|
+
sys.exit("the trailing silence default must be positive; 0 is never answered")
|
|
471
|
+
PY
|
|
472
|
+
pass "the relay and the reader agree on the tool names and the handover argument"
|
|
473
|
+
|
|
474
|
+
# --- credentials -------------------------------------------------------------
|
|
475
|
+
#
|
|
476
|
+
# The relay rebuilds the model session on every turn, on purpose. Resolving AWS
|
|
477
|
+
# credentials belongs to the relay's start rather than to that rebuild: the
|
|
478
|
+
# sandbox profile's credential_process costs about a second, and a second spent
|
|
479
|
+
# there is a second added to the delay this whole build exists to keep honest.
|
|
480
|
+
# Nothing here talks to AWS; the resolver is replaced with a counter.
|
|
481
|
+
|
|
482
|
+
python3 - "$ROOT/bin" <<'PY' || fail "credential reuse"
|
|
483
|
+
import asyncio, datetime, os, sys, time
|
|
484
|
+
sys.path.insert(0, sys.argv[1])
|
|
485
|
+
import importlib.util, pathlib
|
|
486
|
+
spec = importlib.util.spec_from_file_location(
|
|
487
|
+
"relay", str(pathlib.Path(sys.argv[1]) / "fm-voice-relay.py"))
|
|
488
|
+
relay = importlib.util.module_from_spec(spec)
|
|
489
|
+
spec.loader.exec_module(relay)
|
|
490
|
+
|
|
491
|
+
def check(cond, label):
|
|
492
|
+
if not cond:
|
|
493
|
+
sys.exit("credentials: " + label)
|
|
494
|
+
|
|
495
|
+
AWS_VARS = ("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN",
|
|
496
|
+
"AWS_CREDENTIAL_EXPIRATION")
|
|
497
|
+
|
|
498
|
+
def iso(at):
|
|
499
|
+
return datetime.datetime.fromtimestamp(at, datetime.timezone.utc).isoformat()
|
|
500
|
+
|
|
501
|
+
# Credentials taken from the environment cannot be refreshed in place, because
|
|
502
|
+
# os.environ never gets fresher values while this process runs. So they are only
|
|
503
|
+
# preferred while they are usable, and what decides that is the deadline the
|
|
504
|
+
# exporter states beside the keys. Treating one as eternal strands a long-lived
|
|
505
|
+
# relay: every session after the real deadline is rejected for an expired token.
|
|
506
|
+
for name in AWS_VARS:
|
|
507
|
+
os.environ.pop(name, None)
|
|
508
|
+
|
|
509
|
+
check(relay.ambient_credentials() is None,
|
|
510
|
+
"an environment with no keys must send the relay to the profile")
|
|
511
|
+
|
|
512
|
+
os.environ["AWS_ACCESS_KEY_ID"] = "AKIAEXAMPLE"
|
|
513
|
+
check(relay.ambient_credentials() is None,
|
|
514
|
+
"a key id with no secret beside it must be refused, not indexed blindly")
|
|
515
|
+
|
|
516
|
+
os.environ["AWS_SECRET_ACCESS_KEY"] = "s3cret"
|
|
517
|
+
ambient = relay.ambient_credentials()
|
|
518
|
+
check(ambient[0]["aws_access_key_id"] == "AKIAEXAMPLE",
|
|
519
|
+
"a complete environment should be used: %r" % (ambient,))
|
|
520
|
+
check(ambient[1] is None,
|
|
521
|
+
"long-term keys, with no session token and no stated deadline, do not expire")
|
|
522
|
+
|
|
523
|
+
os.environ["AWS_SESSION_TOKEN"] = "t0ken"
|
|
524
|
+
check(relay.ambient_credentials()[1] is relay.EXPIRY_UNKNOWN,
|
|
525
|
+
"a session token has a deadline whether or not the shell stated it")
|
|
526
|
+
|
|
527
|
+
os.environ["AWS_CREDENTIAL_EXPIRATION"] = iso(time.time() + 3600)
|
|
528
|
+
check(isinstance(relay.ambient_credentials()[1], float),
|
|
529
|
+
"a stated deadline should be carried through as the expiry")
|
|
530
|
+
# The environment wins while it is usable, so this never shells out to a profile.
|
|
531
|
+
picked = relay.resolve_credentials("no-such-profile")
|
|
532
|
+
check(picked[0]["aws_session_token"] == "t0ken",
|
|
533
|
+
"the environment should be preferred over the profile while it is usable")
|
|
534
|
+
check(picked[2] == relay.FROM_ENVIRONMENT,
|
|
535
|
+
"the resolver must say where the credentials came from: %r" % (picked[2],))
|
|
536
|
+
|
|
537
|
+
os.environ["AWS_CREDENTIAL_EXPIRATION"] = iso(time.time() - 1)
|
|
538
|
+
check(relay.ambient_credentials() is None,
|
|
539
|
+
"expired ambient credentials must send the relay to the profile instead")
|
|
540
|
+
|
|
541
|
+
os.environ["AWS_CREDENTIAL_EXPIRATION"] = iso(time.time() + 60)
|
|
542
|
+
check(relay.ambient_credentials(margin=300) is None,
|
|
543
|
+
"ambient credentials inside the refresh margin must not start a session")
|
|
544
|
+
check(relay.ambient_credentials(margin=0) is not None,
|
|
545
|
+
"the same credentials are still usable when no margin is asked for")
|
|
546
|
+
|
|
547
|
+
# A profile export that fails must be an ordinary exception. SystemExit would walk
|
|
548
|
+
# straight through the per-turn boundary in handle_uplink_frame and end the relay,
|
|
549
|
+
# and since credentials are resolved lazily this refusal can land mid-conversation.
|
|
550
|
+
class Failed:
|
|
551
|
+
returncode = 1
|
|
552
|
+
stdout = ""
|
|
553
|
+
stderr = "The config profile (nobody) could not be found"
|
|
554
|
+
|
|
555
|
+
real_run = relay.subprocess.run
|
|
556
|
+
relay.subprocess.run = lambda *a, **k: Failed()
|
|
557
|
+
try:
|
|
558
|
+
relay.profile_credentials("nobody")
|
|
559
|
+
sys.exit("credentials: a failed profile export was accepted")
|
|
560
|
+
except relay.CredentialError as exc:
|
|
561
|
+
check(isinstance(exc, Exception),
|
|
562
|
+
"the refusal must be an ordinary exception, not a SystemExit")
|
|
563
|
+
check("nobody" in str(exc), "the refusal should name the profile: %s" % exc)
|
|
564
|
+
except SystemExit:
|
|
565
|
+
sys.exit("credentials: a failed profile export raised SystemExit")
|
|
566
|
+
finally:
|
|
567
|
+
relay.subprocess.run = real_run
|
|
568
|
+
|
|
569
|
+
# No profile and no environment is also a named refusal rather than a traceback
|
|
570
|
+
# from inside the AWS CLI argument list.
|
|
571
|
+
try:
|
|
572
|
+
relay.profile_credentials("")
|
|
573
|
+
sys.exit("credentials: an empty profile was accepted")
|
|
574
|
+
except relay.CredentialError as exc:
|
|
575
|
+
check("voice-profile" in str(exc),
|
|
576
|
+
"the refusal should name the file to write: %s" % exc)
|
|
577
|
+
|
|
578
|
+
for name in AWS_VARS:
|
|
579
|
+
os.environ.pop(name, None)
|
|
580
|
+
|
|
581
|
+
calls = []
|
|
582
|
+
stamp = [""]
|
|
583
|
+
delay = [0.0]
|
|
584
|
+
|
|
585
|
+
def fake(profile, verbose=False, margin=0, allow_ambient=True):
|
|
586
|
+
# Whatever the profile says about expiry reaches the cache through the real
|
|
587
|
+
# parser, so the fixture hands it a stamp rather than a decided answer.
|
|
588
|
+
calls.append(profile)
|
|
589
|
+
time.sleep(delay[0])
|
|
590
|
+
return ({"aws_access_key_id": "AK%d" % len(calls)},
|
|
591
|
+
relay._expires_at(stamp[0]), relay.FROM_PROFILE)
|
|
592
|
+
|
|
593
|
+
real_resolve = relay.resolve_credentials
|
|
594
|
+
relay.resolve_credentials = fake
|
|
595
|
+
|
|
596
|
+
async def take(cache, count):
|
|
597
|
+
return [await cache.get() for _ in range(count)]
|
|
598
|
+
|
|
599
|
+
# Three sessions, one resolution: the second and third turns pay nothing.
|
|
600
|
+
cache = relay.Credentials("a-profile")
|
|
601
|
+
got = asyncio.run(take(cache, 3))
|
|
602
|
+
check(len(calls) == 1, "three sessions resolved credentials %d times" % len(calls))
|
|
603
|
+
check([c["aws_access_key_id"] for c in got] == ["AK1"] * 3,
|
|
604
|
+
"every session should get the same credentials: %s" % got)
|
|
605
|
+
|
|
606
|
+
# A session that edits what it was handed must not edit what the next one gets.
|
|
607
|
+
got[0]["aws_access_key_id"] = "tampered"
|
|
608
|
+
check(asyncio.run(take(cache, 1))[0]["aws_access_key_id"] == "AK1",
|
|
609
|
+
"one session must not be able to corrupt the shared credentials")
|
|
610
|
+
|
|
611
|
+
# Credentials with an expiry are refreshed ahead of it, because a relay left
|
|
612
|
+
# running outlives them and a dead credential is a dead session.
|
|
613
|
+
del calls[:]
|
|
614
|
+
stamp[0] = iso(time.time() + relay.Credentials.REFRESH_MARGIN - 1)
|
|
615
|
+
cache = relay.Credentials("a-profile")
|
|
616
|
+
asyncio.run(take(cache, 2))
|
|
617
|
+
check(len(calls) == 2,
|
|
618
|
+
"credentials near expiry should be refreshed, resolved %d times" % len(calls))
|
|
619
|
+
|
|
620
|
+
# An expiry this interpreter cannot read is NOT an expiry that never comes. The
|
|
621
|
+
# credential works, its deadline does not, so it is held for the same margin and
|
|
622
|
+
# no longer. Read as "never expires" it would be cached past the real deadline
|
|
623
|
+
# and every session from then on would fail to start with no way back.
|
|
624
|
+
class Bounded(relay.Credentials):
|
|
625
|
+
REFRESH_MARGIN = 0.05
|
|
626
|
+
|
|
627
|
+
del calls[:]
|
|
628
|
+
stamp[0] = "expires some time on Tuesday"
|
|
629
|
+
cache = Bounded("a-profile")
|
|
630
|
+
asyncio.run(take(cache, 2))
|
|
631
|
+
check(len(calls) == 1,
|
|
632
|
+
"an unreadable expiry should still be reused within the margin: %d" % len(calls))
|
|
633
|
+
time.sleep(0.1)
|
|
634
|
+
asyncio.run(take(cache, 1))
|
|
635
|
+
check(len(calls) == 2,
|
|
636
|
+
"an unreadable expiry must not be cached for the life of the relay")
|
|
637
|
+
|
|
638
|
+
# An absent expiry keeps meaning what it says: this credential does not expire.
|
|
639
|
+
del calls[:]
|
|
640
|
+
stamp[0] = ""
|
|
641
|
+
cache = Bounded("a-profile")
|
|
642
|
+
asyncio.run(take(cache, 1))
|
|
643
|
+
time.sleep(0.1)
|
|
644
|
+
asyncio.run(take(cache, 1))
|
|
645
|
+
check(len(calls) == 1,
|
|
646
|
+
"a credential with no expiry should not be resolved again: %d" % len(calls))
|
|
647
|
+
|
|
648
|
+
# Whenever a resolution does happen it must stay off the event loop, or the
|
|
649
|
+
# relay stops reading the captain's audio for as long as it takes.
|
|
650
|
+
del calls[:]
|
|
651
|
+
stamp[0] = ""
|
|
652
|
+
delay[0] = 0.3
|
|
653
|
+
|
|
654
|
+
async def resolve_while_the_loop_runs():
|
|
655
|
+
ticks = []
|
|
656
|
+
|
|
657
|
+
async def tick():
|
|
658
|
+
for _ in range(20):
|
|
659
|
+
await asyncio.sleep(0.01)
|
|
660
|
+
ticks.append(1)
|
|
661
|
+
|
|
662
|
+
task = asyncio.create_task(tick())
|
|
663
|
+
await relay.Credentials("slow-profile").get()
|
|
664
|
+
during = len(ticks)
|
|
665
|
+
task.cancel()
|
|
666
|
+
return during
|
|
667
|
+
|
|
668
|
+
during = asyncio.run(resolve_while_the_loop_runs())
|
|
669
|
+
check(during >= 2,
|
|
670
|
+
"the event loop ran %d times during a 0.3s credential resolution" % during)
|
|
671
|
+
|
|
672
|
+
# GIVING UP ON AMBIENT CREDENTIALS HAS TO STICK. A bound that re-reads the same
|
|
673
|
+
# environment is not a bound: os.environ never gets fresher values while this
|
|
674
|
+
# process runs, so the same stale keys would come back every time and every
|
|
675
|
+
# session past the real deadline would fail while a working profile went untried.
|
|
676
|
+
# This drives the real resolver, with only the profile export replaced.
|
|
677
|
+
relay.resolve_credentials = real_resolve
|
|
678
|
+
exports = []
|
|
679
|
+
|
|
680
|
+
def fake_profile(profile, verbose=False):
|
|
681
|
+
exports.append(profile)
|
|
682
|
+
return {"aws_access_key_id": "FROM-PROFILE",
|
|
683
|
+
"aws_secret_access_key": "s", "aws_session_token": None}, None
|
|
684
|
+
|
|
685
|
+
real_profile = relay.profile_credentials
|
|
686
|
+
relay.profile_credentials = fake_profile
|
|
687
|
+
os.environ["AWS_ACCESS_KEY_ID"] = "AKIAENVIRONMENT"
|
|
688
|
+
os.environ["AWS_SECRET_ACCESS_KEY"] = "s3cret"
|
|
689
|
+
os.environ["AWS_SESSION_TOKEN"] = "stale-token"
|
|
690
|
+
os.environ.pop("AWS_CREDENTIAL_EXPIRATION", None)
|
|
691
|
+
|
|
692
|
+
cache = Bounded("a-profile")
|
|
693
|
+
first = asyncio.run(cache.get())
|
|
694
|
+
check(first["aws_session_token"] == "stale-token",
|
|
695
|
+
"usable ambient credentials should be preferred: %r" % first)
|
|
696
|
+
check(exports == [], "the profile should not be consulted while ambient ones hold")
|
|
697
|
+
time.sleep(0.1)
|
|
698
|
+
later = [asyncio.run(cache.get()) for _ in range(3)]
|
|
699
|
+
check([c["aws_access_key_id"] for c in later] == ["FROM-PROFILE"] * 3,
|
|
700
|
+
"past the margin the relay must ask the profile, not re-read the "
|
|
701
|
+
"environment it already gave up on: %r" % later)
|
|
702
|
+
check(len(exports) == 1,
|
|
703
|
+
"and the profile answer is then cached like any other: %d exports" % len(exports))
|
|
704
|
+
|
|
705
|
+
# WITH NO PROFILE THERE IS NOTHING TO ESCALATE TO, and a relay configured that
|
|
706
|
+
# way is a documented shape. Giving up on the environment there would end every
|
|
707
|
+
# turn from the margin onwards, with a message saying there are no credentials in
|
|
708
|
+
# the environment while the process is still holding them. The bound becomes a
|
|
709
|
+
# re-read instead: the keys may be stale, which is between AWS and whoever
|
|
710
|
+
# exported them, but the conversation survives.
|
|
711
|
+
del exports[:]
|
|
712
|
+
cache = Bounded("")
|
|
713
|
+
kept = [asyncio.run(cache.get())]
|
|
714
|
+
for _ in range(3):
|
|
715
|
+
time.sleep(0.1)
|
|
716
|
+
kept.append(asyncio.run(cache.get()))
|
|
717
|
+
check([c["aws_session_token"] for c in kept] == ["stale-token"] * 4,
|
|
718
|
+
"a profile-free relay must keep answering from the environment: %r" % kept)
|
|
719
|
+
check(exports == [], "and must not try to export from a profile it does not have")
|
|
720
|
+
|
|
721
|
+
# Even a credential whose stated deadline has already passed, for the same
|
|
722
|
+
# reason: there is no fresher source, so refusing is a dead relay rather than a
|
|
723
|
+
# safer one.
|
|
724
|
+
os.environ["AWS_CREDENTIAL_EXPIRATION"] = iso(time.time() - 60)
|
|
725
|
+
cache = Bounded("")
|
|
726
|
+
past = asyncio.run(cache.get())
|
|
727
|
+
check(past["aws_session_token"] == "stale-token",
|
|
728
|
+
"an expired ambient credential is still the only answer available: %r" % past)
|
|
729
|
+
# With a profile, that same credential is abandoned for it, as before.
|
|
730
|
+
cache = Bounded("a-profile")
|
|
731
|
+
check(asyncio.run(cache.get())["aws_access_key_id"] == "FROM-PROFILE",
|
|
732
|
+
"an expired ambient credential should be abandoned when a profile exists")
|
|
733
|
+
os.environ.pop("AWS_CREDENTIAL_EXPIRATION", None)
|
|
734
|
+
|
|
735
|
+
# A PROFILE THAT CANNOT ANSWER must not cost the environment. Abandoning ambient
|
|
736
|
+
# credentials is justified by the profile answering, so it is only decided once the
|
|
737
|
+
# profile has: otherwise one failed export strands a relay that is still holding
|
|
738
|
+
# keys, and every later turn names a profile while the answer sits in os.environ.
|
|
739
|
+
del exports[:]
|
|
740
|
+
|
|
741
|
+
def refusing_profile(profile, verbose=False):
|
|
742
|
+
exports.append(profile)
|
|
743
|
+
raise relay.CredentialError(
|
|
744
|
+
"could not get credentials for profile {}".format(profile))
|
|
745
|
+
|
|
746
|
+
relay.profile_credentials = refusing_profile
|
|
747
|
+
os.environ["AWS_ACCESS_KEY_ID"] = "AKIAENVIRONMENT"
|
|
748
|
+
os.environ["AWS_SECRET_ACCESS_KEY"] = "s3cret"
|
|
749
|
+
os.environ["AWS_SESSION_TOKEN"] = "still-held-token"
|
|
750
|
+
os.environ.pop("AWS_CREDENTIAL_EXPIRATION", None)
|
|
751
|
+
|
|
752
|
+
cache = Bounded("a-profile")
|
|
753
|
+
check(asyncio.run(cache.get())["aws_session_token"] == "still-held-token",
|
|
754
|
+
"usable ambient credentials should be preferred while they hold")
|
|
755
|
+
kept = []
|
|
756
|
+
for _ in range(3):
|
|
757
|
+
time.sleep(0.1)
|
|
758
|
+
try:
|
|
759
|
+
kept.append(asyncio.run(cache.get())["aws_session_token"])
|
|
760
|
+
except relay.CredentialError:
|
|
761
|
+
kept.append("CredentialError")
|
|
762
|
+
check(kept == ["still-held-token"] * 3,
|
|
763
|
+
"a refusing profile must cost one attempt, not the conversation: %r" % kept)
|
|
764
|
+
check(exports, "and the profile should have been tried at least once: %r" % exports)
|
|
765
|
+
|
|
766
|
+
# An environment with nothing in it and no profile is still a named refusal, so
|
|
767
|
+
# this restores the real resolver rather than asking the stub to pretend.
|
|
768
|
+
for name in AWS_VARS:
|
|
769
|
+
os.environ.pop(name, None)
|
|
770
|
+
relay.profile_credentials = real_profile
|
|
771
|
+
refused = None
|
|
772
|
+
try:
|
|
773
|
+
asyncio.run(Bounded("").get())
|
|
774
|
+
except relay.CredentialError as exc:
|
|
775
|
+
refused = str(exc)
|
|
776
|
+
check(refused is not None, "no credentials anywhere should refuse")
|
|
777
|
+
check("voice-profile" in refused,
|
|
778
|
+
"and the refusal should name the file to write: %s" % refused)
|
|
779
|
+
PY
|
|
780
|
+
pass "credentials are resolved once per relay, refreshed before expiry, off the event loop"
|
|
781
|
+
|
|
782
|
+
# --- a turn the model refuses -----------------------------------------------
|
|
783
|
+
#
|
|
784
|
+
# The relay rebuilds the model session on every turn by design, so every turn
|
|
785
|
+
# reaches the model and every turn can fail on its own: a throttle, a dropped
|
|
786
|
+
# stream, a token that went stale between turns. That has to cost the captain one
|
|
787
|
+
# turn rather than the whole session, because the alternative is a traceback on
|
|
788
|
+
# the stderr the client inherits and a relay restarted by hand.
|
|
789
|
+
|
|
790
|
+
python3 - "$ROOT/bin" <<'PY' || fail "failed turn"
|
|
791
|
+
import asyncio, sys
|
|
792
|
+
sys.path.insert(0, sys.argv[1])
|
|
793
|
+
import importlib.util, pathlib
|
|
794
|
+
spec = importlib.util.spec_from_file_location(
|
|
795
|
+
"relay", str(pathlib.Path(sys.argv[1]) / "fm-voice-relay.py"))
|
|
796
|
+
relay = importlib.util.module_from_spec(spec)
|
|
797
|
+
spec.loader.exec_module(relay)
|
|
798
|
+
import fm_voice_frame as frame
|
|
799
|
+
|
|
800
|
+
def check(cond, label):
|
|
801
|
+
if not cond:
|
|
802
|
+
sys.exit("failed turn: " + label)
|
|
803
|
+
|
|
804
|
+
class Down:
|
|
805
|
+
def __init__(self):
|
|
806
|
+
self.notices = []
|
|
807
|
+
|
|
808
|
+
def send(self, kind, payload=b""):
|
|
809
|
+
pass
|
|
810
|
+
|
|
811
|
+
def send_json(self, kind, obj):
|
|
812
|
+
if kind == frame.NOTICE:
|
|
813
|
+
self.notices.append(obj)
|
|
814
|
+
|
|
815
|
+
class Stub:
|
|
816
|
+
"""A session that records what it was asked, or raises where the model would."""
|
|
817
|
+
|
|
818
|
+
def __init__(self, raises=None):
|
|
819
|
+
self.raises = raises
|
|
820
|
+
self.replies = 0
|
|
821
|
+
self.failed = False
|
|
822
|
+
self.ended = asyncio.Event()
|
|
823
|
+
self.turn = {}
|
|
824
|
+
self.calls = []
|
|
825
|
+
|
|
826
|
+
async def _step(self, name):
|
|
827
|
+
self.calls.append(name)
|
|
828
|
+
if self.raises is not None:
|
|
829
|
+
raise self.raises
|
|
830
|
+
|
|
831
|
+
async def talk_start(self):
|
|
832
|
+
await self._step("talk_start")
|
|
833
|
+
|
|
834
|
+
async def audio(self, pcm):
|
|
835
|
+
await self._step("audio:%d" % len(pcm))
|
|
836
|
+
|
|
837
|
+
async def talk_end(self):
|
|
838
|
+
await self._step("talk_end")
|
|
839
|
+
|
|
840
|
+
options = relay.parse_args(["--serve"])
|
|
841
|
+
|
|
842
|
+
async def drive(session, items):
|
|
843
|
+
down = Down()
|
|
844
|
+
serving = True
|
|
845
|
+
for kind, payload in items:
|
|
846
|
+
session, serving = await relay.handle_uplink_frame(
|
|
847
|
+
kind, payload, session, options, down)
|
|
848
|
+
if not serving:
|
|
849
|
+
break
|
|
850
|
+
return session, serving, down
|
|
851
|
+
|
|
852
|
+
# The ordinary path is unchanged: the frames reach the session in order.
|
|
853
|
+
good = Stub()
|
|
854
|
+
session, serving, down = asyncio.run(drive(good, [
|
|
855
|
+
(frame.TALK_START, b""), (frame.AUDIO, b"1234"), (frame.TALK_END, b"")]))
|
|
856
|
+
check(good.calls == ["talk_start", "audio:4", "talk_end"],
|
|
857
|
+
"a good turn should reach the session: %s" % good.calls)
|
|
858
|
+
check(serving and not good.failed,
|
|
859
|
+
"a good turn must not mark the session spent")
|
|
860
|
+
check(down.notices == [], "a good turn should not announce a failure")
|
|
861
|
+
|
|
862
|
+
# A model failure mid-turn: the captain is told what happened, the relay stays
|
|
863
|
+
# up, and the session is marked spent so nothing reuses a dead stream.
|
|
864
|
+
broken = Stub(raises=RuntimeError("ThrottlingException"))
|
|
865
|
+
session, serving, down = asyncio.run(drive(broken, [(frame.AUDIO, b"1234")]))
|
|
866
|
+
check(serving, "a failed turn must not stop the relay")
|
|
867
|
+
check(session is broken and broken.failed,
|
|
868
|
+
"a failed session must be marked spent")
|
|
869
|
+
check([n["event"] for n in down.notices] == ["turn-failed"],
|
|
870
|
+
"a failed turn must be announced to the client: %s" % down.notices)
|
|
871
|
+
check("ThrottlingException" in down.notices[0].get("error", ""),
|
|
872
|
+
"the notice should name the failure: %s" % down.notices[0])
|
|
873
|
+
|
|
874
|
+
# ONCE PER TURN, not once per frame. The captain is still holding the talk key
|
|
875
|
+
# when the failure lands, so the rest of that press is another thirty audio
|
|
876
|
+
# frames, one per hundred milliseconds. The client says every notice out loud on
|
|
877
|
+
# stderr, so reporting each one would put ten identical lines a second in front of
|
|
878
|
+
# the captain while they are still speaking, and would keep calling into a session
|
|
879
|
+
# that is already gone.
|
|
880
|
+
held = Stub(raises=RuntimeError("ValidationException"))
|
|
881
|
+
frames = [(frame.TALK_START, b"")] + [(frame.AUDIO, b"x" * 3200)] * 30
|
|
882
|
+
frames.append((frame.TALK_END, b""))
|
|
883
|
+
session, serving, down = asyncio.run(drive(held, frames))
|
|
884
|
+
check(serving, "a failed turn must not stop the relay")
|
|
885
|
+
check(len(down.notices) == 1,
|
|
886
|
+
"a failed turn must be announced once, not once per frame: %d notices"
|
|
887
|
+
% len(down.notices))
|
|
888
|
+
check(held.calls == ["talk_start"],
|
|
889
|
+
"nothing after the failure should reach the dead session: %s" % held.calls)
|
|
890
|
+
|
|
891
|
+
# And the next talk key rebuilds instead of reusing it, which is what marking it
|
|
892
|
+
# spent is for.
|
|
893
|
+
renewed = []
|
|
894
|
+
fresh = Stub()
|
|
895
|
+
real_renew = relay.renew
|
|
896
|
+
|
|
897
|
+
async def fake_renew(session, options, down):
|
|
898
|
+
renewed.append(session)
|
|
899
|
+
return fresh
|
|
900
|
+
|
|
901
|
+
relay.renew = fake_renew
|
|
902
|
+
session, serving, down = asyncio.run(drive(broken, [(frame.TALK_START, b"")]))
|
|
903
|
+
check(renewed == [broken], "a spent session must be replaced on the next turn")
|
|
904
|
+
check(session is fresh and fresh.calls == ["talk_start"],
|
|
905
|
+
"the replacement session must take the turn: %s" % fresh.calls)
|
|
906
|
+
|
|
907
|
+
# A reconnect that fails is itself just a failed turn: the captain presses the
|
|
908
|
+
# key again rather than restarting the relay.
|
|
909
|
+
async def failing_renew(session, options, down):
|
|
910
|
+
raise RuntimeError("EndpointConnectionError")
|
|
911
|
+
|
|
912
|
+
relay.renew = failing_renew
|
|
913
|
+
spent = Stub()
|
|
914
|
+
spent.replies = 1
|
|
915
|
+
session, serving, down = asyncio.run(drive(spent, [(frame.TALK_START, b"")]))
|
|
916
|
+
check(serving, "a failed reconnect must not stop the relay")
|
|
917
|
+
check(spent.failed, "a failed reconnect must leave the session spent")
|
|
918
|
+
check([n["event"] for n in down.notices] == ["turn-failed"],
|
|
919
|
+
"a failed reconnect must be announced: %s" % down.notices)
|
|
920
|
+
check(spent.calls == [], "a session whose reconnect failed must not be spoken to")
|
|
921
|
+
|
|
922
|
+
# Quit still ends the loop, so the relay exits when the client says so.
|
|
923
|
+
session, serving, down = asyncio.run(drive(Stub(), [(frame.QUIT, b"")]))
|
|
924
|
+
check(not serving, "quit must end the loop")
|
|
925
|
+
|
|
926
|
+
# A reconnect that fails part way must not strand the session it was building.
|
|
927
|
+
# start() opens the model stream and a reader task before it sends anything, and
|
|
928
|
+
# the relay now survives the failure and retries, so a session left open here
|
|
929
|
+
# would accumulate one live stream and one live task per retry, all of them still
|
|
930
|
+
# writing into the shared downlink.
|
|
931
|
+
class Partial:
|
|
932
|
+
"""A session whose start fails after it would have opened the stream."""
|
|
933
|
+
|
|
934
|
+
def __init__(self, *args):
|
|
935
|
+
self.closed = 0
|
|
936
|
+
self.credentials = None
|
|
937
|
+
self.connect_seconds = None
|
|
938
|
+
|
|
939
|
+
async def start(self):
|
|
940
|
+
raise RuntimeError("ServiceUnavailableException")
|
|
941
|
+
|
|
942
|
+
async def close(self):
|
|
943
|
+
self.closed += 1
|
|
944
|
+
|
|
945
|
+
built = []
|
|
946
|
+
|
|
947
|
+
def make_partial(options, down, credentials):
|
|
948
|
+
session = Partial()
|
|
949
|
+
built.append(session)
|
|
950
|
+
return session
|
|
951
|
+
|
|
952
|
+
relay.Session = make_partial
|
|
953
|
+
outgoing = Partial()
|
|
954
|
+
try:
|
|
955
|
+
asyncio.run(real_renew(outgoing, options, Down()))
|
|
956
|
+
sys.exit("failed turn: a failed reconnect was reported as success")
|
|
957
|
+
except RuntimeError:
|
|
958
|
+
pass
|
|
959
|
+
check(len(built) == 1, "renew should have built one replacement: %d" % len(built))
|
|
960
|
+
check(built[0].closed == 1,
|
|
961
|
+
"a session whose start failed must be closed, not stranded: %d closes"
|
|
962
|
+
% built[0].closed)
|
|
963
|
+
PY
|
|
964
|
+
pass "a turn the model refuses is announced and costs one turn, not the relay"
|
|
965
|
+
|
|
966
|
+
# --- audio that arrives with no turn open ------------------------------------
|
|
967
|
+
#
|
|
968
|
+
# Both listen modes send a talk start before any audio, so audio outside a turn
|
|
969
|
+
# means the capture callback raced the key release and a stray chunk landed behind
|
|
970
|
+
# the talk end. Opening a block for it would append the captain's stray tenth of a
|
|
971
|
+
# second to a session that is already answering, which is the unconditional
|
|
972
|
+
# barge-in the per-turn reconnect exists to avoid, and would leave that block open
|
|
973
|
+
# so the next turn skipped its own reset and its first-audio mark.
|
|
974
|
+
|
|
975
|
+
python3 - "$ROOT/bin" "$TMP_ROOT/stray-home" <<'PY' || fail "stray audio"
|
|
976
|
+
import asyncio, os, sys
|
|
977
|
+
sys.path.insert(0, sys.argv[1])
|
|
978
|
+
import importlib.util, pathlib
|
|
979
|
+
spec = importlib.util.spec_from_file_location(
|
|
980
|
+
"relay", str(pathlib.Path(sys.argv[1]) / "fm-voice-relay.py"))
|
|
981
|
+
relay = importlib.util.module_from_spec(spec)
|
|
982
|
+
spec.loader.exec_module(relay)
|
|
983
|
+
|
|
984
|
+
home = sys.argv[2]
|
|
985
|
+
os.makedirs(home, exist_ok=True)
|
|
986
|
+
|
|
987
|
+
def check(cond, label):
|
|
988
|
+
if not cond:
|
|
989
|
+
sys.exit("stray audio: " + label)
|
|
990
|
+
|
|
991
|
+
class Down:
|
|
992
|
+
def send(self, kind, payload=b""):
|
|
993
|
+
pass
|
|
994
|
+
|
|
995
|
+
def send_json(self, kind, obj):
|
|
996
|
+
pass
|
|
997
|
+
|
|
998
|
+
def arm_turn(self):
|
|
999
|
+
pass
|
|
1000
|
+
|
|
1001
|
+
def first_audio(self):
|
|
1002
|
+
return None
|
|
1003
|
+
|
|
1004
|
+
sent = []
|
|
1005
|
+
|
|
1006
|
+
async def record(event):
|
|
1007
|
+
sent.append(event)
|
|
1008
|
+
|
|
1009
|
+
session = relay.Session(relay.parse_args(["--serve", "--home", home]), Down(), None)
|
|
1010
|
+
session._send = record
|
|
1011
|
+
|
|
1012
|
+
# No turn open: the stray chunk goes nowhere, and no block is left behind for the
|
|
1013
|
+
# next turn to trip over. This session has no model stream either, so anything
|
|
1014
|
+
# that did try to open a block would raise rather than pass quietly.
|
|
1015
|
+
asyncio.run(session.audio(b"\x01" * 3200))
|
|
1016
|
+
check(sent == [], "audio with no turn open must not be forwarded: %r" % sent)
|
|
1017
|
+
check(session.audio_content is None,
|
|
1018
|
+
"and must not leave an audio block open: %r" % session.audio_content)
|
|
1019
|
+
|
|
1020
|
+
# Inside a turn it flows, so the guard is about the boundary and not about audio.
|
|
1021
|
+
asyncio.run(session.talk_start())
|
|
1022
|
+
del sent[:]
|
|
1023
|
+
asyncio.run(session.audio(b"\x01" * 3200))
|
|
1024
|
+
check([next(iter(event)) for event in sent] == ["audioInput"],
|
|
1025
|
+
"audio inside a turn must still be forwarded: %r" % sent)
|
|
1026
|
+
|
|
1027
|
+
# And talk end still pads with its trailing silence before closing the block,
|
|
1028
|
+
# which is the whole reason a push-to-talk clip gets answered at all.
|
|
1029
|
+
del sent[:]
|
|
1030
|
+
asyncio.run(session.talk_end())
|
|
1031
|
+
kinds = [next(iter(event)) for event in sent]
|
|
1032
|
+
check(kinds.count("audioInput") > 0 and kinds[-1] == "contentEnd",
|
|
1033
|
+
"talk end must pad with silence and then close the block: %r" % kinds)
|
|
1034
|
+
check(session.audio_content is None, "talk end must close the block")
|
|
1035
|
+
PY
|
|
1036
|
+
pass "audio that arrives with no turn open is dropped, not turned into a turn"
|
|
1037
|
+
|
|
1038
|
+
# --- a model stream that dies while answering --------------------------------
|
|
1039
|
+
#
|
|
1040
|
+
# The reader task is the other place a turn can fail, and it fails in the middle
|
|
1041
|
+
# of work: handling an event reaches back into the model to answer a tool call. A
|
|
1042
|
+
# failure there must still tell a waiting turn the session is over, and close()
|
|
1043
|
+
# must absorb it, because close() is the first thing renew does. Neither held
|
|
1044
|
+
# once, and the cost was not one lost turn but every later one: the reader task
|
|
1045
|
+
# kept its exception, close() re-raised it on every await, renew never reached
|
|
1046
|
+
# the line that builds a replacement, and the captain heard the same failure
|
|
1047
|
+
# forever with no way back short of restarting the relay.
|
|
1048
|
+
#
|
|
1049
|
+
# Releasing the waiting turn is only half of it. The client waits for a reply end
|
|
1050
|
+
# or a notice, so a reader failure that says nothing costs the captain their whole
|
|
1051
|
+
# timeout and leaves a record that says the turn was not answered without saying
|
|
1052
|
+
# why. It has to be named, once, and only when it really is a failure: a stream
|
|
1053
|
+
# that simply ends, and a stream that went away because close() asked it to, are
|
|
1054
|
+
# both ordinary and neither may look like one.
|
|
1055
|
+
|
|
1056
|
+
python3 - "$ROOT/bin" "$TMP_ROOT/reader-home" <<'PY' || fail "reader failure"
|
|
1057
|
+
import asyncio, json, os, sys
|
|
1058
|
+
sys.path.insert(0, sys.argv[1])
|
|
1059
|
+
import fm_voice_frame as frame
|
|
1060
|
+
import importlib.util, pathlib
|
|
1061
|
+
spec = importlib.util.spec_from_file_location(
|
|
1062
|
+
"relay", str(pathlib.Path(sys.argv[1]) / "fm-voice-relay.py"))
|
|
1063
|
+
relay = importlib.util.module_from_spec(spec)
|
|
1064
|
+
spec.loader.exec_module(relay)
|
|
1065
|
+
|
|
1066
|
+
home = sys.argv[2]
|
|
1067
|
+
os.makedirs(home, exist_ok=True)
|
|
1068
|
+
options = relay.parse_args(["--serve", "--home", home])
|
|
1069
|
+
|
|
1070
|
+
def check(cond, label):
|
|
1071
|
+
if not cond:
|
|
1072
|
+
sys.exit("reader failure: " + label)
|
|
1073
|
+
|
|
1074
|
+
class Down:
|
|
1075
|
+
def __init__(self):
|
|
1076
|
+
self.notices = []
|
|
1077
|
+
|
|
1078
|
+
def send(self, kind, payload=b""):
|
|
1079
|
+
pass
|
|
1080
|
+
|
|
1081
|
+
def send_json(self, kind, obj):
|
|
1082
|
+
self.notices.append(obj)
|
|
1083
|
+
|
|
1084
|
+
def arm_turn(self):
|
|
1085
|
+
pass
|
|
1086
|
+
|
|
1087
|
+
def first_audio(self):
|
|
1088
|
+
return None
|
|
1089
|
+
|
|
1090
|
+
class Input:
|
|
1091
|
+
async def send(self, chunk):
|
|
1092
|
+
pass
|
|
1093
|
+
|
|
1094
|
+
async def close(self):
|
|
1095
|
+
pass
|
|
1096
|
+
|
|
1097
|
+
class Payload:
|
|
1098
|
+
def __init__(self, raw):
|
|
1099
|
+
self.bytes_ = raw
|
|
1100
|
+
|
|
1101
|
+
class Result:
|
|
1102
|
+
def __init__(self, raw):
|
|
1103
|
+
self.value = Payload(raw)
|
|
1104
|
+
|
|
1105
|
+
class Receiver:
|
|
1106
|
+
def __init__(self, raw):
|
|
1107
|
+
self._raw = raw
|
|
1108
|
+
|
|
1109
|
+
async def receive(self):
|
|
1110
|
+
return None if self._raw is None else Result(self._raw)
|
|
1111
|
+
|
|
1112
|
+
class Stream:
|
|
1113
|
+
"""The scripted events, and then either a clean end or a stream that is gone."""
|
|
1114
|
+
|
|
1115
|
+
def __init__(self, raws, clean=False):
|
|
1116
|
+
self._raws = list(raws)
|
|
1117
|
+
self._clean = clean
|
|
1118
|
+
self.input_stream = Input()
|
|
1119
|
+
|
|
1120
|
+
async def await_output(self):
|
|
1121
|
+
if not self._raws:
|
|
1122
|
+
if self._clean:
|
|
1123
|
+
return (None, Receiver(None))
|
|
1124
|
+
raise RuntimeError("the model stream is gone")
|
|
1125
|
+
return (None, Receiver(self._raws.pop(0)))
|
|
1126
|
+
|
|
1127
|
+
TOOL_EVENT = json.dumps({"event": {"toolUse": {
|
|
1128
|
+
"toolName": "no_such_tool", "toolUseId": "t-1", "content": "{}"}}}).encode()
|
|
1129
|
+
|
|
1130
|
+
def failures(down):
|
|
1131
|
+
return [n for n in down.notices if n.get("event") == "turn-failed"]
|
|
1132
|
+
|
|
1133
|
+
async def poison(how):
|
|
1134
|
+
"""Break a session one of the ways the model side can break it."""
|
|
1135
|
+
down = Down()
|
|
1136
|
+
session = relay.Session(options, down, None)
|
|
1137
|
+
session.stream = Stream([TOOL_EVENT])
|
|
1138
|
+
# A turn is open and waiting for a reply, which is when this costs the most.
|
|
1139
|
+
session.turn["talk_end"] = 0.0
|
|
1140
|
+
if how == "handler":
|
|
1141
|
+
async def boom(event):
|
|
1142
|
+
raise RuntimeError("handling blew up")
|
|
1143
|
+
session._handle = boom
|
|
1144
|
+
elif how == "tool-result":
|
|
1145
|
+
# The real _handle and _run_tool, with the tool-result send failing: the
|
|
1146
|
+
# shape a dropped stream takes while the relay answers a tool call.
|
|
1147
|
+
async def refuse(obj):
|
|
1148
|
+
raise RuntimeError("the model stream is gone")
|
|
1149
|
+
session._send = refuse
|
|
1150
|
+
elif how == "drop":
|
|
1151
|
+
# Nothing to read and no clean end: the stream simply goes away, which is
|
|
1152
|
+
# what a network blip looks like from here.
|
|
1153
|
+
session.stream = Stream([])
|
|
1154
|
+
session.reader_task = asyncio.create_task(session._read_model())
|
|
1155
|
+
await asyncio.wait_for(session.ended.wait(), timeout=5)
|
|
1156
|
+
return down, session
|
|
1157
|
+
|
|
1158
|
+
async def one_case(how):
|
|
1159
|
+
"""Break a session, then take the next turn over the same relay."""
|
|
1160
|
+
down, session = await poison(how)
|
|
1161
|
+
check(session.turn_done.is_set(),
|
|
1162
|
+
"%s: a waiting turn must be released" % how)
|
|
1163
|
+
check(session.failed, "%s: the session must be marked spent" % how)
|
|
1164
|
+
named = failures(down)
|
|
1165
|
+
check(len(named) == 1,
|
|
1166
|
+
"%s: the captain must be told once, not never and not twice: %r"
|
|
1167
|
+
% (how, down.notices))
|
|
1168
|
+
check(named[0].get("error"),
|
|
1169
|
+
"%s: the notice must carry the cause: %r" % (how, named[0]))
|
|
1170
|
+
check(session.turn.get("failed") == named[0]["error"],
|
|
1171
|
+
"%s: the run record must carry the same cause: %r" % (how, session.turn))
|
|
1172
|
+
|
|
1173
|
+
# close() absorbs the stored failure however many times it is asked, which is
|
|
1174
|
+
# what lets the next turn get as far as building a replacement.
|
|
1175
|
+
for _ in range(3):
|
|
1176
|
+
await session.close()
|
|
1177
|
+
|
|
1178
|
+
built = []
|
|
1179
|
+
|
|
1180
|
+
class Fresh:
|
|
1181
|
+
def __init__(self, *args):
|
|
1182
|
+
self.connect_seconds = 0.02
|
|
1183
|
+
self.failed = False
|
|
1184
|
+
self.replies = 0
|
|
1185
|
+
self.ended = asyncio.Event()
|
|
1186
|
+
self.turns = 0
|
|
1187
|
+
built.append(self)
|
|
1188
|
+
|
|
1189
|
+
async def start(self):
|
|
1190
|
+
pass
|
|
1191
|
+
|
|
1192
|
+
async def talk_start(self):
|
|
1193
|
+
self.turns += 1
|
|
1194
|
+
|
|
1195
|
+
real, relay.Session = relay.Session, Fresh
|
|
1196
|
+
try:
|
|
1197
|
+
used, serving = await relay.handle_uplink_frame(
|
|
1198
|
+
frame.TALK_START, b"", session, options, down)
|
|
1199
|
+
finally:
|
|
1200
|
+
relay.Session = real
|
|
1201
|
+
check(len(built) == 1,
|
|
1202
|
+
"%s: the next talk key must build a session: %r" % (how, built))
|
|
1203
|
+
check(used is built[0] and serving,
|
|
1204
|
+
"%s: the relay must go on serving with it: %r %r" % (how, used, serving))
|
|
1205
|
+
check(used.turns == 1, "%s: and give it the new turn: %r" % (how, used.turns))
|
|
1206
|
+
check(len(failures(down)) == 1,
|
|
1207
|
+
"%s: recovering must not name the turn again: %r" % (how, down.notices))
|
|
1208
|
+
|
|
1209
|
+
for how in ("handler", "tool-result", "drop"):
|
|
1210
|
+
asyncio.run(one_case(how))
|
|
1211
|
+
|
|
1212
|
+
# A stream that simply ends is the end of a session, not a failed turn. It is
|
|
1213
|
+
# still said out loud, once, and it says what it is: the client is waiting on a
|
|
1214
|
+
# turn that is not coming, and only a notice releases it, but calling it a failure
|
|
1215
|
+
# would tell the captain something broke when the model merely finished.
|
|
1216
|
+
async def clean_end():
|
|
1217
|
+
down = Down()
|
|
1218
|
+
session = relay.Session(options, down, None)
|
|
1219
|
+
session.stream = Stream([], clean=True)
|
|
1220
|
+
session.reader_task = asyncio.create_task(session._read_model())
|
|
1221
|
+
await asyncio.wait_for(session.ended.wait(), timeout=5)
|
|
1222
|
+
return down, session
|
|
1223
|
+
|
|
1224
|
+
down, ended = asyncio.run(clean_end())
|
|
1225
|
+
check(ended.turn_done.is_set(), "a clean end must release a waiting turn too")
|
|
1226
|
+
check(not ended.failed, "a clean end of stream is not a turn failure")
|
|
1227
|
+
check(not failures(down),
|
|
1228
|
+
"and no failure should be named to the captain: %r" % (down.notices,))
|
|
1229
|
+
check([n.get("event") for n in down.notices] == ["session-ended"],
|
|
1230
|
+
"a clean end must be announced once, as the end it is: %r" % (down.notices,))
|
|
1231
|
+
|
|
1232
|
+
# Nor is a stream that went away because close() asked it to. renew closes the
|
|
1233
|
+
# old session on every single turn, so announcing that would put a failure notice
|
|
1234
|
+
# in front of the captain on every ordinary turn.
|
|
1235
|
+
async def torn_down():
|
|
1236
|
+
down = Down()
|
|
1237
|
+
session = relay.Session(options, down, None)
|
|
1238
|
+
gone = asyncio.Event()
|
|
1239
|
+
|
|
1240
|
+
class Closer:
|
|
1241
|
+
async def send(self, chunk):
|
|
1242
|
+
pass
|
|
1243
|
+
|
|
1244
|
+
async def close(self):
|
|
1245
|
+
# The stream goes away exactly when close() closes the input half,
|
|
1246
|
+
# which is the ordering every renewed turn goes through.
|
|
1247
|
+
gone.set()
|
|
1248
|
+
|
|
1249
|
+
class Blocking:
|
|
1250
|
+
def __init__(self):
|
|
1251
|
+
self.input_stream = Closer()
|
|
1252
|
+
|
|
1253
|
+
async def await_output(self):
|
|
1254
|
+
await gone.wait()
|
|
1255
|
+
raise RuntimeError("the model stream is gone")
|
|
1256
|
+
|
|
1257
|
+
async def quiet(obj):
|
|
1258
|
+
pass
|
|
1259
|
+
|
|
1260
|
+
session.stream = Blocking()
|
|
1261
|
+
# The real one builds an SDK event, and the SDK is deliberately not installed
|
|
1262
|
+
# here; what this case needs is close() getting as far as the input half.
|
|
1263
|
+
session._send = quiet
|
|
1264
|
+
session.reader_task = asyncio.create_task(session._read_model())
|
|
1265
|
+
await asyncio.sleep(0)
|
|
1266
|
+
await session.close()
|
|
1267
|
+
await asyncio.wait_for(session.ended.wait(), timeout=5)
|
|
1268
|
+
return down, session
|
|
1269
|
+
|
|
1270
|
+
down, closed = asyncio.run(torn_down())
|
|
1271
|
+
check(closed.ended.is_set(), "the reader must still report the session over")
|
|
1272
|
+
check(not closed.failed, "a deliberate close is not a turn failure")
|
|
1273
|
+
check(not down.notices,
|
|
1274
|
+
"and an ordinary renew must say nothing at all: %r" % (down.notices,))
|
|
1275
|
+
PY
|
|
1276
|
+
pass "a failure inside the model reader costs one turn, not every later one"
|
|
1277
|
+
pass "a reader failure is named to the captain, a clean end and a close are not"
|
|
1278
|
+
|
|
1279
|
+
# --- the laptop end ---------------------------------------------------------
|
|
1280
|
+
#
|
|
1281
|
+
# The microphone and speaker DEVICES cannot be opened on a host with neither, so
|
|
1282
|
+
# the first live run is their test and nothing below touches audio hardware. What
|
|
1283
|
+
# IS testable is everything around them, and these are the pieces whose failure is
|
|
1284
|
+
# hardest to read from the symptom. A missing -T corrupts audio rather than
|
|
1285
|
+
# erroring, and a banner-printing login shell desynchronises the stream in a way
|
|
1286
|
+
# that looks like a protocol bug and is not. The speaker's byte accounting is
|
|
1287
|
+
# testable as well, being arithmetic rather than device work, and is covered
|
|
1288
|
+
# further down this block against a stub stream with the callback driven by hand.
|
|
1289
|
+
|
|
1290
|
+
mkdir -p "$TMP_ROOT/client-files"
|
|
1291
|
+
printf '\0\0\0\0' > "$TMP_ROOT/client-files/clip.pcm"
|
|
1292
|
+
|
|
1293
|
+
python3 - "$ROOT/bin" "$TMP_ROOT/client-files" <<'PY' || fail "laptop client"
|
|
1294
|
+
import io, os, sys, types
|
|
1295
|
+
sys.path.insert(0, sys.argv[1])
|
|
1296
|
+
import importlib.util, pathlib
|
|
1297
|
+
spec = importlib.util.spec_from_file_location(
|
|
1298
|
+
"client", str(pathlib.Path(sys.argv[1]) / "fm-voice-client.py"))
|
|
1299
|
+
client = importlib.util.module_from_spec(spec)
|
|
1300
|
+
spec.loader.exec_module(client)
|
|
1301
|
+
import fm_voice_frame as frame
|
|
1302
|
+
|
|
1303
|
+
TMP = sys.argv[2]
|
|
1304
|
+
|
|
1305
|
+
def check(cond, label):
|
|
1306
|
+
if not cond:
|
|
1307
|
+
sys.exit("client: " + label)
|
|
1308
|
+
|
|
1309
|
+
# Where the relay lives on the desktop is one operator's directory layout, so the
|
|
1310
|
+
# client carries no default for it and says so rather than trying a path that
|
|
1311
|
+
# belongs to somebody else. The refusal is checked before the variable below is
|
|
1312
|
+
# set, because after that every other case supplies it.
|
|
1313
|
+
os.environ.pop("FM_VOICE_RELAY", None)
|
|
1314
|
+
try:
|
|
1315
|
+
client.parse_args(["--host", "desk"])
|
|
1316
|
+
sys.exit("client: started with no relay path at all")
|
|
1317
|
+
except SystemExit as exc:
|
|
1318
|
+
check(exc.code != 0, "a missing relay path must be a refusal, not a default")
|
|
1319
|
+
|
|
1320
|
+
os.environ["FM_VOICE_RELAY"] = "/desktop/firstmate/bin/fm-voice-relay.py"
|
|
1321
|
+
check(client.parse_args(["--host", "desk"]).relay
|
|
1322
|
+
== "/desktop/firstmate/bin/fm-voice-relay.py",
|
|
1323
|
+
"FM_VOICE_RELAY should supply the relay path for a whole shell")
|
|
1324
|
+
check(client.parse_args(["--host", "desk", "--relay", "/other/relay.py"]).relay
|
|
1325
|
+
== "/other/relay.py", "an explicit --relay must win over the variable")
|
|
1326
|
+
|
|
1327
|
+
# Push to talk is the default for this build, and the only mode that runs.
|
|
1328
|
+
check(client.parse_args(["--host", "h"]).listen == client.PUSH_TO_TALK,
|
|
1329
|
+
"push to talk must be the default")
|
|
1330
|
+
|
|
1331
|
+
# An open microphone needs to know when the captain stopped speaking, and this
|
|
1332
|
+
# client cannot: it would open a turn and stream forever without ever marking a
|
|
1333
|
+
# boundary. So the setting is accepted as a value and refuses at parse time,
|
|
1334
|
+
# before any ssh connection is opened or any model session is paid for. The value
|
|
1335
|
+
# stays in the accepted set so switching it on later is a small change.
|
|
1336
|
+
check(client.OPEN_MIC in client.LISTEN_MODES,
|
|
1337
|
+
"open mic must stay a value the flag accepts")
|
|
1338
|
+
refusal = None
|
|
1339
|
+
try:
|
|
1340
|
+
client.parse_args(["--host", "h", "--listen", "open-mic"])
|
|
1341
|
+
except SystemExit as exc:
|
|
1342
|
+
refusal = exc.code
|
|
1343
|
+
check(refusal not in (None, 0),
|
|
1344
|
+
"open mic must refuse rather than start: %r" % (refusal,))
|
|
1345
|
+
|
|
1346
|
+
# The audio devices themselves cannot be reached from here, but their SELECTOR
|
|
1347
|
+
# can be, and it is typed: sounddevice reads an int as an index into its device
|
|
1348
|
+
# list and a str as a name to match, so an index left as text is looked up as a
|
|
1349
|
+
# device literally called "3" and raises on the captain's first live run.
|
|
1350
|
+
picked = client.parse_args(["--host", "h", "--input-device", "3",
|
|
1351
|
+
"--output-device", "External Headphones"])
|
|
1352
|
+
check(picked.input_device == 3 and not isinstance(picked.input_device, str),
|
|
1353
|
+
"a numeric device must arrive as an index: %r" % picked.input_device)
|
|
1354
|
+
check(picked.output_device == "External Headphones",
|
|
1355
|
+
"a named device must stay a name: %r" % picked.output_device)
|
|
1356
|
+
check(client.parse_args(
|
|
1357
|
+
["--host", "h", "--input-device", "2 - Built-in Microphone"]
|
|
1358
|
+
).input_device == "2 - Built-in Microphone",
|
|
1359
|
+
"a device name that begins with a digit must stay a name")
|
|
1360
|
+
check(client.parse_args(["--host", "h"]).input_device is None,
|
|
1361
|
+
"no device flag must stay unset, so sounddevice picks the default")
|
|
1362
|
+
|
|
1363
|
+
# Over SSH: no pty, or the audio stream is silently rewritten.
|
|
1364
|
+
argv = client.relay_command(client.parse_args(["--host", "desk"]))
|
|
1365
|
+
check(argv[:3] == ["ssh", "-T", "desk"], "ssh must be invoked with -T: %s" % argv)
|
|
1366
|
+
check("--serve" in argv, "the relay must be started in serve mode")
|
|
1367
|
+
|
|
1368
|
+
# Locally: no ssh at all, so the same client can be measured on this host.
|
|
1369
|
+
argv = client.relay_command(client.parse_args(["--local"]))
|
|
1370
|
+
check(argv[0] != "ssh", "--local must not invoke ssh: %s" % argv)
|
|
1371
|
+
|
|
1372
|
+
# The interpreter is a setting because the relay needs a virtual environment the
|
|
1373
|
+
# system interpreter does not have.
|
|
1374
|
+
argv = client.relay_command(client.parse_args(
|
|
1375
|
+
["--host", "desk", "--relay-python", "/opt/venv/bin/python",
|
|
1376
|
+
"--relay-arg=--scope", "--relay-arg=counts"]))
|
|
1377
|
+
check("/opt/venv/bin/python" in argv, "the relay interpreter must be passed: %s" % argv)
|
|
1378
|
+
check(argv[-2:] == ["--scope", "counts"],
|
|
1379
|
+
"relay arguments must reach the relay: %s" % argv)
|
|
1380
|
+
|
|
1381
|
+
# A relay that dies after the handshake must be reported at once rather than at
|
|
1382
|
+
# the end of the timeout. Its own one-line error is already on the captain's
|
|
1383
|
+
# terminal, because the relay's stderr is inherited rather than piped, so the only
|
|
1384
|
+
# thing a full timeout adds is thirty seconds of watching nothing. This is the
|
|
1385
|
+
# likely first-run shape: the Bedrock SDK is imported inside the model session, so
|
|
1386
|
+
# a forgotten --relay-python exits the relay after the handshake.
|
|
1387
|
+
import time as clock
|
|
1388
|
+
waiting = client.Client(client.parse_args(["--host", "desk", "--timeout", "5"]))
|
|
1389
|
+
waiting.closed.set()
|
|
1390
|
+
began = clock.monotonic()
|
|
1391
|
+
refused = None
|
|
1392
|
+
try:
|
|
1393
|
+
waiting._wait_ready()
|
|
1394
|
+
except SystemExit as exc:
|
|
1395
|
+
refused = str(exc)
|
|
1396
|
+
check(refused is not None, "a closed relay was treated as ready")
|
|
1397
|
+
check("closed the connection" in refused,
|
|
1398
|
+
"the refusal should name the closed connection: %s" % refused)
|
|
1399
|
+
check("by hand" in refused, "and should give the next step: %s" % refused)
|
|
1400
|
+
took = clock.monotonic() - began
|
|
1401
|
+
check(took < 2, "a closed relay should be reported at once, waited %.1fs" % took)
|
|
1402
|
+
|
|
1403
|
+
# Ready still wins, and a relay that says nothing at all still times out with the
|
|
1404
|
+
# message that fits that case instead.
|
|
1405
|
+
ready = client.Client(client.parse_args(["--host", "desk", "--timeout", "5"]))
|
|
1406
|
+
ready.ready.set()
|
|
1407
|
+
ready._wait_ready()
|
|
1408
|
+
|
|
1409
|
+
silent = client.Client(client.parse_args(["--host", "desk", "--timeout", "0.3"]))
|
|
1410
|
+
timed_out = None
|
|
1411
|
+
try:
|
|
1412
|
+
silent._wait_ready()
|
|
1413
|
+
except SystemExit as exc:
|
|
1414
|
+
timed_out = str(exc)
|
|
1415
|
+
check(timed_out is not None, "a silent relay was treated as ready")
|
|
1416
|
+
check("never reported ready" in timed_out,
|
|
1417
|
+
"a silent relay should time out with its own message: %s" % timed_out)
|
|
1418
|
+
|
|
1419
|
+
# The uplink can die mid-session - the SSH connection drops, or the relay exits -
|
|
1420
|
+
# and the next talk start or talk end is then a write to a dead pipe. Every frame
|
|
1421
|
+
# a turn is made of goes through the one sender thread, so that write has to end
|
|
1422
|
+
# the thread the same quiet way a dead audio write does. Raising instead killed
|
|
1423
|
+
# the thread with a traceback and left the queue unserved, so each remaining run
|
|
1424
|
+
# sat out the full timeout with nothing sending its frames and was reported as an
|
|
1425
|
+
# unanswered turn rather than as the lost connection the downlink had already seen.
|
|
1426
|
+
class DeadPipe:
|
|
1427
|
+
def __init__(self):
|
|
1428
|
+
self.sent = []
|
|
1429
|
+
|
|
1430
|
+
def send(self, kind, payload=b""):
|
|
1431
|
+
self.sent.append(kind)
|
|
1432
|
+
raise BrokenPipeError(32, "Broken pipe")
|
|
1433
|
+
|
|
1434
|
+
for label, opening in (("talk start", client.START), ("talk end", client.END),
|
|
1435
|
+
("audio", b"\x00\x00")):
|
|
1436
|
+
sending = client.Client(client.parse_args(["--host", "desk"]))
|
|
1437
|
+
sending.uplink = DeadPipe()
|
|
1438
|
+
sending.up_q.put(opening)
|
|
1439
|
+
sending.up_q.put(b"\x01\x01")
|
|
1440
|
+
sending.up_q.put(None)
|
|
1441
|
+
try:
|
|
1442
|
+
sending._sender()
|
|
1443
|
+
except BaseException as exc: # noqa: BLE001
|
|
1444
|
+
sys.exit("client: a broken pipe on %s killed the sender thread: %s: %s"
|
|
1445
|
+
% (label, type(exc).__name__, exc))
|
|
1446
|
+
check(sending.uplink.sent and len(sending.uplink.sent) == 1,
|
|
1447
|
+
"the sender should stop at the broken pipe on %s rather than keep "
|
|
1448
|
+
"writing into it: %r" % (label, sending.uplink.sent))
|
|
1449
|
+
if opening is client.END:
|
|
1450
|
+
# Talk end stamps the moment it reached the wire before the write is
|
|
1451
|
+
# attempted, so uplink_drain_s survives a turn the connection cut short.
|
|
1452
|
+
check("wire_end" in sending.turn,
|
|
1453
|
+
"talk end must still record when it reached the wire: %r"
|
|
1454
|
+
% sending.turn)
|
|
1455
|
+
|
|
1456
|
+
# A connection that drops mid-turn does not wait for a frame boundary, so the
|
|
1457
|
+
# downlink meets a header cut in half. That is a transport failure and the turn
|
|
1458
|
+
# record has to say so: a run that only reports answered: false reads in
|
|
1459
|
+
# runs.jsonl exactly like a turn the model declined, and the latency spread
|
|
1460
|
+
# docs/voice-relay.md publishes is computed from that file.
|
|
1461
|
+
import threading as thread_lib
|
|
1462
|
+
|
|
1463
|
+
|
|
1464
|
+
class CutStream:
|
|
1465
|
+
"""A downlink that drops mid-header once the turn is under way.
|
|
1466
|
+
|
|
1467
|
+
Held closed until the client has actually opened the turn, so the cut lands
|
|
1468
|
+
inside the turn being measured rather than before it, which is the sequence
|
|
1469
|
+
a dropped SSH connection produces and the only one whose record matters.
|
|
1470
|
+
"""
|
|
1471
|
+
|
|
1472
|
+
def __init__(self, gate):
|
|
1473
|
+
self._gate = gate
|
|
1474
|
+
self._half = frame.encode(frame.BYE)[:2]
|
|
1475
|
+
self._at = 0
|
|
1476
|
+
|
|
1477
|
+
def read(self, count):
|
|
1478
|
+
check(self._gate.wait(10), "the turn never opened, so nothing was cut")
|
|
1479
|
+
chunk = self._half[self._at:self._at + count]
|
|
1480
|
+
self._at += len(chunk)
|
|
1481
|
+
return chunk
|
|
1482
|
+
|
|
1483
|
+
|
|
1484
|
+
opened = thread_lib.Event()
|
|
1485
|
+
|
|
1486
|
+
|
|
1487
|
+
class GateOpeningUplink:
|
|
1488
|
+
"""Discards the uplink and reports when the turn's first frame went out."""
|
|
1489
|
+
|
|
1490
|
+
def send(self, kind, payload=b""):
|
|
1491
|
+
if kind == frame.TALK_START:
|
|
1492
|
+
opened.set()
|
|
1493
|
+
|
|
1494
|
+
|
|
1495
|
+
cut = client.Client(client.parse_args(
|
|
1496
|
+
["--host", "desk", "--in-file", os.path.join(TMP, "clip.pcm"),
|
|
1497
|
+
"--out-file", os.path.join(TMP, "reply-cut.pcm"), "--timeout", "5"]))
|
|
1498
|
+
cut.reader = frame.Reader(CutStream(opened))
|
|
1499
|
+
cut.uplink = GateOpeningUplink()
|
|
1500
|
+
cut.playback = client.FilePlayback(os.path.join(TMP, "reply-cut.pcm"))
|
|
1501
|
+
cut.capture = client.FileCapture(os.path.join(TMP, "clip.pcm"))
|
|
1502
|
+
cut.capture.start(cut.up_q, cut.talking)
|
|
1503
|
+
thread_lib.Thread(target=cut._sender, daemon=True).start()
|
|
1504
|
+
thread_lib.Thread(target=cut._downlink, daemon=True).start()
|
|
1505
|
+
cut_record = cut.take_turn(1)
|
|
1506
|
+
cut.up_q.put(None)
|
|
1507
|
+
cut.playback.close()
|
|
1508
|
+
|
|
1509
|
+
check(cut.closed.wait(10), "a cut header must end the downlink, not hang it")
|
|
1510
|
+
check(not cut_record["answered"], "a cut connection cannot have answered: %r"
|
|
1511
|
+
% cut_record)
|
|
1512
|
+
check(cut_record["relay_error"],
|
|
1513
|
+
"a dropped connection must be reported in the turn record rather than "
|
|
1514
|
+
"leaving it indistinguishable from a turn nobody answered: %r"
|
|
1515
|
+
% cut_record)
|
|
1516
|
+
check("the connection was lost before this turn was answered"
|
|
1517
|
+
in cut_record["relay_error"],
|
|
1518
|
+
"and it should say the connection went before this turn had its answer, "
|
|
1519
|
+
"which is the half of that fault this case is: %r"
|
|
1520
|
+
% cut_record["relay_error"])
|
|
1521
|
+
|
|
1522
|
+
# The other moment a connection can go is BETWEEN two turns, during the seconds
|
|
1523
|
+
# the client spends letting the previous answer finish. That wait is most of a
|
|
1524
|
+
# multi-run session, so it is where a relay that dies between questions dies.
|
|
1525
|
+
# Opening the next turn anyway cleared the failure the downlink had recorded,
|
|
1526
|
+
# left nothing on the far end to answer it, and produced a run that came back
|
|
1527
|
+
# after the whole reply timeout saying answered: false with relay_error: null.
|
|
1528
|
+
# In runs.jsonl that is indistinguishable from a turn the model declined, and
|
|
1529
|
+
# runs.jsonl is the file docs/voice-relay.md computes its latency spread from, so
|
|
1530
|
+
# the invented turn would be averaged into a published number.
|
|
1531
|
+
import contextlib
|
|
1532
|
+
import json as json_lib
|
|
1533
|
+
|
|
1534
|
+
|
|
1535
|
+
class ClosingStream:
|
|
1536
|
+
"""Serves one whole turn, then closes during the wait after it.
|
|
1537
|
+
|
|
1538
|
+
Both moments are released by the client reaching them rather than by a
|
|
1539
|
+
timer: the frames wait for the turn to open, and the close waits for the
|
|
1540
|
+
client to enter the inter-turn wait. So the sequence under test is the same
|
|
1541
|
+
on a loaded host as on an idle one.
|
|
1542
|
+
"""
|
|
1543
|
+
|
|
1544
|
+
def __init__(self, gate, waiting):
|
|
1545
|
+
self._gate = gate
|
|
1546
|
+
self._waiting = waiting
|
|
1547
|
+
self._reply = (
|
|
1548
|
+
frame.encode(frame.AUDIO, b"\x00\x00" * 1200)
|
|
1549
|
+
+ frame.encode_json(frame.MARK, {"mark": "reply_end",
|
|
1550
|
+
"since_talk_end": 0.4,
|
|
1551
|
+
"tool_calls": 1}))
|
|
1552
|
+
self._at = 0
|
|
1553
|
+
|
|
1554
|
+
def read(self, count):
|
|
1555
|
+
check(self._gate.wait(10), "the turn never opened, so nothing was served")
|
|
1556
|
+
if self._at >= len(self._reply):
|
|
1557
|
+
check(self._waiting.wait(10),
|
|
1558
|
+
"fixture: the client never reached the wait between turns")
|
|
1559
|
+
return b""
|
|
1560
|
+
chunk = self._reply[self._at:self._at + count]
|
|
1561
|
+
self._at += len(chunk)
|
|
1562
|
+
return chunk
|
|
1563
|
+
|
|
1564
|
+
|
|
1565
|
+
class StartGate:
|
|
1566
|
+
"""Discards the uplink and reports when a turn's first frame went out."""
|
|
1567
|
+
|
|
1568
|
+
def __init__(self, gate):
|
|
1569
|
+
self._gate = gate
|
|
1570
|
+
|
|
1571
|
+
def send(self, kind, payload=b""):
|
|
1572
|
+
if kind == frame.TALK_START:
|
|
1573
|
+
self._gate.set()
|
|
1574
|
+
|
|
1575
|
+
|
|
1576
|
+
served, waiting_between = thread_lib.Event(), thread_lib.Event()
|
|
1577
|
+
closing = client.Client(client.parse_args(
|
|
1578
|
+
["--host", "desk", "--in-file", os.path.join(TMP, "clip.pcm"),
|
|
1579
|
+
"--out-file", os.path.join(TMP, "reply-closing.pcm"), "--runs", "2",
|
|
1580
|
+
"--timeout", "2", "--audio-idle", "0.05", "--gap-seconds", "0.05"]))
|
|
1581
|
+
closing.reader = frame.Reader(ClosingStream(served, waiting_between))
|
|
1582
|
+
closing.uplink = StartGate(served)
|
|
1583
|
+
closing.playback = client.FilePlayback(os.path.join(TMP, "reply-closing.pcm"))
|
|
1584
|
+
closing.capture = client.FileCapture(os.path.join(TMP, "clip.pcm"))
|
|
1585
|
+
closing.capture.start(closing.up_q, closing.talking)
|
|
1586
|
+
thread_lib.Thread(target=closing._sender, daemon=True).start()
|
|
1587
|
+
thread_lib.Thread(target=closing._downlink, daemon=True).start()
|
|
1588
|
+
|
|
1589
|
+
# The wait between turns is the seam: the connection goes while the client is
|
|
1590
|
+
# inside it, after the first run was reported and before the second could open.
|
|
1591
|
+
# Waiting for the downlink to see it keeps that ordering exact rather than
|
|
1592
|
+
# leaving it to whichever thread the scheduler runs next.
|
|
1593
|
+
finish_wait = closing._let_reply_finish
|
|
1594
|
+
|
|
1595
|
+
|
|
1596
|
+
def lose_connection_while_waiting(record):
|
|
1597
|
+
waiting_between.set()
|
|
1598
|
+
check(closing.closed.wait(10),
|
|
1599
|
+
"fixture: the connection never closed during the wait between turns")
|
|
1600
|
+
return finish_wait(record)
|
|
1601
|
+
|
|
1602
|
+
|
|
1603
|
+
closing._let_reply_finish = lose_connection_while_waiting
|
|
1604
|
+
emitted, spoken = io.StringIO(), io.StringIO()
|
|
1605
|
+
with contextlib.redirect_stdout(emitted), contextlib.redirect_stderr(spoken):
|
|
1606
|
+
closing_code = closing.run()
|
|
1607
|
+
closing.up_q.put(None)
|
|
1608
|
+
closing.playback.close()
|
|
1609
|
+
|
|
1610
|
+
closing_runs = [json_lib.loads(line) for line in emitted.getvalue().splitlines()
|
|
1611
|
+
if line.strip()]
|
|
1612
|
+
check(len(closing_runs) == 1,
|
|
1613
|
+
"a connection lost between turns must end the session rather than invent a "
|
|
1614
|
+
"turn nobody took: %d run(s) reported, %r" % (len(closing_runs), closing_runs))
|
|
1615
|
+
check(closing_runs[0]["answered"] and closing_runs[0]["relay_error"] is None,
|
|
1616
|
+
"fixture is wrong: the turn before the connection went should be a good "
|
|
1617
|
+
"one, so the case cannot pass on a run that failed anyway: %r"
|
|
1618
|
+
% closing_runs[0])
|
|
1619
|
+
# Two runs were asked for and one was taken, so the exit code has to be the
|
|
1620
|
+
# unhappy one; a session that stops early while reporting success is a
|
|
1621
|
+
# measurement someone reads as complete.
|
|
1622
|
+
check(closing_code != 0,
|
|
1623
|
+
"a session that took 1 of 2 runs must not exit 0, got %r" % closing_code)
|
|
1624
|
+
check("the connection ended before run 2 of 2" in spoken.getvalue(),
|
|
1625
|
+
"and the captain should be told why it stopped, in the words the path that "
|
|
1626
|
+
"stopped it recorded: %r" % spoken.getvalue())
|
|
1627
|
+
|
|
1628
|
+
# The case above is the connection going while the run loop is watching for it.
|
|
1629
|
+
# The loop cannot only be watching, though: the downlink names the failure and
|
|
1630
|
+
# marks the connection closed, and a loop that decides by reading the mark alone
|
|
1631
|
+
# is blind for as long as those are two separate writes - the connection is gone,
|
|
1632
|
+
# the failure is on the record, and the check has already passed. So the decision
|
|
1633
|
+
# belongs where the turn is opened, under the lock both writes are made under.
|
|
1634
|
+
# With the connection already gone, opening the turn anyway erased the failure the
|
|
1635
|
+
# downlink had recorded, pushed talk-start into a dead pipe, and printed a run
|
|
1636
|
+
# that waited out the whole reply timeout as answered: false with relay_error:
|
|
1637
|
+
# null - the invented turn this whole seam exists to keep out of runs.jsonl.
|
|
1638
|
+
class LostStream:
|
|
1639
|
+
"""A downlink that is already gone the first time it is read."""
|
|
1640
|
+
|
|
1641
|
+
def read(self, count):
|
|
1642
|
+
raise OSError(104, "Connection reset by peer")
|
|
1643
|
+
|
|
1644
|
+
|
|
1645
|
+
class CountingUplink:
|
|
1646
|
+
"""Accepts frames and remembers which kinds were pushed at it."""
|
|
1647
|
+
|
|
1648
|
+
def __init__(self):
|
|
1649
|
+
self.sent = []
|
|
1650
|
+
|
|
1651
|
+
def send(self, kind, payload=b""):
|
|
1652
|
+
self.sent.append(kind)
|
|
1653
|
+
|
|
1654
|
+
|
|
1655
|
+
gone = client.Client(client.parse_args(
|
|
1656
|
+
["--host", "desk", "--in-file", os.path.join(TMP, "clip.pcm"),
|
|
1657
|
+
"--out-file", os.path.join(TMP, "reply-gone.pcm"), "--runs", "2",
|
|
1658
|
+
"--timeout", "2", "--audio-idle", "0.05", "--gap-seconds", "0.05"]))
|
|
1659
|
+
gone.reader = frame.Reader(LostStream())
|
|
1660
|
+
gone.uplink = CountingUplink()
|
|
1661
|
+
gone.playback = client.FilePlayback(os.path.join(TMP, "reply-gone.pcm"))
|
|
1662
|
+
gone.capture = client.FileCapture(os.path.join(TMP, "clip.pcm"))
|
|
1663
|
+
gone.capture.start(gone.up_q, gone.talking)
|
|
1664
|
+
thread_lib.Thread(target=gone._sender, daemon=True).start()
|
|
1665
|
+
# Run to completion in this thread rather than in a started one: the connection
|
|
1666
|
+
# is gone before the first read returns, so the whole downlink is over by the
|
|
1667
|
+
# time run() begins and there is no ordering left for the scheduler to decide.
|
|
1668
|
+
gone._downlink()
|
|
1669
|
+
check(gone.closed.is_set(),
|
|
1670
|
+
"fixture: a connection lost on the first read must leave the client closed")
|
|
1671
|
+
|
|
1672
|
+
gone_out, gone_said = io.StringIO(), io.StringIO()
|
|
1673
|
+
with contextlib.redirect_stdout(gone_out), contextlib.redirect_stderr(gone_said):
|
|
1674
|
+
gone_code = gone.run()
|
|
1675
|
+
gone.up_q.put(None)
|
|
1676
|
+
gone.playback.close()
|
|
1677
|
+
|
|
1678
|
+
gone_runs = [json_lib.loads(line) for line in gone_out.getvalue().splitlines()
|
|
1679
|
+
if line.strip()]
|
|
1680
|
+
check(gone_runs == [],
|
|
1681
|
+
"a turn must not be opened on a connection already known gone, and no run "
|
|
1682
|
+
"reported for one that never opened: %r" % gone_runs)
|
|
1683
|
+
check(gone.uplink.sent == [],
|
|
1684
|
+
"and nothing should be pushed into the dead pipe: %r" % gone.uplink.sent)
|
|
1685
|
+
check(gone_code != 0,
|
|
1686
|
+
"a session that took none of its 2 runs must not exit 0, got %r" % gone_code)
|
|
1687
|
+
check("the connection was lost before run 1 of 2" in gone_said.getvalue(),
|
|
1688
|
+
"and the captain should be told why nothing was taken, naming the run it "
|
|
1689
|
+
"stopped at and the cause the read that failed recorded: %r"
|
|
1690
|
+
% gone_said.getvalue())
|
|
1691
|
+
|
|
1692
|
+
# The third moment is a connection that goes DURING a turn that answered anyway,
|
|
1693
|
+
# with runs still to take. The answer is real and its record stands, so nothing
|
|
1694
|
+
# here is a turn failure; what must not happen is the session ending quietly on a
|
|
1695
|
+
# happy exit code, because two of the three runs asked for are missing and a
|
|
1696
|
+
# runs.jsonl short of its runs, reported as success, is read later as the whole
|
|
1697
|
+
# measurement. Refusing the next turn is the one place a closed connection stops
|
|
1698
|
+
# a session, so it reports the same way wherever the connection went.
|
|
1699
|
+
class EndingStream:
|
|
1700
|
+
"""Serves one whole turn, then reports end of file without waiting."""
|
|
1701
|
+
|
|
1702
|
+
def __init__(self, gate):
|
|
1703
|
+
self._gate = gate
|
|
1704
|
+
self._reply = (
|
|
1705
|
+
frame.encode(frame.AUDIO, b"\x00\x00" * 1200)
|
|
1706
|
+
+ frame.encode_json(frame.MARK, {"mark": "reply_end",
|
|
1707
|
+
"since_talk_end": 0.4,
|
|
1708
|
+
"tool_calls": 0}))
|
|
1709
|
+
self._at = 0
|
|
1710
|
+
|
|
1711
|
+
def read(self, count):
|
|
1712
|
+
check(self._gate.wait(10), "the turn never opened, so nothing was served")
|
|
1713
|
+
chunk = self._reply[self._at:self._at + count]
|
|
1714
|
+
self._at += len(chunk)
|
|
1715
|
+
return chunk
|
|
1716
|
+
|
|
1717
|
+
|
|
1718
|
+
ending_served = thread_lib.Event()
|
|
1719
|
+
ending = client.Client(client.parse_args(
|
|
1720
|
+
["--host", "desk", "--in-file", os.path.join(TMP, "clip.pcm"),
|
|
1721
|
+
"--out-file", os.path.join(TMP, "reply-ending.pcm"), "--runs", "3",
|
|
1722
|
+
"--timeout", "2", "--audio-idle", "0.05", "--gap-seconds", "0.05"]))
|
|
1723
|
+
ending.reader = frame.Reader(EndingStream(ending_served))
|
|
1724
|
+
ending.uplink = StartGate(ending_served)
|
|
1725
|
+
ending.playback = client.FilePlayback(os.path.join(TMP, "reply-ending.pcm"))
|
|
1726
|
+
ending.capture = client.FileCapture(os.path.join(TMP, "clip.pcm"))
|
|
1727
|
+
ending.capture.start(ending.up_q, ending.talking)
|
|
1728
|
+
thread_lib.Thread(target=ending._sender, daemon=True).start()
|
|
1729
|
+
thread_lib.Thread(target=ending._downlink, daemon=True).start()
|
|
1730
|
+
|
|
1731
|
+
# The end of file lands inside the first turn, so it is already seen by the time
|
|
1732
|
+
# the wait after that turn begins. Confirming it here rather than trusting the
|
|
1733
|
+
# timing keeps the sequence exact on a loaded host as well as an idle one.
|
|
1734
|
+
finish_after_end = ending._let_reply_finish
|
|
1735
|
+
|
|
1736
|
+
|
|
1737
|
+
def confirm_ended_then_wait(record):
|
|
1738
|
+
check(ending.closed.wait(10),
|
|
1739
|
+
"fixture: the end of file never reached the client during the turn")
|
|
1740
|
+
return finish_after_end(record)
|
|
1741
|
+
|
|
1742
|
+
|
|
1743
|
+
ending._let_reply_finish = confirm_ended_then_wait
|
|
1744
|
+
ending_out, ending_said = io.StringIO(), io.StringIO()
|
|
1745
|
+
with contextlib.redirect_stdout(ending_out), contextlib.redirect_stderr(ending_said):
|
|
1746
|
+
ending_code = ending.run()
|
|
1747
|
+
ending.up_q.put(None)
|
|
1748
|
+
ending.playback.close()
|
|
1749
|
+
|
|
1750
|
+
ending_runs = [json_lib.loads(line) for line in ending_out.getvalue().splitlines()
|
|
1751
|
+
if line.strip()]
|
|
1752
|
+
check(len(ending_runs) == 1,
|
|
1753
|
+
"one turn was served, so exactly one run belongs in the file: %d, %r"
|
|
1754
|
+
% (len(ending_runs), ending_runs))
|
|
1755
|
+
check(ending_runs[0]["answered"] and ending_runs[0]["relay_error"] is None,
|
|
1756
|
+
"an answered turn whose connection then ended cleanly is not a turn "
|
|
1757
|
+
"failure, and its record stands: %r" % ending_runs[0])
|
|
1758
|
+
check(ending_code != 0,
|
|
1759
|
+
"but a session that took 1 of 3 runs must not exit 0, got %r" % ending_code)
|
|
1760
|
+
check("run 2 of 3" in ending_said.getvalue(),
|
|
1761
|
+
"and it should name the run it stopped at: %r" % ending_said.getvalue())
|
|
1762
|
+
|
|
1763
|
+
# The case above is the connection ending AFTER a turn was answered, which is the
|
|
1764
|
+
# negative case: nothing broke inside a turn, so nothing is named. The three
|
|
1765
|
+
# below are the same three endings landing INSIDE a turn, while it is still
|
|
1766
|
+
# waiting for its reply, and there each of them has to name itself.
|
|
1767
|
+
#
|
|
1768
|
+
# None of them raises. frame.Writer sends whole frames and the relay spends
|
|
1769
|
+
# almost all of a turn awaiting the model, so a relay killed mid-turn - SIGKILL,
|
|
1770
|
+
# the host going, the SSH connection dropping between frames - ends its stdout on
|
|
1771
|
+
# a frame boundary. Reader then reads nothing at all where a header should start
|
|
1772
|
+
# and reports end of input by design. A goodbye is the same shape by another
|
|
1773
|
+
# route: the relay sends one from its own teardown after a fault. Released with
|
|
1774
|
+
# no reason recorded, both come back as answered: false with relay_error: null,
|
|
1775
|
+
# which in runs.jsonl is the shape of a turn the model declined, and runs.jsonl
|
|
1776
|
+
# is the file docs/voice-relay.md computes its published latency spread from. An
|
|
1777
|
+
# infrastructure failure averaged into that number is the whole thing being kept
|
|
1778
|
+
# out of it.
|
|
1779
|
+
class ScriptedStream:
|
|
1780
|
+
"""Serves prepared downlink bytes, held until the turn under test has opened.
|
|
1781
|
+
|
|
1782
|
+
One class for every case below, because each of them differs only in the bytes
|
|
1783
|
+
it serves: the gate and the slicing are the same everywhere, and an empty
|
|
1784
|
+
script is what an end of stream at a frame boundary looks like. The label is
|
|
1785
|
+
per case so a fixture that never fires still names which case it belonged to.
|
|
1786
|
+
|
|
1787
|
+
What happens once the script is spent is the one thing a case may choose, and
|
|
1788
|
+
the default is the clean end of stream every case but one wants. at_end takes
|
|
1789
|
+
a callable for the case that needs the other shape a dropped link has, a reset
|
|
1790
|
+
rather than a close, which reaches the client as a raised error instead of an
|
|
1791
|
+
empty read.
|
|
1792
|
+
"""
|
|
1793
|
+
|
|
1794
|
+
def __init__(self, gate, payload, label, at_end=None):
|
|
1795
|
+
self._gate = gate
|
|
1796
|
+
self._bytes = payload
|
|
1797
|
+
self._label = label
|
|
1798
|
+
self._at_end = at_end
|
|
1799
|
+
self._at = 0
|
|
1800
|
+
|
|
1801
|
+
def read(self, count):
|
|
1802
|
+
check(self._gate.wait(10),
|
|
1803
|
+
"the turn never opened, so %s was never served" % self._label)
|
|
1804
|
+
if self._at >= len(self._bytes) and self._at_end is not None:
|
|
1805
|
+
self._at_end()
|
|
1806
|
+
chunk = self._bytes[self._at:self._at + count]
|
|
1807
|
+
self._at += len(chunk)
|
|
1808
|
+
return chunk
|
|
1809
|
+
|
|
1810
|
+
|
|
1811
|
+
RELAY_REASON = "FrameError: unknown frame kind: b'\\xff'"
|
|
1812
|
+
REPLY_END = frame.encode_json(frame.MARK, {"mark": "reply_end",
|
|
1813
|
+
"since_talk_end": 0.4,
|
|
1814
|
+
"tool_calls": 0})
|
|
1815
|
+
|
|
1816
|
+
|
|
1817
|
+
def wire_client(args, reader, uplink, playback=None, start=True):
|
|
1818
|
+
"""Build a client wired to stand-ins for both its ends, threads running.
|
|
1819
|
+
|
|
1820
|
+
The same seven lines were written out at every case below. playback defaults to
|
|
1821
|
+
a file at whatever --out-file the arguments name, which is what most of them
|
|
1822
|
+
want; a case needing a playback of its own passes one. start=False is for the
|
|
1823
|
+
one case that has to arrange something before the threads may read.
|
|
1824
|
+
"""
|
|
1825
|
+
wired = client.Client(client.parse_args(args))
|
|
1826
|
+
wired.reader = frame.Reader(reader)
|
|
1827
|
+
wired.uplink = uplink
|
|
1828
|
+
wired.playback = (client.FilePlayback(wired.options.out_file)
|
|
1829
|
+
if playback is None else playback)
|
|
1830
|
+
wired.capture = client.FileCapture(os.path.join(TMP, "clip.pcm"))
|
|
1831
|
+
wired.capture.start(wired.up_q, wired.talking)
|
|
1832
|
+
if start:
|
|
1833
|
+
thread_lib.Thread(target=wired._sender, daemon=True).start()
|
|
1834
|
+
thread_lib.Thread(target=wired._downlink, daemon=True).start()
|
|
1835
|
+
return wired
|
|
1836
|
+
|
|
1837
|
+
|
|
1838
|
+
def turn_lost(label, make_stream, out_name, runs="1"):
|
|
1839
|
+
"""Take one turn against a downlink that ends during it, and return run and stderr."""
|
|
1840
|
+
gate = thread_lib.Event()
|
|
1841
|
+
lost = wire_client(
|
|
1842
|
+
["--host", "desk", "--in-file", os.path.join(TMP, "clip.pcm"),
|
|
1843
|
+
"--out-file", os.path.join(TMP, out_name), "--runs", runs,
|
|
1844
|
+
"--timeout", "2", "--audio-idle", "0.05"],
|
|
1845
|
+
make_stream(gate), StartGate(gate))
|
|
1846
|
+
out, said = io.StringIO(), io.StringIO()
|
|
1847
|
+
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(said):
|
|
1848
|
+
code = lost.run()
|
|
1849
|
+
lost.up_q.put(None)
|
|
1850
|
+
lost.playback.close()
|
|
1851
|
+
runs = [json_lib.loads(line) for line in out.getvalue().splitlines()
|
|
1852
|
+
if line.strip()]
|
|
1853
|
+
# The record is still emitted: the turn was taken and it really did go
|
|
1854
|
+
# unanswered, so dropping it would hide the failure instead of naming it.
|
|
1855
|
+
check(len(runs) == 1,
|
|
1856
|
+
"%s: the turn was taken, so its run belongs in the file: %d, %r"
|
|
1857
|
+
% (label, len(runs), runs))
|
|
1858
|
+
check(not runs[0]["answered"],
|
|
1859
|
+
"%s: fixture is wrong, the turn was answered after all: %r"
|
|
1860
|
+
% (label, runs[0]))
|
|
1861
|
+
check(code != 0, "%s: an unanswered turn must not exit 0, got %r"
|
|
1862
|
+
% (label, code))
|
|
1863
|
+
check(runs[0]["relay_error"],
|
|
1864
|
+
"%s: a turn lost to the connection must say so rather than reading in "
|
|
1865
|
+
"runs.jsonl exactly like a turn the model declined: %r"
|
|
1866
|
+
% (label, runs[0]))
|
|
1867
|
+
# With one run asked for, the run loop never reaches its "and the rest were
|
|
1868
|
+
# not taken" line. A record on stdout and silence on stderr is a captain who
|
|
1869
|
+
# spoke, heard nothing back, and was told nothing either; the module docstring
|
|
1870
|
+
# puts everything human on stderr.
|
|
1871
|
+
check("client:" in said.getvalue(),
|
|
1872
|
+
"%s: the captain must be told the connection went, not only the file: "
|
|
1873
|
+
"%r" % (label, said.getvalue()))
|
|
1874
|
+
return runs[0], said.getvalue()
|
|
1875
|
+
|
|
1876
|
+
|
|
1877
|
+
eof_run, eof_said = turn_lost(
|
|
1878
|
+
"end of stream",
|
|
1879
|
+
lambda gate: ScriptedStream(gate, b"", "the end of stream"),
|
|
1880
|
+
"reply-eof.pcm")
|
|
1881
|
+
check("ended" in eof_run["relay_error"],
|
|
1882
|
+
"an end of stream should say the connection ended: %r"
|
|
1883
|
+
% eof_run["relay_error"])
|
|
1884
|
+
check("the connection ended" in eof_said,
|
|
1885
|
+
"and should say so on stderr as well: %r" % eof_said)
|
|
1886
|
+
|
|
1887
|
+
# A goodbye nobody asked for. This end sent no quit, so the relay stopping is the
|
|
1888
|
+
# relay's own decision, and mid-turn it costs the captain their question.
|
|
1889
|
+
bye_run, bye_said = turn_lost(
|
|
1890
|
+
"goodbye",
|
|
1891
|
+
lambda gate: ScriptedStream(gate, frame.encode(frame.BYE), "the goodbye"),
|
|
1892
|
+
"reply-bye.pcm")
|
|
1893
|
+
check("the relay stopped" in bye_run["relay_error"],
|
|
1894
|
+
"a goodbye nobody asked for mid-turn should say the relay stopped, because "
|
|
1895
|
+
"that is the relay deciding rather than the connection dying: %r"
|
|
1896
|
+
% bye_run["relay_error"])
|
|
1897
|
+
# Distinct from the line above, because they are distinct faults: a stream that
|
|
1898
|
+
# stopped, versus a relay that chose to stop. One line for both would send the
|
|
1899
|
+
# captain looking for the wrong thing.
|
|
1900
|
+
check("stopped without being asked" in bye_said
|
|
1901
|
+
and "the connection ended" not in bye_said,
|
|
1902
|
+
"a goodbye should name itself on stderr rather than borrowing the wording "
|
|
1903
|
+
"of an ended stream: %r" % bye_said)
|
|
1904
|
+
|
|
1905
|
+
# And when the relay does name the fault, its words are what the record carries.
|
|
1906
|
+
# This end can only infer that a goodbye arrived; the relay knows what happened,
|
|
1907
|
+
# so a reason it sent is never replaced by one inferred here.
|
|
1908
|
+
named_run, _ = turn_lost(
|
|
1909
|
+
"named fault",
|
|
1910
|
+
lambda gate: ScriptedStream(
|
|
1911
|
+
gate,
|
|
1912
|
+
frame.encode_json(frame.NOTICE,
|
|
1913
|
+
{"event": "turn-failed", "error": RELAY_REASON})
|
|
1914
|
+
+ frame.encode(frame.BYE),
|
|
1915
|
+
"the named fault"),
|
|
1916
|
+
"reply-named.pcm")
|
|
1917
|
+
check(named_run["relay_error"] == RELAY_REASON,
|
|
1918
|
+
"the relay's own reason must reach the record unchanged rather than being "
|
|
1919
|
+
"overwritten by the goodbye behind it: %r" % named_run["relay_error"])
|
|
1920
|
+
|
|
1921
|
+
# THE MIDDLE OF THE SCENARIO, which the cases above and below both miss. A relay
|
|
1922
|
+
# killed mid-turn ends its stdout on a frame boundary, and it spends nearly the
|
|
1923
|
+
# whole turn streaming a reply, so dying AFTER some of that reply has played is
|
|
1924
|
+
# the likelier half of the very fault this work exists to name. The cases above
|
|
1925
|
+
# cover dying before any of it, and the answered-turn case below covers a reply
|
|
1926
|
+
# that finished, so nothing pinned the middle.
|
|
1927
|
+
#
|
|
1928
|
+
# In the middle both halves of the record are true at once and must say so: sound
|
|
1929
|
+
# reached the captain, so the turn was answered and first_audio_s measures when,
|
|
1930
|
+
# AND the answer stopped partway, so the reason has to say the reply did not
|
|
1931
|
+
# finish rather than that the turn was never answered. A record asserting both
|
|
1932
|
+
# "answered" and "before this turn was answered" sends whoever reads it to the
|
|
1933
|
+
# wrong end. At the default of one run this also used to clear every path to a
|
|
1934
|
+
# non-zero exit code and report the session a success.
|
|
1935
|
+
SOME_REPLY = frame.encode(frame.AUDIO, b"\x00\x00" * 1200)
|
|
1936
|
+
SESSION_ENDED = frame.encode_json(frame.NOTICE, {"event": "session-ended"})
|
|
1937
|
+
|
|
1938
|
+
|
|
1939
|
+
def turn_cut(label, payload, out_name, at_end=None):
|
|
1940
|
+
"""Take one turn, at the default run count, against a scripted downlink."""
|
|
1941
|
+
gate = thread_lib.Event()
|
|
1942
|
+
cut = wire_client(
|
|
1943
|
+
["--host", "desk", "--in-file", os.path.join(TMP, "clip.pcm"),
|
|
1944
|
+
"--out-file", os.path.join(TMP, out_name),
|
|
1945
|
+
"--timeout", "2", "--audio-idle", "0.05"],
|
|
1946
|
+
ScriptedStream(gate, payload, label, at_end), StartGate(gate))
|
|
1947
|
+
out, said = io.StringIO(), io.StringIO()
|
|
1948
|
+
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(said):
|
|
1949
|
+
code = cut.run()
|
|
1950
|
+
cut.up_q.put(None)
|
|
1951
|
+
cut.playback.close()
|
|
1952
|
+
records = [json_lib.loads(line) for line in out.getvalue().splitlines()
|
|
1953
|
+
if line.strip()]
|
|
1954
|
+
check(len(records) == 1,
|
|
1955
|
+
"%s: one turn was taken, so one record belongs in the file: %r"
|
|
1956
|
+
% (label, records))
|
|
1957
|
+
check(cut.options.runs == 1,
|
|
1958
|
+
"%s: fixture is wrong, this case is about the default run count: %r"
|
|
1959
|
+
% (label, cut.options.runs))
|
|
1960
|
+
return records[0], said.getvalue(), code
|
|
1961
|
+
|
|
1962
|
+
|
|
1963
|
+
for label, payload, subject in (
|
|
1964
|
+
("end of stream", b"", "the connection ended"),
|
|
1965
|
+
("relay stopped", frame.encode(frame.BYE), "the relay stopped"),
|
|
1966
|
+
("session ended", SESSION_ENDED, "the relay ended the session")):
|
|
1967
|
+
slug = label.replace(" ", "-")
|
|
1968
|
+
# No audio at all. This half keeps the wording it already had, so the two can
|
|
1969
|
+
# never be collapsed back into one sentence that fits neither.
|
|
1970
|
+
silent, _, silent_code = turn_cut(
|
|
1971
|
+
"%s, nothing played" % label, payload, "reply-silent-%s.pcm" % slug)
|
|
1972
|
+
check(not silent["answered"] and silent["reply_audio_seconds"] == 0,
|
|
1973
|
+
"%s: fixture is wrong, this half is the one where nothing played: %r"
|
|
1974
|
+
% (label, silent))
|
|
1975
|
+
check(silent["relay_error"] == "%s before this turn was answered" % subject,
|
|
1976
|
+
"%s: a turn that got no audio went unanswered and the reason should say "
|
|
1977
|
+
"so: %r" % (label, silent["relay_error"]))
|
|
1978
|
+
check(silent_code != 0,
|
|
1979
|
+
"%s: an unanswered turn must not exit 0, got %r" % (label, silent_code))
|
|
1980
|
+
|
|
1981
|
+
# Some of the reply played, then the same ending. Same fault, different turn,
|
|
1982
|
+
# and the record has to describe the turn it actually got.
|
|
1983
|
+
partial, _, partial_code = turn_cut(
|
|
1984
|
+
"%s, part of a reply played" % label, SOME_REPLY + payload,
|
|
1985
|
+
"reply-partial-%s.pcm" % slug)
|
|
1986
|
+
check(partial["answered"] and partial["reply_audio_seconds"] > 0,
|
|
1987
|
+
"%s: audio reached the captain, so the turn was answered: %r"
|
|
1988
|
+
% (label, partial))
|
|
1989
|
+
check(partial["first_audio_s"] is not None,
|
|
1990
|
+
"%s: and when it reached them is a real measurement, not a null: %r"
|
|
1991
|
+
% (label, partial))
|
|
1992
|
+
check(partial["relay_error"] == "%s before the reply finished" % subject,
|
|
1993
|
+
"%s: a reply cut short must say the reply did not finish: %r"
|
|
1994
|
+
% (label, partial["relay_error"]))
|
|
1995
|
+
check("before this turn was answered" not in partial["relay_error"],
|
|
1996
|
+
"%s: and must not claim nothing arrived, which its own answered field "
|
|
1997
|
+
"contradicts: %r" % (label, partial["relay_error"]))
|
|
1998
|
+
# The record and the exit code have to agree with each other as well.
|
|
1999
|
+
check(partial_code != 0,
|
|
2000
|
+
"%s: a record naming a fault must not exit 0, even at the default of "
|
|
2001
|
+
"one run and even though the turn was answered, got %r"
|
|
2002
|
+
% (label, partial_code))
|
|
2003
|
+
|
|
2004
|
+
# THE FOURTH SUBJECT, and the one a real captain is likeliest to meet. The three
|
|
2005
|
+
# above all reach the client as an orderly end: a close, a goodbye frame, or a
|
|
2006
|
+
# notice. A dropped SSH link is a reset instead, which arrives as a raised error
|
|
2007
|
+
# from the read rather than as anything the far end chose to send, and that is a
|
|
2008
|
+
# separate path in the downlink from all three. It has to tell the same two halves
|
|
2009
|
+
# apart, because the fault is no different from the captain's side.
|
|
2010
|
+
#
|
|
2011
|
+
# Its never-played half is the cut-header case far above, which pins the other
|
|
2012
|
+
# wording, so this is the half that was missing.
|
|
2013
|
+
|
|
2014
|
+
|
|
2015
|
+
def a_reset():
|
|
2016
|
+
raise OSError(104, "Connection reset by peer")
|
|
2017
|
+
|
|
2018
|
+
|
|
2019
|
+
reset, _, reset_code = turn_cut(
|
|
2020
|
+
"reset, part of a reply played", SOME_REPLY, "reply-partial-reset.pcm",
|
|
2021
|
+
at_end=a_reset)
|
|
2022
|
+
check(reset["answered"] and reset["reply_audio_seconds"] > 0,
|
|
2023
|
+
"a reset after some of the reply played still reached the captain, so the "
|
|
2024
|
+
"turn was answered: %r" % reset)
|
|
2025
|
+
check(reset["first_audio_s"] is not None,
|
|
2026
|
+
"and when it reached them is a real measurement, not a null: %r" % reset)
|
|
2027
|
+
check(reset["relay_error"].startswith(
|
|
2028
|
+
"the connection was lost before the reply finished"),
|
|
2029
|
+
"a reply cut short by a reset must say the reply did not finish, the same "
|
|
2030
|
+
"as one cut short by a close: %r" % reset["relay_error"])
|
|
2031
|
+
check("before this turn was answered" not in reset["relay_error"],
|
|
2032
|
+
"and must not claim nothing arrived, which its own answered field "
|
|
2033
|
+
"contradicts: %r" % reset["relay_error"])
|
|
2034
|
+
# The clause is what the reader acts on and the detail is what tells a reset from
|
|
2035
|
+
# a header cut in half, so neither may be lost to the other.
|
|
2036
|
+
check("Connection reset by peer" in reset["relay_error"],
|
|
2037
|
+
"and must still carry what the kernel said, which is the only thing that "
|
|
2038
|
+
"tells a reset from a truncated frame: %r" % reset["relay_error"])
|
|
2039
|
+
check(reset_code != 0,
|
|
2040
|
+
"a record naming a fault must not exit 0, even at the default of one run "
|
|
2041
|
+
"and even though the turn was answered, got %r" % reset_code)
|
|
2042
|
+
|
|
2043
|
+
# THE OTHER SIDE OF THE SAME LINE, and the complement of the answered case further
|
|
2044
|
+
# up this block. Those cases are faults that landed while the turn was still owed
|
|
2045
|
+
# an answer. These two land AFTER the answer was complete, in the gap between the
|
|
2046
|
+
# reply_end mark arriving and the record being copied, which the client spends
|
|
2047
|
+
# waiting for the reply audio to go quiet. A turn answered in full must carry no
|
|
2048
|
+
# reason and must not fail the session, whichever path the late fault takes: an
|
|
2049
|
+
# end of stream and a reset differ only in what the kernel delivered, and one relay
|
|
2050
|
+
# death must not produce two different exit codes depending on which it was.
|
|
2051
|
+
#
|
|
2052
|
+
# Gated at both ends rather than timed. The fault cannot be delivered until the
|
|
2053
|
+
# client has passed the reply_end wait, and the record cannot be copied until the
|
|
2054
|
+
# downlink has marked the connection finished with, which it does only after the
|
|
2055
|
+
# fault has been applied. Nothing here sleeps and nothing races.
|
|
2056
|
+
class RaisingAfterReply:
|
|
2057
|
+
"""Serves one whole answer, then raises once the record window is open."""
|
|
2058
|
+
|
|
2059
|
+
def __init__(self, gate, window):
|
|
2060
|
+
self._gate = gate
|
|
2061
|
+
self._window = window
|
|
2062
|
+
self._bytes = frame.encode(frame.AUDIO, b"\x00\x00" * 1200) + REPLY_END
|
|
2063
|
+
self._at = 0
|
|
2064
|
+
|
|
2065
|
+
def read(self, count):
|
|
2066
|
+
check(self._gate.wait(10), "the turn never opened, so nothing was served")
|
|
2067
|
+
if self._at < len(self._bytes):
|
|
2068
|
+
chunk = self._bytes[self._at:self._at + count]
|
|
2069
|
+
self._at += len(chunk)
|
|
2070
|
+
return chunk
|
|
2071
|
+
check(self._window.wait(10),
|
|
2072
|
+
"fixture: the record window never opened, so nothing raced it")
|
|
2073
|
+
raise OSError(104, "Connection reset by peer")
|
|
2074
|
+
|
|
2075
|
+
|
|
2076
|
+
class FailingAfterReply:
|
|
2077
|
+
"""Serves one whole answer, then a named turn failure in that same window."""
|
|
2078
|
+
|
|
2079
|
+
def __init__(self, gate, window):
|
|
2080
|
+
self._gate = gate
|
|
2081
|
+
self._window = window
|
|
2082
|
+
self._reply = frame.encode(frame.AUDIO, b"\x00\x00" * 1200) + REPLY_END
|
|
2083
|
+
self._late = frame.encode_json(
|
|
2084
|
+
frame.NOTICE, {"event": "turn-failed",
|
|
2085
|
+
"error": "the model stream dropped"})
|
|
2086
|
+
self._at = 0
|
|
2087
|
+
self._late_at = 0
|
|
2088
|
+
|
|
2089
|
+
def read(self, count):
|
|
2090
|
+
check(self._gate.wait(10), "the turn never opened, so nothing was served")
|
|
2091
|
+
if self._at < len(self._reply):
|
|
2092
|
+
chunk = self._reply[self._at:self._at + count]
|
|
2093
|
+
self._at += len(chunk)
|
|
2094
|
+
return chunk
|
|
2095
|
+
check(self._window.wait(10),
|
|
2096
|
+
"fixture: the record window never opened, so nothing raced it")
|
|
2097
|
+
if self._late_at < len(self._late):
|
|
2098
|
+
chunk = self._late[self._late_at:self._late_at + count]
|
|
2099
|
+
self._late_at += len(chunk)
|
|
2100
|
+
return chunk
|
|
2101
|
+
# The notice is handled before this read is reached again, and the end of
|
|
2102
|
+
# stream behind it is what marks the connection finished with, which is the
|
|
2103
|
+
# gate the record waits on below.
|
|
2104
|
+
return b""
|
|
2105
|
+
|
|
2106
|
+
|
|
2107
|
+
def fault_after_answer(label, make_stream, out_name):
|
|
2108
|
+
"""Answer one turn in full, then land a fault before the record is copied."""
|
|
2109
|
+
gate, window = thread_lib.Event(), thread_lib.Event()
|
|
2110
|
+
after = wire_client(
|
|
2111
|
+
["--host", "desk", "--in-file", os.path.join(TMP, "clip.pcm"),
|
|
2112
|
+
"--out-file", os.path.join(TMP, out_name),
|
|
2113
|
+
"--timeout", "2", "--audio-idle", "0.05"],
|
|
2114
|
+
make_stream(gate, window), StartGate(gate))
|
|
2115
|
+
quiet_wait = after._wait_audio_quiet
|
|
2116
|
+
|
|
2117
|
+
def open_the_window(deadline):
|
|
2118
|
+
window.set()
|
|
2119
|
+
quiet_wait(deadline)
|
|
2120
|
+
# take_turn copies the record on the line after this returns, so the fault
|
|
2121
|
+
# has to be in before it. closed is the downlink's own mark that it has
|
|
2122
|
+
# finished with the connection and it is set only after the fault has been
|
|
2123
|
+
# applied, which makes it the gate rather than any elapsed time.
|
|
2124
|
+
check(after.closed.wait(10),
|
|
2125
|
+
"%s: fixture: the late fault never landed before the record"
|
|
2126
|
+
% label)
|
|
2127
|
+
|
|
2128
|
+
after._wait_audio_quiet = open_the_window
|
|
2129
|
+
out, said = io.StringIO(), io.StringIO()
|
|
2130
|
+
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(said):
|
|
2131
|
+
code = after.run()
|
|
2132
|
+
after.up_q.put(None)
|
|
2133
|
+
after.playback.close()
|
|
2134
|
+
records = [json_lib.loads(line) for line in out.getvalue().splitlines()
|
|
2135
|
+
if line.strip()]
|
|
2136
|
+
check(len(records) == 1,
|
|
2137
|
+
"%s: one turn was answered, so one record belongs in the file: %r"
|
|
2138
|
+
% (label, records))
|
|
2139
|
+
check(after.options.runs == 1,
|
|
2140
|
+
"%s: fixture is wrong, this case is about the default run count: %r"
|
|
2141
|
+
% (label, after.options.runs))
|
|
2142
|
+
check(records[0]["answered"] and records[0]["reply_audio_seconds"] > 0,
|
|
2143
|
+
"%s: fixture is wrong, the whole reply should have arrived: %r"
|
|
2144
|
+
% (label, records[0]))
|
|
2145
|
+
check(records[0]["relay_error"] is None,
|
|
2146
|
+
"%s: a turn answered in full must carry no reason, whatever arrived "
|
|
2147
|
+
"afterwards: %r" % (label, records[0]["relay_error"]))
|
|
2148
|
+
check(code == 0,
|
|
2149
|
+
"%s: and a session that delivered its answer must exit 0, got %r"
|
|
2150
|
+
% (label, code))
|
|
2151
|
+
return said.getvalue()
|
|
2152
|
+
|
|
2153
|
+
|
|
2154
|
+
# The read-failure path: a reset rather than a clean end of stream.
|
|
2155
|
+
rst_said = fault_after_answer(
|
|
2156
|
+
"a reset after the answer", RaisingAfterReply, "reply-rst.pcm")
|
|
2157
|
+
# The guard is on the record and nothing else, so the captain is still told.
|
|
2158
|
+
check("connection lost" in rst_said,
|
|
2159
|
+
"the connection going is still said on stderr, only not recorded against a "
|
|
2160
|
+
"turn it did not cost: %r" % rst_said)
|
|
2161
|
+
|
|
2162
|
+
# The relay fail_turn path: its own model stream broke, after this turn's answer.
|
|
2163
|
+
failed_said = fault_after_answer(
|
|
2164
|
+
"a named relay failure after the answer", FailingAfterReply,
|
|
2165
|
+
"reply-late-fail.pcm")
|
|
2166
|
+
check("could not finish that turn" in failed_said,
|
|
2167
|
+
"the relay's own words are still said on stderr for the same reason: %r"
|
|
2168
|
+
% failed_said)
|
|
2169
|
+
|
|
2170
|
+
# With runs still to take, the loop also has to say why they were not taken, and
|
|
2171
|
+
# that line has to restate the cause that was recorded rather than asserting a
|
|
2172
|
+
# default. Told "the relay stopped" and then "the connection closed", the captain
|
|
2173
|
+
# has been given two causes for one event and has to guess which end to look at.
|
|
2174
|
+
for label, payload, expected in (
|
|
2175
|
+
("end of stream", b"", "the connection ended"),
|
|
2176
|
+
("goodbye", frame.encode(frame.BYE),
|
|
2177
|
+
"the relay stopped without being asked to")):
|
|
2178
|
+
_, stopped_said = turn_lost(
|
|
2179
|
+
"%s, runs remaining" % label,
|
|
2180
|
+
lambda gate, payload=payload, label=label: ScriptedStream(
|
|
2181
|
+
gate, payload, "the %s" % label),
|
|
2182
|
+
"reply-stop-%s.pcm" % label.replace(" ", "-"), runs="2")
|
|
2183
|
+
check("%s before run 2 of 2" % expected in stopped_said,
|
|
2184
|
+
"the run loop should restate the recorded cause for %s: %r"
|
|
2185
|
+
% (label, stopped_said))
|
|
2186
|
+
check("the connection closed" not in stopped_said,
|
|
2187
|
+
"and must not fall back to the default wording for %s: %r"
|
|
2188
|
+
% (label, stopped_said))
|
|
2189
|
+
|
|
2190
|
+
# A fault on THIS end, handling a reply that did arrive. The try in the downlink
|
|
2191
|
+
# used to cover only the read, so the output file refusing the audio, or a payload
|
|
2192
|
+
# that is not the JSON the wire format promises, killed the reader thread outright
|
|
2193
|
+
# with the connection neither marked closed nor released. run() then opened every
|
|
2194
|
+
# remaining turn, each waiting out the whole timeout and printing answered false
|
|
2195
|
+
# with no reason, so ONE fault cost the session instead of one turn.
|
|
2196
|
+
#
|
|
2197
|
+
# The reason it records must also be the right reason: the connection here is
|
|
2198
|
+
# perfectly healthy, and a record or a message blaming it sends whoever reads it
|
|
2199
|
+
# to the wrong end of a working link.
|
|
2200
|
+
class FullDiskPlayback(client.FilePlayback):
|
|
2201
|
+
"""Refuses reply audio the way a filesystem with nothing left does."""
|
|
2202
|
+
|
|
2203
|
+
def write(self, pcm, turn):
|
|
2204
|
+
raise OSError(28, "No space left on device")
|
|
2205
|
+
|
|
2206
|
+
|
|
2207
|
+
handling_served = thread_lib.Event()
|
|
2208
|
+
handling = wire_client(
|
|
2209
|
+
["--host", "desk", "--in-file", os.path.join(TMP, "clip.pcm"),
|
|
2210
|
+
"--out-file", os.path.join(TMP, "reply-handling.pcm"), "--runs", "2",
|
|
2211
|
+
"--timeout", "2", "--audio-idle", "0.05", "--gap-seconds", "0.05"],
|
|
2212
|
+
ScriptedStream(handling_served, frame.encode(frame.AUDIO, b"\x00\x00" * 600),
|
|
2213
|
+
"the one reply frame"),
|
|
2214
|
+
StartGate(handling_served),
|
|
2215
|
+
FullDiskPlayback(os.path.join(TMP, "reply-handling.pcm")))
|
|
2216
|
+
handling_out, handling_said = io.StringIO(), io.StringIO()
|
|
2217
|
+
with contextlib.redirect_stdout(handling_out), \
|
|
2218
|
+
contextlib.redirect_stderr(handling_said):
|
|
2219
|
+
handling_code = handling.run()
|
|
2220
|
+
handling.up_q.put(None)
|
|
2221
|
+
handling.playback.close()
|
|
2222
|
+
handling_said = handling_said.getvalue()
|
|
2223
|
+
|
|
2224
|
+
handling_runs = [json_lib.loads(line)
|
|
2225
|
+
for line in handling_out.getvalue().splitlines() if line.strip()]
|
|
2226
|
+
# One fault, one lost turn. The reader surviving is the whole point: without it
|
|
2227
|
+
# the second run opens as well and burns another whole timeout for nothing.
|
|
2228
|
+
check(len(handling_runs) == 1,
|
|
2229
|
+
"a fault handling a reply must stop the session rather than opening every "
|
|
2230
|
+
"remaining turn: %d run(s), %r" % (len(handling_runs), handling_runs))
|
|
2231
|
+
check(not handling_runs[0]["answered"],
|
|
2232
|
+
"the audio never reached the file, so the turn was not answered: %r"
|
|
2233
|
+
% handling_runs[0])
|
|
2234
|
+
check(handling_runs[0]["relay_error"],
|
|
2235
|
+
"a fault this end could name must not be recorded as a turn that merely "
|
|
2236
|
+
"went unanswered: %r" % handling_runs[0])
|
|
2237
|
+
check("No space left on device" in handling_runs[0]["relay_error"],
|
|
2238
|
+
"the record should carry what actually raised: %r"
|
|
2239
|
+
% handling_runs[0]["relay_error"])
|
|
2240
|
+
# The overriding rule: a wrong cause is worse than a missing one.
|
|
2241
|
+
check("connection" not in handling_runs[0]["relay_error"],
|
|
2242
|
+
"the connection was never lost here, so the record must not say it was: %r"
|
|
2243
|
+
% handling_runs[0]["relay_error"])
|
|
2244
|
+
handling_lines = [line for line in handling_said.splitlines()
|
|
2245
|
+
if line.startswith("client:")]
|
|
2246
|
+
check(not any("connection" in line for line in handling_lines),
|
|
2247
|
+
"and the captain must not be sent to a connection that is working: %r"
|
|
2248
|
+
% handling_lines)
|
|
2249
|
+
check("No space left on device" in handling_said
|
|
2250
|
+
and "run 2 of 2" in handling_said,
|
|
2251
|
+
"the run loop should name the real reason and the run it stopped at: %r"
|
|
2252
|
+
% handling_said)
|
|
2253
|
+
# The one line is what the captain and the record get. The traceback is what
|
|
2254
|
+
# whoever has to find the bug behind it gets, and before the guard existed a dying
|
|
2255
|
+
# thread printed one, so losing it would make a programming error in the frame
|
|
2256
|
+
# handling strictly harder to locate than it used to be.
|
|
2257
|
+
check("Traceback (most recent call last)" in handling_said,
|
|
2258
|
+
"a fault this end raised should still leave its traceback on stderr: %r"
|
|
2259
|
+
% handling_said)
|
|
2260
|
+
check("Traceback" not in (handling_runs[0]["relay_error"] or ""),
|
|
2261
|
+
"but the record is machine read, so the one-line reason belongs there: %r"
|
|
2262
|
+
% handling_runs[0]["relay_error"])
|
|
2263
|
+
check(handling_code != 0,
|
|
2264
|
+
"a session that took none of its runs must not exit 0, got %r"
|
|
2265
|
+
% handling_code)
|
|
2266
|
+
|
|
2267
|
+
# The other end of the same handler: reply audio that arrives after THIS end has
|
|
2268
|
+
# released the output. close() joins the downlink at five seconds while the relay
|
|
2269
|
+
# teardown it waits on can take up to ten, so the join can expire with audio still
|
|
2270
|
+
# in flight, and the chunk behind it then met a closed file. That raised into the
|
|
2271
|
+
# guard above and printed its fault line and a full traceback on a session that
|
|
2272
|
+
# answered its turn and exited 0, which is an alarm firing on success: the reader
|
|
2273
|
+
# learns to skip the line, and the real one is then invisible too. The connection
|
|
2274
|
+
# was never the problem either, so it was a wrong cause as well as a false one.
|
|
2275
|
+
#
|
|
2276
|
+
# Driven through the events the client itself reaches, not a sleep: the turn is
|
|
2277
|
+
# answered and recorded, the output is released exactly as close() releases it, and
|
|
2278
|
+
# only then is the late chunk let through.
|
|
2279
|
+
class LateAudioStream:
|
|
2280
|
+
"""Serves one whole answer, then one more chunk once the output is released."""
|
|
2281
|
+
|
|
2282
|
+
def __init__(self, opened, released):
|
|
2283
|
+
self._opened = opened
|
|
2284
|
+
self._released = released
|
|
2285
|
+
self._reply = frame.encode(frame.AUDIO, b"\x00\x00" * 1200) + REPLY_END
|
|
2286
|
+
self._late = frame.encode(frame.AUDIO, b"\x00\x00" * 1200)
|
|
2287
|
+
self._at = 0
|
|
2288
|
+
self._late_at = 0
|
|
2289
|
+
self._never = thread_lib.Event()
|
|
2290
|
+
|
|
2291
|
+
def read(self, count):
|
|
2292
|
+
check(self._opened.wait(10), "the turn never opened, so nothing was served")
|
|
2293
|
+
if self._at < len(self._reply):
|
|
2294
|
+
chunk = self._reply[self._at:self._at + count]
|
|
2295
|
+
self._at += len(chunk)
|
|
2296
|
+
return chunk
|
|
2297
|
+
check(self._released.wait(10),
|
|
2298
|
+
"fixture: the output was never released, so nothing arrived late")
|
|
2299
|
+
if self._late_at < len(self._late):
|
|
2300
|
+
chunk = self._late[self._late_at:self._late_at + count]
|
|
2301
|
+
self._late_at += len(chunk)
|
|
2302
|
+
return chunk
|
|
2303
|
+
# Parked rather than ending the stream, so the only lines on stderr are the
|
|
2304
|
+
# ones this case is about.
|
|
2305
|
+
self._never.wait(30)
|
|
2306
|
+
return b""
|
|
2307
|
+
|
|
2308
|
+
|
|
2309
|
+
class CountingPlayback(client.FilePlayback):
|
|
2310
|
+
"""Reports each chunk once it has been handed over, so the test can wait."""
|
|
2311
|
+
|
|
2312
|
+
def __init__(self, path, handled):
|
|
2313
|
+
client.FilePlayback.__init__(self, path)
|
|
2314
|
+
self._handled = handled
|
|
2315
|
+
self.calls = 0
|
|
2316
|
+
|
|
2317
|
+
def write(self, pcm, turn):
|
|
2318
|
+
self.calls += 1
|
|
2319
|
+
try:
|
|
2320
|
+
client.FilePlayback.write(self, pcm, turn)
|
|
2321
|
+
finally:
|
|
2322
|
+
# In a finally, so the late chunk raising is as observable as the late
|
|
2323
|
+
# chunk being discarded and neither outcome hangs the case.
|
|
2324
|
+
if self.calls >= 2:
|
|
2325
|
+
self._handled.set()
|
|
2326
|
+
|
|
2327
|
+
|
|
2328
|
+
def audio_after_close(label, extra):
|
|
2329
|
+
"""Answer one turn, release the output, then let a late chunk arrive."""
|
|
2330
|
+
opened, released, handled = (thread_lib.Event(), thread_lib.Event(),
|
|
2331
|
+
thread_lib.Event())
|
|
2332
|
+
out_file = os.path.join(TMP, "reply-late-%s.pcm" % label)
|
|
2333
|
+
late = wire_client(
|
|
2334
|
+
["--host", "desk", "--in-file", os.path.join(TMP, "clip.pcm"),
|
|
2335
|
+
"--out-file", out_file, "--timeout", "2", "--audio-idle", "0.05"]
|
|
2336
|
+
+ extra,
|
|
2337
|
+
LateAudioStream(opened, released), StartGate(opened),
|
|
2338
|
+
CountingPlayback(out_file, handled))
|
|
2339
|
+
out, said = io.StringIO(), io.StringIO()
|
|
2340
|
+
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(said):
|
|
2341
|
+
code = late.run()
|
|
2342
|
+
# Exactly what close() does with the output, at the point close() does it.
|
|
2343
|
+
late.playback.close()
|
|
2344
|
+
released.set()
|
|
2345
|
+
check(handled.wait(10), "fixture: the late chunk never reached the output")
|
|
2346
|
+
# Through close(), not the reporting method directly, so the wiring is
|
|
2347
|
+
# covered as well as the report: nothing else drives the only production
|
|
2348
|
+
# caller, and hand-calling it would pass with that call deleted. Returns at
|
|
2349
|
+
# once here, because the uplink stand-in swallows the quit, no downlink
|
|
2350
|
+
# thread or relay process was ever assigned, and the file drain is a no-op.
|
|
2351
|
+
late.close()
|
|
2352
|
+
late.up_q.put(None)
|
|
2353
|
+
records = [json_lib.loads(line) for line in out.getvalue().splitlines()
|
|
2354
|
+
if line.strip()]
|
|
2355
|
+
check(len(records) == 1,
|
|
2356
|
+
"%s: one turn was answered, so one record belongs in the file: %r"
|
|
2357
|
+
% (label, records))
|
|
2358
|
+
return late, records[0], said.getvalue(), code, out_file
|
|
2359
|
+
|
|
2360
|
+
|
|
2361
|
+
quiet, quiet_run, quiet_said, quiet_code, quiet_file = audio_after_close(
|
|
2362
|
+
"quiet", [])
|
|
2363
|
+
# THE POINT. The session worked, so nothing may read as a fault.
|
|
2364
|
+
check("could not handle the relay's reply" not in quiet_said,
|
|
2365
|
+
"audio arriving after the output was released is this end's own teardown, "
|
|
2366
|
+
"not a fault to alarm on: %r" % quiet_said)
|
|
2367
|
+
check("Traceback" not in quiet_said,
|
|
2368
|
+
"and it must not leave a traceback behind either: %r" % quiet_said)
|
|
2369
|
+
check(quiet_code == 0,
|
|
2370
|
+
"a session whose only turn was answered must still exit 0, got %r"
|
|
2371
|
+
% quiet_code)
|
|
2372
|
+
check(quiet_run["answered"] and quiet_run["relay_error"] is None,
|
|
2373
|
+
"and its record stands, reason-free: %r" % quiet_run)
|
|
2374
|
+
# Condition 2: the discard is diagnostic only. The late chunk is the same size as
|
|
2375
|
+
# the answer, so anything crediting it would double both figures.
|
|
2376
|
+
check(quiet.playback.discarded == 1,
|
|
2377
|
+
"the late chunk must be counted as discarded: %r"
|
|
2378
|
+
% quiet.playback.discarded)
|
|
2379
|
+
check(quiet.playback.turn_bytes == 2400,
|
|
2380
|
+
"but must not be credited to the turn: %r" % quiet.playback.turn_bytes)
|
|
2381
|
+
check(quiet_run["reply_audio_seconds"] == 0.05,
|
|
2382
|
+
"so the reply's own duration stands: %r" % quiet_run["reply_audio_seconds"])
|
|
2383
|
+
check(quiet_run["first_audio_s"] is not None,
|
|
2384
|
+
"and the answer's real first-audio figure stands: %r"
|
|
2385
|
+
% quiet_run["first_audio_s"])
|
|
2386
|
+
check(os.path.getsize(quiet_file) == 2400,
|
|
2387
|
+
"and the discarded chunk reached no file: %d bytes"
|
|
2388
|
+
% os.path.getsize(quiet_file))
|
|
2389
|
+
|
|
2390
|
+
# Condition 1: discarded, not silently swallowed. A write after close outside
|
|
2391
|
+
# teardown is a real logic bug, so the count has somewhere to be read.
|
|
2392
|
+
loud, _, loud_said, loud_code, _ = audio_after_close("loud", ["--verbose"])
|
|
2393
|
+
check("discarded 1 reply audio chunk" in loud_said,
|
|
2394
|
+
"--verbose must say how many chunks were dropped: %r" % loud_said)
|
|
2395
|
+
check("could not handle the relay's reply" not in loud_said
|
|
2396
|
+
and "Traceback" not in loud_said,
|
|
2397
|
+
"and saying so is not the same as calling it a fault: %r" % loud_said)
|
|
2398
|
+
check(loud_code == 0 and loud.playback.turn_bytes == 2400,
|
|
2399
|
+
"and reporting it changes neither the exit code nor the turn's bytes: %r %r"
|
|
2400
|
+
% (loud_code, loud.playback.turn_bytes))
|
|
2401
|
+
|
|
2402
|
+
# Which of the playback's two locks covers what, driven rather than asserted about.
|
|
2403
|
+
# take_turn calls turn_reset while holding the client's own turn lock, so anything
|
|
2404
|
+
# turn_reset can wait behind stalls the whole client: neither the downlink nor the
|
|
2405
|
+
# sender can stamp a thing without that lock. The file write is the one slow step
|
|
2406
|
+
# here, so it must sit outside the lock turn_reset takes. The handle below parks
|
|
2407
|
+
# instead of writing, which makes a slow filesystem exact rather than simulated.
|
|
2408
|
+
parked, release_write, reset_done = (thread_lib.Event(), thread_lib.Event(),
|
|
2409
|
+
thread_lib.Event())
|
|
2410
|
+
|
|
2411
|
+
|
|
2412
|
+
class ParkingHandle:
|
|
2413
|
+
"""A file whose write blocks until this case lets it through."""
|
|
2414
|
+
|
|
2415
|
+
def __init__(self):
|
|
2416
|
+
self.written = 0
|
|
2417
|
+
|
|
2418
|
+
def write(self, pcm):
|
|
2419
|
+
self.written += len(pcm)
|
|
2420
|
+
parked.set()
|
|
2421
|
+
check(release_write.wait(10), "fixture: the parked write was never freed")
|
|
2422
|
+
|
|
2423
|
+
def close(self):
|
|
2424
|
+
pass
|
|
2425
|
+
|
|
2426
|
+
|
|
2427
|
+
slow = client.FilePlayback(os.path.join(TMP, "reply-slow.pcm"))
|
|
2428
|
+
slow._handle.close()
|
|
2429
|
+
slow._handle = ParkingHandle()
|
|
2430
|
+
slow.turn_reset(1)
|
|
2431
|
+
thread_lib.Thread(target=slow.write, args=(b"\x00\x00" * 600, 1),
|
|
2432
|
+
daemon=True).start()
|
|
2433
|
+
check(parked.wait(10), "fixture: the write never reached the handle")
|
|
2434
|
+
|
|
2435
|
+
|
|
2436
|
+
def advance_the_turn():
|
|
2437
|
+
slow.turn_reset(2)
|
|
2438
|
+
reset_done.set()
|
|
2439
|
+
|
|
2440
|
+
|
|
2441
|
+
thread_lib.Thread(target=advance_the_turn, daemon=True).start()
|
|
2442
|
+
# Bounded, and it discriminates in both directions: with the write outside that
|
|
2443
|
+
# lock the reset completes at once, and with the write inside it the reset cannot
|
|
2444
|
+
# complete until the line below runs, whatever the machine is doing.
|
|
2445
|
+
check(reset_done.wait(5),
|
|
2446
|
+
"a turn advance must not wait behind a file write, because take_turn makes "
|
|
2447
|
+
"it while holding the lock the downlink and the sender both need")
|
|
2448
|
+
release_write.set()
|
|
2449
|
+
# The accounting still had to happen, and under the lock: the chunk was turn one's.
|
|
2450
|
+
check(slow.turn_bytes == 0 and slow.first_played is None,
|
|
2451
|
+
"and turn two starts owed nothing and unstamped: %r %r"
|
|
2452
|
+
% (slow.turn_bytes, slow.first_played))
|
|
2453
|
+
|
|
2454
|
+
# The record's own two figures for one turn's audio. reply_audio_seconds and
|
|
2455
|
+
# answered are the same measurement asked twice, and the record is built outside
|
|
2456
|
+
# the playback's lock, so reading that measurement twice lets a chunk land between
|
|
2457
|
+
# the two reads and produce a record saying both that no audio arrived and that the
|
|
2458
|
+
# turn was answered. The reader of runs.jsonl then has to guess which half to
|
|
2459
|
+
# believe, which is the whole failure this work exists to remove.
|
|
2460
|
+
#
|
|
2461
|
+
# The playback below is that interleaving made exact rather than raced for: the
|
|
2462
|
+
# count grows between one read of it and the next, so a record built from two reads
|
|
2463
|
+
# cannot agree with itself and a record built from one always does.
|
|
2464
|
+
class SlippingCount(client.FilePlayback):
|
|
2465
|
+
"""A playback whose byte count grows between one read of it and the next."""
|
|
2466
|
+
|
|
2467
|
+
def __init__(self, path):
|
|
2468
|
+
self.reads = 0
|
|
2469
|
+
client.FilePlayback.__init__(self, path)
|
|
2470
|
+
|
|
2471
|
+
@property
|
|
2472
|
+
def turn_bytes(self):
|
|
2473
|
+
self.reads += 1
|
|
2474
|
+
return self._counted if self.reads == 1 else self._counted + 2400
|
|
2475
|
+
|
|
2476
|
+
@turn_bytes.setter
|
|
2477
|
+
def turn_bytes(self, count):
|
|
2478
|
+
self._counted = count
|
|
2479
|
+
|
|
2480
|
+
|
|
2481
|
+
slip_open, slip_parked = thread_lib.Event(), thread_lib.Event()
|
|
2482
|
+
|
|
2483
|
+
|
|
2484
|
+
def park_the_downlink():
|
|
2485
|
+
# Parked rather than ended, because an end of stream would make the downlink
|
|
2486
|
+
# name a reason, and naming one reads the byte count itself. The turn has to
|
|
2487
|
+
# reach its record with the count still unread by anything else.
|
|
2488
|
+
slip_parked.wait(30)
|
|
2489
|
+
|
|
2490
|
+
|
|
2491
|
+
slipping = SlippingCount(os.path.join(TMP, "reply-slip.pcm"))
|
|
2492
|
+
# The timeout boundary is the one moment the first chunk of a reply can land while
|
|
2493
|
+
# the record is being built, because a turn that timed out with nothing played is
|
|
2494
|
+
# the only one whose count is still zero when the record reads it.
|
|
2495
|
+
slipped = wire_client(
|
|
2496
|
+
["--host", "desk", "--in-file", os.path.join(TMP, "clip.pcm"),
|
|
2497
|
+
"--out-file", os.path.join(TMP, "reply-slip.pcm"),
|
|
2498
|
+
"--timeout", "0.3", "--audio-idle", "0.05"],
|
|
2499
|
+
ScriptedStream(slip_open, b"", "nothing at all", park_the_downlink),
|
|
2500
|
+
StartGate(slip_open), slipping)
|
|
2501
|
+
slip_out, slip_said = io.StringIO(), io.StringIO()
|
|
2502
|
+
with contextlib.redirect_stdout(slip_out), contextlib.redirect_stderr(slip_said):
|
|
2503
|
+
slip_code = slipped.run()
|
|
2504
|
+
slipped.up_q.put(None)
|
|
2505
|
+
slip_records = [json_lib.loads(line) for line in slip_out.getvalue().splitlines()
|
|
2506
|
+
if line.strip()]
|
|
2507
|
+
check(len(slip_records) == 1,
|
|
2508
|
+
"the turn was taken, so its record belongs in the file: %r" % slip_records)
|
|
2509
|
+
slip = slip_records[0]
|
|
2510
|
+
check(slipping.reads >= 1,
|
|
2511
|
+
"fixture: the record never read the byte count at all, so this case proves "
|
|
2512
|
+
"nothing: %r" % slipping.reads)
|
|
2513
|
+
# THE POINT. One record may not answer the same question two ways.
|
|
2514
|
+
check(slip["answered"] == (slip["reply_audio_seconds"] > 0),
|
|
2515
|
+
"a record that counts no reply audio must not also call the turn answered: "
|
|
2516
|
+
"%r" % slip)
|
|
2517
|
+
check(slip["reply_audio_seconds"] == 0.0 and slip["answered"] is False,
|
|
2518
|
+
"and the figures are the ones true when the record was built, not one from "
|
|
2519
|
+
"before the chunk and one from after: %r" % slip)
|
|
2520
|
+
check(slip_code != 0,
|
|
2521
|
+
"an unanswered turn must not exit 0, got %r" % slip_code)
|
|
2522
|
+
# Released under a redirect so the end of stream this case parked cannot print into
|
|
2523
|
+
# the suite's own output, and waited for so it cannot land after the case has ended.
|
|
2524
|
+
with contextlib.redirect_stderr(io.StringIO()):
|
|
2525
|
+
slip_parked.set()
|
|
2526
|
+
slip_drained = slipped.closed.wait(10)
|
|
2527
|
+
check(slip_drained, "fixture: the parked stream never ended")
|
|
2528
|
+
|
|
2529
|
+
# An end of stream arriving during THIS end's own teardown. close() bounds the
|
|
2530
|
+
# relay's exit and the relay's own teardown can outlast that bound, so the child is
|
|
2531
|
+
# killed with no goodbye written and the still-live downlink reads the empty stream
|
|
2532
|
+
# it left behind. Every turn was answered and the session exits 0, so the mid-turn
|
|
2533
|
+
# fault line must not be the last thing the captain reads: an alarm that also fires
|
|
2534
|
+
# on success is one they learn to skip past, and then the real one is invisible too.
|
|
2535
|
+
#
|
|
2536
|
+
# The discriminator is the same one the goodbye branch uses, whether this end asked
|
|
2537
|
+
# to stop, and the mid-session case further up this file is the other direction of
|
|
2538
|
+
# it: there the stream ends long before close() is reached, so the line still fires.
|
|
2539
|
+
class QuitThenEnd:
|
|
2540
|
+
"""Serves one whole answer, then ends the stream once this end says goodbye."""
|
|
2541
|
+
|
|
2542
|
+
def __init__(self, opened, quit_seen):
|
|
2543
|
+
self._opened = opened
|
|
2544
|
+
self._quit = quit_seen
|
|
2545
|
+
self._bytes = frame.encode(frame.AUDIO, b"\x00\x00" * 1200) + REPLY_END
|
|
2546
|
+
self._at = 0
|
|
2547
|
+
|
|
2548
|
+
def read(self, count):
|
|
2549
|
+
check(self._opened.wait(10), "the turn never opened, so nothing was served")
|
|
2550
|
+
if self._at < len(self._bytes):
|
|
2551
|
+
chunk = self._bytes[self._at:self._at + count]
|
|
2552
|
+
self._at += len(chunk)
|
|
2553
|
+
return chunk
|
|
2554
|
+
check(self._quit.wait(10),
|
|
2555
|
+
"fixture: this end never asked to stop, so the stream never ended")
|
|
2556
|
+
return b""
|
|
2557
|
+
|
|
2558
|
+
|
|
2559
|
+
class GoodbyeWatchingGate:
|
|
2560
|
+
"""Opens on the turn's first frame and reports this end asking to stop."""
|
|
2561
|
+
|
|
2562
|
+
def __init__(self, opened, quit_seen):
|
|
2563
|
+
self._opened = opened
|
|
2564
|
+
self._quit = quit_seen
|
|
2565
|
+
|
|
2566
|
+
def send(self, kind, payload=b""):
|
|
2567
|
+
if kind == frame.TALK_START:
|
|
2568
|
+
self._opened.set()
|
|
2569
|
+
elif kind == frame.QUIT:
|
|
2570
|
+
self._quit.set()
|
|
2571
|
+
|
|
2572
|
+
|
|
2573
|
+
tidy_open, tidy_quit = thread_lib.Event(), thread_lib.Event()
|
|
2574
|
+
tidy = wire_client(
|
|
2575
|
+
["--host", "desk", "--in-file", os.path.join(TMP, "clip.pcm"),
|
|
2576
|
+
"--out-file", os.path.join(TMP, "reply-tidy.pcm"),
|
|
2577
|
+
"--timeout", "2", "--audio-idle", "0.05"],
|
|
2578
|
+
QuitThenEnd(tidy_open, tidy_quit), GoodbyeWatchingGate(tidy_open, tidy_quit))
|
|
2579
|
+
tidy_out, tidy_said = io.StringIO(), io.StringIO()
|
|
2580
|
+
with contextlib.redirect_stdout(tidy_out), contextlib.redirect_stderr(tidy_said):
|
|
2581
|
+
tidy_code = tidy.run()
|
|
2582
|
+
# Gated on the downlink reaching the end of stream rather than on any elapsed
|
|
2583
|
+
# time: close() does not wait for it here, so without this the case could pass
|
|
2584
|
+
# by asserting on output the downlink had not written yet.
|
|
2585
|
+
tidy.close()
|
|
2586
|
+
tidy_drained = tidy.closed.wait(10)
|
|
2587
|
+
tidy.up_q.put(None)
|
|
2588
|
+
check(tidy_drained, "fixture: the downlink never saw the stream end")
|
|
2589
|
+
tidy_records = [json_lib.loads(line) for line in tidy_out.getvalue().splitlines()
|
|
2590
|
+
if line.strip()]
|
|
2591
|
+
check(len(tidy_records) == 1,
|
|
2592
|
+
"one turn was answered, so one record belongs in the file: %r" % tidy_records)
|
|
2593
|
+
# Proof the end of stream really was taken, so the silence below is a decision
|
|
2594
|
+
# rather than a branch this case never reached: the cause is recorded outside the
|
|
2595
|
+
# test that decides whether to speak.
|
|
2596
|
+
check(tidy.closed_because == "the connection ended",
|
|
2597
|
+
"fixture: the downlink did not take the end of stream at all: %r"
|
|
2598
|
+
% tidy.closed_because)
|
|
2599
|
+
# THE POINT. The session answered everything asked of it, so nothing may read as a
|
|
2600
|
+
# fault, least of all as the last line the captain sees.
|
|
2601
|
+
check("the connection ended" not in tidy_said.getvalue(),
|
|
2602
|
+
"an end of stream during this end's own teardown is the relay doing as it "
|
|
2603
|
+
"was told, not a fault to alarm on: %r" % tidy_said.getvalue())
|
|
2604
|
+
check(tidy_code == 0,
|
|
2605
|
+
"and a session whose only turn was answered must still exit 0, got %r"
|
|
2606
|
+
% tidy_code)
|
|
2607
|
+
check(tidy_records[0]["answered"] and tidy_records[0]["relay_error"] is None,
|
|
2608
|
+
"and its record stands, reason-free: %r" % tidy_records[0])
|
|
2609
|
+
|
|
2610
|
+
# A reply that is handled AFTER the turn it belongs to has already been recorded.
|
|
2611
|
+
# The downlink reads a frame on one thread and applies it on another, so a turn
|
|
2612
|
+
# that times out while a notice is in flight used to have that notice applied to
|
|
2613
|
+
# whatever turn came next: it named the new turn with the old turn's fault, and
|
|
2614
|
+
# released it before its own answer had arrived. A turn that would have been
|
|
2615
|
+
# answered normally was reported unanswered, carrying a reason belonging to a turn
|
|
2616
|
+
# the captain had already been told about.
|
|
2617
|
+
#
|
|
2618
|
+
# Driven through the client's own say(), not a sleep: the notice is parked exactly
|
|
2619
|
+
# where the real window is, after the frame has arrived and before its handler
|
|
2620
|
+
# takes the lock, and it is released only once the NEXT turn has really opened,
|
|
2621
|
+
# which the uplink reports when that turn's talk start goes out.
|
|
2622
|
+
class TurnCountingGate:
|
|
2623
|
+
"""Reports the first turn opening, and the second one separately."""
|
|
2624
|
+
|
|
2625
|
+
def __init__(self, first, second):
|
|
2626
|
+
self._first = first
|
|
2627
|
+
self._second = second
|
|
2628
|
+
self.starts = 0
|
|
2629
|
+
|
|
2630
|
+
def send(self, kind, payload=b""):
|
|
2631
|
+
if kind == frame.TALK_START:
|
|
2632
|
+
self.starts += 1
|
|
2633
|
+
(self._first if self.starts == 1 else self._second).set()
|
|
2634
|
+
|
|
2635
|
+
|
|
2636
|
+
stale_open, stale_next = thread_lib.Event(), thread_lib.Event()
|
|
2637
|
+
# Wired without starting its threads, because the notice has to be parked before
|
|
2638
|
+
# the downlink is allowed to read it.
|
|
2639
|
+
stale = wire_client(
|
|
2640
|
+
["--host", "desk", "--in-file", os.path.join(TMP, "clip.pcm"),
|
|
2641
|
+
"--out-file", os.path.join(TMP, "reply-stale.pcm"), "--runs", "2",
|
|
2642
|
+
"--timeout", "2", "--audio-idle", "0.05", "--gap-seconds", "0.05"],
|
|
2643
|
+
ScriptedStream(
|
|
2644
|
+
stale_open,
|
|
2645
|
+
frame.encode_json(frame.NOTICE, {"event": "turn-failed",
|
|
2646
|
+
"error": "the model dropped turn one"})
|
|
2647
|
+
+ frame.encode(frame.AUDIO, b"\x00\x00" * 1200) + REPLY_END,
|
|
2648
|
+
"turn one's failure and turn two's answer"),
|
|
2649
|
+
TurnCountingGate(stale_open, stale_next), start=False)
|
|
2650
|
+
|
|
2651
|
+
# The notice is held between arriving and being applied, which is the window the
|
|
2652
|
+
# turn identity exists to close. Turn one then times out and is recorded, turn two
|
|
2653
|
+
# opens, and only then is the notice allowed to finish being handled.
|
|
2654
|
+
said_plainly = client.say
|
|
2655
|
+
|
|
2656
|
+
|
|
2657
|
+
def park_the_notice(message):
|
|
2658
|
+
said_plainly(message)
|
|
2659
|
+
if "could not finish that turn" in message:
|
|
2660
|
+
check(stale_next.wait(10),
|
|
2661
|
+
"fixture: the second turn never opened, so nothing went stale")
|
|
2662
|
+
|
|
2663
|
+
|
|
2664
|
+
client.say = park_the_notice
|
|
2665
|
+
try:
|
|
2666
|
+
thread_lib.Thread(target=stale._sender, daemon=True).start()
|
|
2667
|
+
thread_lib.Thread(target=stale._downlink, daemon=True).start()
|
|
2668
|
+
stale_out, stale_said = io.StringIO(), io.StringIO()
|
|
2669
|
+
with contextlib.redirect_stdout(stale_out), contextlib.redirect_stderr(stale_said):
|
|
2670
|
+
stale_code = stale.run()
|
|
2671
|
+
finally:
|
|
2672
|
+
client.say = said_plainly
|
|
2673
|
+
stale.up_q.put(None)
|
|
2674
|
+
stale.playback.close()
|
|
2675
|
+
|
|
2676
|
+
stale_runs = [json_lib.loads(line) for line in stale_out.getvalue().splitlines()
|
|
2677
|
+
if line.strip()]
|
|
2678
|
+
check(len(stale_runs) == 2,
|
|
2679
|
+
"both turns were taken, so both belong in the file: %d, %r"
|
|
2680
|
+
% (len(stale_runs), stale_runs))
|
|
2681
|
+
check(not stale_runs[0]["answered"],
|
|
2682
|
+
"fixture is wrong: the first turn should time out unanswered: %r"
|
|
2683
|
+
% stale_runs[0])
|
|
2684
|
+
# THE POINT. The stale notice belonged to turn one and must touch nothing else.
|
|
2685
|
+
check(stale_runs[1]["relay_error"] is None,
|
|
2686
|
+
"a fault from a turn that has already ended must not be recorded against "
|
|
2687
|
+
"the next one: %r" % stale_runs[1])
|
|
2688
|
+
check(stale_runs[1]["answered"] and stale_runs[1]["reply_audio_seconds"] > 0,
|
|
2689
|
+
"and the next turn must be left to run normally rather than released "
|
|
2690
|
+
"before its answer arrived: %r" % stale_runs[1])
|
|
2691
|
+
check(stale_code != 0,
|
|
2692
|
+
"one turn of two was still lost, so the exit code must say so, got %r"
|
|
2693
|
+
% stale_code)
|
|
2694
|
+
|
|
2695
|
+
# The same window, with AUDIO in it instead of a notice, which is the worse half.
|
|
2696
|
+
# The frame's timing marks are taken under the turn lock, but the audio itself goes
|
|
2697
|
+
# to the playback, and the playback is where first_played is stamped, where the
|
|
2698
|
+
# reply's duration is counted, and where answered comes from. Credited to whatever
|
|
2699
|
+
# turn is open, one stale chunk gives a turn that was never answered a headline
|
|
2700
|
+
# latency figure measured from somebody else's reply, reports it answered, and with
|
|
2701
|
+
# that the whole session exits 0 having lost a turn.
|
|
2702
|
+
#
|
|
2703
|
+
# Parked at the playback rather than at say(), because that IS the accounting
|
|
2704
|
+
# point: the write is released only once the next turn has really opened, which the
|
|
2705
|
+
# uplink reports when that turn's talk start goes out. No sleeps.
|
|
2706
|
+
class LatePlayback(client.FilePlayback):
|
|
2707
|
+
"""Holds the first chunk until the turn after the one it belongs to has opened."""
|
|
2708
|
+
|
|
2709
|
+
def __init__(self, path, opened):
|
|
2710
|
+
client.FilePlayback.__init__(self, path)
|
|
2711
|
+
self._opened = opened
|
|
2712
|
+
self._held = False
|
|
2713
|
+
|
|
2714
|
+
def write(self, pcm, turn):
|
|
2715
|
+
if not self._held:
|
|
2716
|
+
self._held = True
|
|
2717
|
+
check(self._opened.wait(10),
|
|
2718
|
+
"fixture: the next turn never opened, so nothing went stale")
|
|
2719
|
+
client.FilePlayback.write(self, pcm, turn)
|
|
2720
|
+
|
|
2721
|
+
|
|
2722
|
+
late_open, late_next = thread_lib.Event(), thread_lib.Event()
|
|
2723
|
+
late_out_file = os.path.join(TMP, "reply-late.pcm")
|
|
2724
|
+
late = wire_client(
|
|
2725
|
+
["--host", "desk", "--in-file", os.path.join(TMP, "clip.pcm"),
|
|
2726
|
+
"--out-file", late_out_file, "--runs", "2",
|
|
2727
|
+
"--timeout", "2", "--audio-idle", "0.05", "--gap-seconds", "0.05"],
|
|
2728
|
+
ScriptedStream(late_open, frame.encode(frame.AUDIO, b"\x00\x00" * 1200),
|
|
2729
|
+
"turn one's audio"),
|
|
2730
|
+
TurnCountingGate(late_open, late_next),
|
|
2731
|
+
LatePlayback(late_out_file, late_next))
|
|
2732
|
+
late_out, late_said = io.StringIO(), io.StringIO()
|
|
2733
|
+
with contextlib.redirect_stdout(late_out), contextlib.redirect_stderr(late_said):
|
|
2734
|
+
late_code = late.run()
|
|
2735
|
+
late.up_q.put(None)
|
|
2736
|
+
late.playback.close()
|
|
2737
|
+
|
|
2738
|
+
late_runs = [json_lib.loads(line) for line in late_out.getvalue().splitlines()
|
|
2739
|
+
if line.strip()]
|
|
2740
|
+
check(len(late_runs) == 2,
|
|
2741
|
+
"both turns were taken, so both belong in the file: %d, %r"
|
|
2742
|
+
% (len(late_runs), late_runs))
|
|
2743
|
+
# THE POINT. Turn two was served nothing at all, and the stale chunk must not make
|
|
2744
|
+
# it look otherwise on any of the four figures the playback feeds.
|
|
2745
|
+
second = late_runs[1]
|
|
2746
|
+
check(not second["answered"],
|
|
2747
|
+
"a turn served no audio of its own must not be reported answered because "
|
|
2748
|
+
"an earlier turn's chunk arrived during it: %r" % second)
|
|
2749
|
+
check(second["reply_audio_seconds"] == 0,
|
|
2750
|
+
"and none of that chunk's duration belongs to it: %r" % second)
|
|
2751
|
+
check(second["first_played_s"] is None,
|
|
2752
|
+
"and it must not be stamped with when that chunk reached the output: %r"
|
|
2753
|
+
% second)
|
|
2754
|
+
check(second["first_audio_s"] is None,
|
|
2755
|
+
"and the headline figure, which prefers the playback stamp, must be absent "
|
|
2756
|
+
"too rather than measured from another turn's reply: %r" % second)
|
|
2757
|
+
check(second["first_frame_s"] is None,
|
|
2758
|
+
"and the frame stamp stays absent, as the turn identity already ensured: %r"
|
|
2759
|
+
% second)
|
|
2760
|
+
check(late_code != 0,
|
|
2761
|
+
"a session that lost a turn must not exit 0, got %r" % late_code)
|
|
2762
|
+
# And the audio was still written. Attributing it to nobody must not mean dropping
|
|
2763
|
+
# it: it is the tail of an answer the captain is still listening to.
|
|
2764
|
+
check(os.path.getsize(late_out_file) == 2400,
|
|
2765
|
+
"the late chunk must still reach the output, not be discarded to make the "
|
|
2766
|
+
"figures tidy: %d bytes" % os.path.getsize(late_out_file))
|
|
2767
|
+
|
|
2768
|
+
# The wait between turns is the last thing a lost session should do. It exists
|
|
2769
|
+
# only to avoid talking over the model's own speech, and a relay that is already
|
|
2770
|
+
# gone is not speaking, so waiting it out leaves the captain sitting through the
|
|
2771
|
+
# previous reply's whole spoken duration before anyone tells them the session
|
|
2772
|
+
# stopped. The gap here is deliberately long and the reply deliberately short, so
|
|
2773
|
+
# the elapsed time distinguishes stopping from waiting; nothing in the case sleeps
|
|
2774
|
+
# on purpose, the code under test is the sleep.
|
|
2775
|
+
prompt_served = thread_lib.Event()
|
|
2776
|
+
prompt = wire_client(
|
|
2777
|
+
["--host", "desk", "--in-file", os.path.join(TMP, "clip.pcm"),
|
|
2778
|
+
"--out-file", os.path.join(TMP, "reply-prompt.pcm"), "--runs", "3",
|
|
2779
|
+
"--timeout", "2", "--audio-idle", "0.05", "--gap-seconds", "5"],
|
|
2780
|
+
ScriptedStream(prompt_served,
|
|
2781
|
+
frame.encode(frame.AUDIO, b"\x00\x00" * 1200) + REPLY_END,
|
|
2782
|
+
"one whole answer and then the end of stream"),
|
|
2783
|
+
StartGate(prompt_served))
|
|
2784
|
+
prompt_out, prompt_said = io.StringIO(), io.StringIO()
|
|
2785
|
+
prompt_began = clock.monotonic()
|
|
2786
|
+
with contextlib.redirect_stdout(prompt_out), contextlib.redirect_stderr(prompt_said):
|
|
2787
|
+
prompt_code = prompt.run()
|
|
2788
|
+
prompt_took = clock.monotonic() - prompt_began
|
|
2789
|
+
prompt.up_q.put(None)
|
|
2790
|
+
prompt.playback.close()
|
|
2791
|
+
|
|
2792
|
+
prompt_runs = [json_lib.loads(line) for line in prompt_out.getvalue().splitlines()
|
|
2793
|
+
if line.strip()]
|
|
2794
|
+
check(len(prompt_runs) == 1 and prompt_runs[0]["answered"],
|
|
2795
|
+
"fixture is wrong: one answered turn should be taken and no more: %r"
|
|
2796
|
+
% prompt_runs)
|
|
2797
|
+
check(prompt_code != 0,
|
|
2798
|
+
"a session that took 1 of 3 runs must not exit 0, got %r" % prompt_code)
|
|
2799
|
+
check(prompt_took < 3,
|
|
2800
|
+
"a session that already knows the connection is gone must stop rather than "
|
|
2801
|
+
"wait out the 5s gap first: took %.2fs" % prompt_took)
|
|
2802
|
+
check("the connection ended before run 2 of 3" in prompt_said.getvalue(),
|
|
2803
|
+
"and it should name the cause and the first run it lost: %r"
|
|
2804
|
+
% prompt_said.getvalue())
|
|
2805
|
+
|
|
2806
|
+
# The speaker's byte ACCOUNTING, which is not the speaker. No audio device is
|
|
2807
|
+
# reached here and none can be on this host: the stream is a stub and the device
|
|
2808
|
+
# callback is called by hand, so nothing below says anything about how a real
|
|
2809
|
+
# output device behaves. What it does cover is the arithmetic deciding which turn
|
|
2810
|
+
# a chunk is credited to and whose first-audio clock it stamps, which until now
|
|
2811
|
+
# was the one piece of that logic with no coverage at all while the file path it
|
|
2812
|
+
# mirrors had plenty. Its correctness rests on the earlier turns' bytes being a
|
|
2813
|
+
# prefix of the buffer, and a sign or bound slip in the prefix count would
|
|
2814
|
+
# silently mismeasure the headline latency on the path the captain will use.
|
|
2815
|
+
class StubStream:
|
|
2816
|
+
"""Stands in for the output device: accepts the settings, plays nothing."""
|
|
2817
|
+
|
|
2818
|
+
def __init__(self, **settings):
|
|
2819
|
+
self.settings = settings
|
|
2820
|
+
self.latency = 0.011
|
|
2821
|
+
self.started = False
|
|
2822
|
+
|
|
2823
|
+
def start(self):
|
|
2824
|
+
self.started = True
|
|
2825
|
+
|
|
2826
|
+
def stop(self):
|
|
2827
|
+
pass
|
|
2828
|
+
|
|
2829
|
+
def close(self):
|
|
2830
|
+
pass
|
|
2831
|
+
|
|
2832
|
+
|
|
2833
|
+
stub_audio = types.ModuleType("sounddevice")
|
|
2834
|
+
stub_audio.RawOutputStream = StubStream
|
|
2835
|
+
sys.modules["sounddevice"] = stub_audio
|
|
2836
|
+
|
|
2837
|
+
|
|
2838
|
+
def fresh_speaker():
|
|
2839
|
+
return client.SpeakerPlayback()
|
|
2840
|
+
|
|
2841
|
+
|
|
2842
|
+
def pull(speaker, frames):
|
|
2843
|
+
"""Ask the stub device for one block, and return exactly what it was handed."""
|
|
2844
|
+
block = bytearray(frames * 2)
|
|
2845
|
+
speaker._callback(block, frames, None, None)
|
|
2846
|
+
return bytes(block)
|
|
2847
|
+
|
|
2848
|
+
|
|
2849
|
+
opened = fresh_speaker()
|
|
2850
|
+
check(opened.device_latency == 0.011,
|
|
2851
|
+
"the reported device latency should come from the stream: %r"
|
|
2852
|
+
% opened.device_latency)
|
|
2853
|
+
|
|
2854
|
+
# No earlier bytes at all: the first chunk of this turn's own reply is this turn's
|
|
2855
|
+
# first audio.
|
|
2856
|
+
speaker = fresh_speaker()
|
|
2857
|
+
speaker.turn_reset(1)
|
|
2858
|
+
speaker.write(b"\x01\x01" * 300, 1)
|
|
2859
|
+
check(speaker.turn_bytes == 600,
|
|
2860
|
+
"this turn's own audio is credited to it: %r" % speaker.turn_bytes)
|
|
2861
|
+
check(pull(speaker, 300) == b"\x01\x01" * 300, "and is played unchanged")
|
|
2862
|
+
check(speaker.first_played is not None,
|
|
2863
|
+
"and stamps this turn's first audio when it reaches the device")
|
|
2864
|
+
|
|
2865
|
+
# An empty buffer. Silence handed to the device is not the reply arriving, so the
|
|
2866
|
+
# count taken and the count skipped are equal at zero and nothing is stamped.
|
|
2867
|
+
speaker = fresh_speaker()
|
|
2868
|
+
speaker.turn_reset(1)
|
|
2869
|
+
check(pull(speaker, 100) == b"\x00" * 200,
|
|
2870
|
+
"an empty buffer is padded with silence")
|
|
2871
|
+
check(speaker.first_played is None,
|
|
2872
|
+
"and stamps nothing: no reply audio has reached the device yet")
|
|
2873
|
+
|
|
2874
|
+
# Earlier bytes exactly equal to the block taken. The whole block belongs to a turn
|
|
2875
|
+
# already recorded, so it plays and stamps nothing.
|
|
2876
|
+
speaker = fresh_speaker()
|
|
2877
|
+
speaker.turn_reset(2)
|
|
2878
|
+
speaker.write(b"\x02\x02" * 300, 1)
|
|
2879
|
+
check(speaker.turn_bytes == 0,
|
|
2880
|
+
"a chunk from an earlier turn is credited to nobody: %r" % speaker.turn_bytes)
|
|
2881
|
+
check(pull(speaker, 300) == b"\x02\x02" * 300,
|
|
2882
|
+
"but is still played, because it is the tail of an answer being listened to")
|
|
2883
|
+
check(speaker.first_played is None,
|
|
2884
|
+
"and must not stamp the later turn's first audio")
|
|
2885
|
+
|
|
2886
|
+
# Earlier bytes larger than the block taken, drained across two blocks. The count
|
|
2887
|
+
# has to come down by what was taken and no more, or the turn's real first audio
|
|
2888
|
+
# is stamped by somebody else's tail.
|
|
2889
|
+
speaker = fresh_speaker()
|
|
2890
|
+
speaker.turn_reset(2)
|
|
2891
|
+
speaker.write(b"\x03\x03" * 300, 1)
|
|
2892
|
+
speaker.write(b"\x04\x04" * 300, 1)
|
|
2893
|
+
check(pull(speaker, 150) == b"\x03\x03" * 150, "the earlier tail plays in order")
|
|
2894
|
+
check(speaker.first_played is None, "and stamps nothing on the first block")
|
|
2895
|
+
check(pull(speaker, 150) == b"\x03\x03" * 150, "nor on the second")
|
|
2896
|
+
check(speaker.first_played is None, "still nothing stamped")
|
|
2897
|
+
check(pull(speaker, 300) == b"\x04\x04" * 300, "nor while the rest of it drains")
|
|
2898
|
+
check(speaker.first_played is None,
|
|
2899
|
+
"1200 earlier bytes must take 1200 bytes to drain, not one block")
|
|
2900
|
+
speaker.write(b"\x05\x05" * 300, 2)
|
|
2901
|
+
check(pull(speaker, 300) == b"\x05\x05" * 300, "then this turn's own reply plays")
|
|
2902
|
+
check(speaker.first_played is not None, "and that is what stamps the clock")
|
|
2903
|
+
|
|
2904
|
+
# Earlier bytes smaller than the block taken, so one block spans the boundary. The
|
|
2905
|
+
# first byte past the earlier tail is this turn's first audio, and it is inside a
|
|
2906
|
+
# block that started with somebody else's.
|
|
2907
|
+
speaker = fresh_speaker()
|
|
2908
|
+
speaker.turn_reset(2)
|
|
2909
|
+
speaker.write(b"\x06\x06" * 150, 1)
|
|
2910
|
+
speaker.write(b"\x07\x07" * 150, 2)
|
|
2911
|
+
check(pull(speaker, 300) == b"\x06\x06" * 150 + b"\x07\x07" * 150,
|
|
2912
|
+
"a block spanning the boundary is played whole")
|
|
2913
|
+
check(speaker.first_played is not None,
|
|
2914
|
+
"and the first byte past the earlier tail stamps this turn's first audio")
|
|
2915
|
+
check(speaker.turn_bytes == 300,
|
|
2916
|
+
"with only this turn's half credited to it: %r" % speaker.turn_bytes)
|
|
2917
|
+
|
|
2918
|
+
# A turn advancing while the previous answer is still queued, which is the case
|
|
2919
|
+
# turn_reset itself has to handle: everything already buffered belongs to the turn
|
|
2920
|
+
# that is being closed, however it got there.
|
|
2921
|
+
speaker = fresh_speaker()
|
|
2922
|
+
speaker.turn_reset(1)
|
|
2923
|
+
speaker.write(b"\x08\x08" * 300, 1)
|
|
2924
|
+
check(speaker.turn_bytes == 600, "turn one is credited its own reply")
|
|
2925
|
+
speaker.turn_reset(2)
|
|
2926
|
+
check(speaker.turn_bytes == 0 and speaker.first_played is None,
|
|
2927
|
+
"turn two starts owed nothing and unstamped: %r %r"
|
|
2928
|
+
% (speaker.turn_bytes, speaker.first_played))
|
|
2929
|
+
check(pull(speaker, 300) == b"\x08\x08" * 300,
|
|
2930
|
+
"turn one's undrained tail still reaches the device")
|
|
2931
|
+
check(speaker.first_played is None,
|
|
2932
|
+
"but it must not stamp turn two, which is what it would have done before "
|
|
2933
|
+
"turn_reset counted what was already queued")
|
|
2934
|
+
speaker.write(b"\x09\x09" * 300, 2)
|
|
2935
|
+
check(pull(speaker, 300) == b"\x09\x09" * 300 and speaker.first_played is not None,
|
|
2936
|
+
"and turn two's own reply is what stamps turn two")
|
|
2937
|
+
check(speaker.turn_bytes == 600, "credited to turn two: %r" % speaker.turn_bytes)
|
|
2938
|
+
|
|
2939
|
+
# A short buffer of earlier bytes: padded with silence, and the padding is not
|
|
2940
|
+
# mistaken for this turn's reply either.
|
|
2941
|
+
speaker = fresh_speaker()
|
|
2942
|
+
speaker.turn_reset(2)
|
|
2943
|
+
speaker.write(b"\x0a\x0a" * 100, 1)
|
|
2944
|
+
check(pull(speaker, 300) == b"\x0a\x0a" * 100 + b"\x00" * 400,
|
|
2945
|
+
"a short earlier tail is padded rather than repeated")
|
|
2946
|
+
check(speaker.first_played is None, "and the padding stamps nothing")
|
|
2947
|
+
|
|
2948
|
+
# A chunk arriving after close, which is the same teardown race the file path
|
|
2949
|
+
# already discards and counts. Nothing here goes near a device: the stream is the
|
|
2950
|
+
# same stub, and stopping it is what makes the chunk unplayable, so queuing it
|
|
2951
|
+
# would be recording a measurement the captain never heard.
|
|
2952
|
+
speaker = fresh_speaker()
|
|
2953
|
+
speaker.turn_reset(1)
|
|
2954
|
+
speaker.write(b"\x0b\x0b" * 300, 1)
|
|
2955
|
+
speaker.turn_reset(2)
|
|
2956
|
+
speaker.write(b"\x0c\x0c" * 150, 2)
|
|
2957
|
+
was = (speaker.turn_bytes, speaker._earlier, speaker.first_played,
|
|
2958
|
+
bytes(speaker._buffer))
|
|
2959
|
+
check(was == (300, 600, None, b"\x0b\x0b" * 300 + b"\x0c\x0c" * 150),
|
|
2960
|
+
"fixture: turn two should be owed its own bytes behind turn one's tail: %r"
|
|
2961
|
+
% (was,))
|
|
2962
|
+
speaker.close()
|
|
2963
|
+
speaker.write(b"\x0d\x0d" * 300, 2)
|
|
2964
|
+
check(speaker.discarded == 1,
|
|
2965
|
+
"a chunk arriving after close must be counted, not silently swallowed: %r"
|
|
2966
|
+
% speaker.discarded)
|
|
2967
|
+
check(bytes(speaker._buffer) == was[3],
|
|
2968
|
+
"and must not be queued for a stream that has already stopped")
|
|
2969
|
+
check((speaker.turn_bytes, speaker._earlier, speaker.first_played) == was[:3],
|
|
2970
|
+
"and must leave every per-turn figure exactly as it was: %r"
|
|
2971
|
+
% ((speaker.turn_bytes, speaker._earlier, speaker.first_played),))
|
|
2972
|
+
|
|
2973
|
+
# And the count reaches the captain on this path through the same close() that
|
|
2974
|
+
# reports it on the file path, so the counter is not merely present here.
|
|
2975
|
+
heard = fresh_speaker()
|
|
2976
|
+
heard.turn_reset(1)
|
|
2977
|
+
heard.write(b"\x0e\x0e" * 300, 1)
|
|
2978
|
+
check(pull(heard, 300) == b"\x0e\x0e" * 300,
|
|
2979
|
+
"fixture: the turn's own reply should drain before the output is released")
|
|
2980
|
+
heard.close()
|
|
2981
|
+
heard.write(b"\x0f\x0f" * 300, 1)
|
|
2982
|
+
reported = client.Client(client.parse_args(["--host", "desk", "--verbose"]))
|
|
2983
|
+
reported.playback = heard
|
|
2984
|
+
speaker_said = io.StringIO()
|
|
2985
|
+
with contextlib.redirect_stderr(speaker_said):
|
|
2986
|
+
reported.close()
|
|
2987
|
+
check("discarded 1 reply audio chunk" in speaker_said.getvalue(),
|
|
2988
|
+
"--verbose must report the count on this path as it does on the file one: %r"
|
|
2989
|
+
% speaker_said.getvalue())
|
|
2990
|
+
|
|
2991
|
+
del sys.modules["sounddevice"]
|
|
2992
|
+
|
|
2993
|
+
|
|
2994
|
+
|
|
2995
|
+
# A startup that refuses part way through releases what it already started, and
|
|
2996
|
+
# close() therefore has to survive a half-built client. The real devices cannot be
|
|
2997
|
+
# opened on this host, so these stand in for them; what is tested here is the
|
|
2998
|
+
# release path and the refusal, not the devices themselves.
|
|
2999
|
+
class Recorder:
|
|
3000
|
+
def __init__(self, closed):
|
|
3001
|
+
self._closed = closed
|
|
3002
|
+
self.first_played = None
|
|
3003
|
+
self.device_latency = None
|
|
3004
|
+
self.discarded = 0
|
|
3005
|
+
|
|
3006
|
+
def drain(self, timeout=5):
|
|
3007
|
+
pass
|
|
3008
|
+
|
|
3009
|
+
def close(self):
|
|
3010
|
+
self._closed.append("closed")
|
|
3011
|
+
|
|
3012
|
+
def start(self, out_q, talking):
|
|
3013
|
+
pass
|
|
3014
|
+
|
|
3015
|
+
# Nothing built yet: close() must not trip over the fields that are still None.
|
|
3016
|
+
client.Client(client.parse_args(["--host", "desk"])).close()
|
|
3017
|
+
|
|
3018
|
+
# Built part way, then refused: whatever was started is released once.
|
|
3019
|
+
half = client.Client(client.parse_args(["--host", "desk"]))
|
|
3020
|
+
speaker, microphone = [], []
|
|
3021
|
+
half.playback = Recorder(speaker)
|
|
3022
|
+
half.capture = Recorder(microphone)
|
|
3023
|
+
half.close()
|
|
3024
|
+
check(speaker == ["closed"] and microphone == ["closed"],
|
|
3025
|
+
"a half-built client must release both devices: %r %r" % (speaker, microphone))
|
|
3026
|
+
|
|
3027
|
+
# open() releases them itself when a later step refuses, so no caller has to.
|
|
3028
|
+
refusing = client.Client(client.parse_args(["--host", "desk"]))
|
|
3029
|
+
speaker, microphone = [], []
|
|
3030
|
+
|
|
3031
|
+
def half_start():
|
|
3032
|
+
refusing.playback = Recorder(speaker)
|
|
3033
|
+
refusing.capture = Recorder(microphone)
|
|
3034
|
+
raise SystemExit("fm-voice-client: the relay closed the connection")
|
|
3035
|
+
|
|
3036
|
+
refusing._start = half_start
|
|
3037
|
+
raised = None
|
|
3038
|
+
try:
|
|
3039
|
+
refusing.open()
|
|
3040
|
+
except SystemExit as exc:
|
|
3041
|
+
raised = str(exc)
|
|
3042
|
+
check(raised is not None, "open must not swallow the refusal")
|
|
3043
|
+
check(speaker == ["closed"] and microphone == ["closed"],
|
|
3044
|
+
"open must release the devices it started: %r %r" % (speaker, microphone))
|
|
3045
|
+
|
|
3046
|
+
# A device that cannot be opened is one named line with a next step, not a
|
|
3047
|
+
# traceback, because this is the path the guide warns will fail first. The relay
|
|
3048
|
+
# side is stubbed out here so nothing is launched: the subject is the refusal.
|
|
3049
|
+
class Boom:
|
|
3050
|
+
def __init__(self, *args, **kwargs):
|
|
3051
|
+
raise RuntimeError("PortAudio said no")
|
|
3052
|
+
|
|
3053
|
+
class FakeProc:
|
|
3054
|
+
def __init__(self, *args, **kwargs):
|
|
3055
|
+
self.stdin = io.BytesIO()
|
|
3056
|
+
self.stdout = io.BytesIO(frame.MAGIC)
|
|
3057
|
+
|
|
3058
|
+
def wait(self, timeout=None):
|
|
3059
|
+
return 0
|
|
3060
|
+
|
|
3061
|
+
def kill(self):
|
|
3062
|
+
pass
|
|
3063
|
+
|
|
3064
|
+
real_speaker = client.SpeakerPlayback
|
|
3065
|
+
real_popen = client.subprocess.Popen
|
|
3066
|
+
real_sync = client.sync_magic
|
|
3067
|
+
client.SpeakerPlayback = Boom
|
|
3068
|
+
client.subprocess.Popen = FakeProc
|
|
3069
|
+
client.sync_magic = lambda stream, verbose=False: None
|
|
3070
|
+
|
|
3071
|
+
def refusal_for(argv):
|
|
3072
|
+
"""Return how open() refuses, as (exception type name, message)."""
|
|
3073
|
+
try:
|
|
3074
|
+
client.Client(client.parse_args(["--host", "desk"] + argv)).open()
|
|
3075
|
+
except BaseException as exc: # noqa: BLE001
|
|
3076
|
+
return type(exc).__name__, str(exc)
|
|
3077
|
+
return None, "open() did not refuse"
|
|
3078
|
+
|
|
3079
|
+
kind, named = refusal_for([])
|
|
3080
|
+
try:
|
|
3081
|
+
check(kind == "DeviceError",
|
|
3082
|
+
"a device failure should be a named refusal, got %s: %s" % (kind, named))
|
|
3083
|
+
check("could not open the audio device" in named,
|
|
3084
|
+
"and should say what it could not open: %s" % named)
|
|
3085
|
+
check("--output-device" in named,
|
|
3086
|
+
"and should name the flag for that end: %s" % named)
|
|
3087
|
+
check("--in-file" in named,
|
|
3088
|
+
"and should name a way to run without a device: %s" % named)
|
|
3089
|
+
|
|
3090
|
+
# The file ends are the ones this host runs and the ones every measured
|
|
3091
|
+
# figure was taken with, so a path that cannot be opened must say so and name
|
|
3092
|
+
# the flag that chose it. Calling it a device failure sends the reader to
|
|
3093
|
+
# --input-device when the thing to fix is the path.
|
|
3094
|
+
missing = os.path.join(TMP, "no-such-clip.pcm")
|
|
3095
|
+
kind, named = refusal_for(["--in-file", missing, "--out-file",
|
|
3096
|
+
os.path.join(TMP, "reply.pcm")])
|
|
3097
|
+
check(kind in ("OSError", "FileNotFoundError"),
|
|
3098
|
+
"a missing clip is not a device failure, got %s: %s" % (kind, named))
|
|
3099
|
+
check(missing in named, "the refusal must name the path: %s" % named)
|
|
3100
|
+
check("--in-file" in named and "--output-device" not in named,
|
|
3101
|
+
"and the flag that named it, and no device advice: %s" % named)
|
|
3102
|
+
|
|
3103
|
+
nowhere = os.path.join(TMP, "no-such-dir", "reply.pcm")
|
|
3104
|
+
kind, named = refusal_for(["--in-file", os.path.join(TMP, "clip.pcm"),
|
|
3105
|
+
"--out-file", nowhere])
|
|
3106
|
+
check(kind in ("OSError", "FileNotFoundError"),
|
|
3107
|
+
"an unwritable reply file is not a device failure either: %s" % named)
|
|
3108
|
+
check(nowhere in named and "--out-file" in named,
|
|
3109
|
+
"and must name the path and its flag: %s" % named)
|
|
3110
|
+
finally:
|
|
3111
|
+
client.SpeakerPlayback = real_speaker
|
|
3112
|
+
client.subprocess.Popen = real_popen
|
|
3113
|
+
client.sync_magic = real_sync
|
|
3114
|
+
|
|
3115
|
+
# A login shell banner is discarded with a warning naming it, not an error.
|
|
3116
|
+
noise = b"You have mail.\n"
|
|
3117
|
+
stream = io.BytesIO(noise + frame.MAGIC + frame.encode(frame.BYE))
|
|
3118
|
+
client.sync_magic(stream)
|
|
3119
|
+
check(frame.Reader(stream).read()[0] == frame.BYE,
|
|
3120
|
+
"the first frame after the handshake must still be readable")
|
|
3121
|
+
|
|
3122
|
+
# Junk with no handshake at all must be a named refusal rather than a hang.
|
|
3123
|
+
try:
|
|
3124
|
+
client.sync_magic(io.BytesIO(b"x" * (client.MAX_PREAMBLE + 64)))
|
|
3125
|
+
sys.exit("client: accepted a stream with no handshake")
|
|
3126
|
+
except frame.FrameError as exc:
|
|
3127
|
+
check("not fm-voice-relay.py" in str(exc),
|
|
3128
|
+
"the refusal should say what is on the far end: %s" % exc)
|
|
3129
|
+
|
|
3130
|
+
# A far end that dies before saying hello must say that, because the useful next
|
|
3131
|
+
# step is running the relay command by hand.
|
|
3132
|
+
try:
|
|
3133
|
+
client.sync_magic(io.BytesIO(b""))
|
|
3134
|
+
sys.exit("client: accepted a closed stream")
|
|
3135
|
+
except frame.FrameError as exc:
|
|
3136
|
+
check("before it said hello" in str(exc),
|
|
3137
|
+
"the refusal should name the early close: %s" % exc)
|
|
3138
|
+
|
|
3139
|
+
# The two ends must agree on the sample rates, or the reply plays at the wrong
|
|
3140
|
+
# pitch and nothing reports an error.
|
|
3141
|
+
relay_spec = importlib.util.spec_from_file_location(
|
|
3142
|
+
"relay", str(pathlib.Path(sys.argv[1]) / "fm-voice-relay.py"))
|
|
3143
|
+
relay = importlib.util.module_from_spec(relay_spec)
|
|
3144
|
+
relay_spec.loader.exec_module(relay)
|
|
3145
|
+
check((client.IN_RATE, client.OUT_RATE) == (relay.IN_RATE, relay.OUT_RATE),
|
|
3146
|
+
"the two ends disagree on the sample rates")
|
|
3147
|
+
PY
|
|
3148
|
+
pass "the laptop client builds the right remote command and survives a chatty login shell"
|
|
3149
|
+
|
|
3150
|
+
# docs/voice-relay.md tells the captain to copy exactly two files to the laptop,
|
|
3151
|
+
# so the real property is that the client runs from a directory holding exactly
|
|
3152
|
+
# those two and nothing else from bin/. A third local import would leave that
|
|
3153
|
+
# instruction wrong and the laptop dying at import time, a long way from the
|
|
3154
|
+
# change that caused it. Run there with no PYTHONPATH, so bin/ cannot supply the
|
|
3155
|
+
# missing piece the way it does on this host.
|
|
3156
|
+
LAPTOP="$TMP_ROOT/laptop"
|
|
3157
|
+
mkdir -p "$LAPTOP"
|
|
3158
|
+
cp "$ROOT/bin/fm-voice-client.py" "$ROOT/bin/fm_voice_frame.py" "$LAPTOP/"
|
|
3159
|
+
|
|
3160
|
+
set +e
|
|
3161
|
+
copied_out=$(cd "$LAPTOP" && env -u PYTHONPATH python3 ./fm-voice-client.py --help 2>&1)
|
|
3162
|
+
copied_code=$?
|
|
3163
|
+
set -e
|
|
3164
|
+
expect_code 0 "$copied_code" \
|
|
3165
|
+
"the client must start with only the two copied files: $copied_out"
|
|
3166
|
+
assert_contains "$copied_out" 'fm-voice-client.py' \
|
|
3167
|
+
"the copied client should print its own usage"
|
|
3168
|
+
|
|
3169
|
+
# The negative half, so the case above is not passing because bin/ was reachable
|
|
3170
|
+
# after all: without its one companion the client must fail at import and name it.
|
|
3171
|
+
SHORT="$TMP_ROOT/laptop-missing-companion"
|
|
3172
|
+
mkdir -p "$SHORT"
|
|
3173
|
+
cp "$ROOT/bin/fm-voice-client.py" "$SHORT/"
|
|
3174
|
+
set +e
|
|
3175
|
+
short_out=$(cd "$SHORT" && env -u PYTHONPATH python3 ./fm-voice-client.py --help 2>&1)
|
|
3176
|
+
short_code=$?
|
|
3177
|
+
set -e
|
|
3178
|
+
[ "$short_code" -ne 0 ] || fail "the client started without fm_voice_frame.py beside it"
|
|
3179
|
+
assert_contains "$short_out" 'fm_voice_frame' \
|
|
3180
|
+
"the import failure should name the file the laptop is missing"
|
|
3181
|
+
pass "the client runs from a laptop holding only the two files the guide names"
|
|
3182
|
+
|
|
3183
|
+
# --listen open-mic refuses, out loud and early. The mode has no way to tell when
|
|
3184
|
+
# the captain stopped speaking, so it would stream a turn that never ends; the
|
|
3185
|
+
# captain should be told that rather than watching it half work. Early matters as
|
|
3186
|
+
# much as loud: an ssh stub here records any attempt to reach the desktop, and the
|
|
3187
|
+
# refusal must come before it, so nothing is opened and nothing is spent.
|
|
3188
|
+
OPENMIC_FAKEBIN=$(fm_fakebin "$TMP_ROOT/openmic-fake")
|
|
3189
|
+
SSH_CALLED="$TMP_ROOT/ssh-was-called"
|
|
3190
|
+
cat > "$OPENMIC_FAKEBIN/ssh" <<SH
|
|
3191
|
+
#!/usr/bin/env bash
|
|
3192
|
+
printf 'ssh %s\n' "\$*" >> "$SSH_CALLED"
|
|
3193
|
+
exit 9
|
|
3194
|
+
SH
|
|
3195
|
+
chmod +x "$OPENMIC_FAKEBIN/ssh"
|
|
3196
|
+
|
|
3197
|
+
set +e
|
|
3198
|
+
openmic_out=$(PATH="$OPENMIC_FAKEBIN:$PATH" python3 "$ROOT/bin/fm-voice-client.py" \
|
|
3199
|
+
--host a-desktop --relay /desktop/bin/fm-voice-relay.py --listen open-mic 2>&1)
|
|
3200
|
+
openmic_code=$?
|
|
3201
|
+
set -e
|
|
3202
|
+
[ "$openmic_code" -ne 0 ] || fail "--listen open-mic started instead of refusing"
|
|
3203
|
+
assert_contains "$openmic_out" 'end-of-speech' \
|
|
3204
|
+
"the refusal should name the missing piece: $openmic_out"
|
|
3205
|
+
assert_contains "$openmic_out" 'push-to-talk' \
|
|
3206
|
+
"the refusal should name the mode that does work: $openmic_out"
|
|
3207
|
+
assert_absent "$SSH_CALLED" \
|
|
3208
|
+
"the refusal must come before anything reaches the desktop"
|
|
3209
|
+
# The default still starts far enough to try the connection, so the case above is
|
|
3210
|
+
# a property of the setting rather than of the fixture refusing everything.
|
|
3211
|
+
set +e
|
|
3212
|
+
PATH="$OPENMIC_FAKEBIN:$PATH" python3 "$ROOT/bin/fm-voice-client.py" \
|
|
3213
|
+
--host a-desktop --relay /desktop/bin/fm-voice-relay.py --talk-seconds 0 \
|
|
3214
|
+
>/dev/null 2>&1
|
|
3215
|
+
set -e
|
|
3216
|
+
assert_present "$SSH_CALLED" \
|
|
3217
|
+
"fixture is wrong: push to talk should have reached the ssh stub"
|
|
3218
|
+
pass "--listen open-mic refuses at startup, before it opens anything"
|
|
3219
|
+
|
|
3220
|
+
# --- read scope -------------------------------------------------------------
|
|
3221
|
+
|
|
3222
|
+
narrow=$(records_status --scope counts) || fail "counts scope failed"
|
|
3223
|
+
assert_contains "$narrow" '"scope": "counts"' "counts scope should say so"
|
|
3224
|
+
assert_contains "$narrow" '"in_flight": 3' "counts scope should still count in-flight work"
|
|
3225
|
+
assert_contains "$narrow" '"queued": 2' "counts scope should still count queued work"
|
|
3226
|
+
assert_contains "$narrow" '"awaiting_captain": 2' "counts scope should count what waits on the captain"
|
|
3227
|
+
assert_contains "$narrow" '"open_pull_requests": 1' "counts scope should count open pull requests"
|
|
3228
|
+
# No record free text is assembled at all at this scope, so there is nothing to
|
|
3229
|
+
# filter and nothing to get wrong.
|
|
3230
|
+
assert_not_contains "$narrow" 'alpha-one' "counts scope must not name work"
|
|
3231
|
+
assert_not_contains "$narrow" 'sign-in redirect' "counts scope must not carry titles"
|
|
3232
|
+
assert_not_contains "$narrow" 'github.com' "counts scope must not carry pull request links"
|
|
3233
|
+
pass "the narrow scope answers how much is waiting without saying what it is"
|
|
3234
|
+
|
|
3235
|
+
wide=$(records_status --scope full) || fail "full scope failed"
|
|
3236
|
+
assert_contains "$wide" '"scope": "full"' "full scope should say so"
|
|
3237
|
+
assert_contains "$wide" 'alpha-one' "full scope should name in-flight work"
|
|
3238
|
+
assert_contains "$wide" 'sign-in redirect' "full scope should carry titles"
|
|
3239
|
+
assert_contains "$wide" 'https://github.com/example/alpha/pull/7' \
|
|
3240
|
+
"full scope should carry the pull request link"
|
|
3241
|
+
assert_contains "$wide" 'beta-two' "full scope should name what waits on the captain"
|
|
3242
|
+
# The state verb only. The agent speaks to the captain and must not read an
|
|
3243
|
+
# internal event line aloud.
|
|
3244
|
+
assert_contains "$wide" '"state": "working"' "full scope should carry the state verb"
|
|
3245
|
+
assert_not_contains "$wide" 'reading the failing test' \
|
|
3246
|
+
"full scope must not carry the raw event line"
|
|
3247
|
+
# The same rule against the bracketed shape: the verb is still the verb, and the
|
|
3248
|
+
# metadata and the note stay unspoken.
|
|
3249
|
+
assert_contains "$wide" '"state": "blocked"' \
|
|
3250
|
+
"a status line with a metadata token before the colon should still report its verb"
|
|
3251
|
+
assert_not_contains "$wide" 'key=api-shape' \
|
|
3252
|
+
"full scope must not carry status metadata"
|
|
3253
|
+
assert_not_contains "$wide" 'needs a credential' \
|
|
3254
|
+
"full scope must not carry the raw event line of a bracketed status"
|
|
3255
|
+
pass "the wide scope names open work and reports state without quoting event lines"
|
|
3256
|
+
|
|
3257
|
+
# THE DEFAULT IS THE NARROW SCOPE. A home that has configured nothing has granted
|
|
3258
|
+
# nothing, and sending task identifiers, titles and pull request links to a model
|
|
3259
|
+
# in another region is not something to inherit from somebody else's settings
|
|
3260
|
+
# file. Widening is one line the captain of those records writes themselves.
|
|
3261
|
+
default=$(records_status) || fail "default scope failed"
|
|
3262
|
+
assert_contains "$default" '"scope": "counts"' \
|
|
3263
|
+
"an unconfigured home should get the narrow scope"
|
|
3264
|
+
assert_not_contains "$default" 'alpha-one' \
|
|
3265
|
+
"an unconfigured home must not name work"
|
|
3266
|
+
assert_not_contains "$default" 'sign-in redirect' \
|
|
3267
|
+
"an unconfigured home must not carry titles"
|
|
3268
|
+
assert_not_contains "$default" 'github.com' \
|
|
3269
|
+
"an unconfigured home must not carry pull request links"
|
|
3270
|
+
assert_contains "$default" '"in_flight": 3' \
|
|
3271
|
+
"an unconfigured home should still say how much is waiting"
|
|
3272
|
+
pass "an absent read-scope setting means the narrowest answer, not the widest"
|
|
3273
|
+
|
|
3274
|
+
# Widening is what the file is for, and it takes effect without a flag.
|
|
3275
|
+
printf 'full\n' > "$HOME_FIXTURE/config/voice-read-scope"
|
|
3276
|
+
widened=$(records_status) || fail "configured wide scope failed"
|
|
3277
|
+
assert_contains "$widened" '"scope": "full"' \
|
|
3278
|
+
"writing full into config/voice-read-scope should widen the answer"
|
|
3279
|
+
assert_contains "$widened" 'alpha-one' "the wide scope should then name work"
|
|
3280
|
+
rm -f "$HOME_FIXTURE/config/voice-read-scope"
|
|
3281
|
+
pass "a home widens its own read scope by writing the setting"
|
|
3282
|
+
|
|
3283
|
+
# --- the confidentiality boundary -------------------------------------------
|
|
3284
|
+
#
|
|
3285
|
+
# This is the case that lets a home widen to the full scope at all.
|
|
3286
|
+
|
|
3287
|
+
for scope in full counts; do
|
|
3288
|
+
answer=$(records_status --scope "$scope") || fail "scope $scope failed"
|
|
3289
|
+
assert_not_contains "$answer" "$NEVER_TOKEN" \
|
|
3290
|
+
"finished work and note bodies must never reach a $scope answer"
|
|
3291
|
+
assert_not_contains "$answer" 'old-six' \
|
|
3292
|
+
"finished work must not be named in a $scope answer"
|
|
3293
|
+
assert_not_contains "$answer" 'old-seven' \
|
|
3294
|
+
"an unticked line under finished work must not be named in a $scope answer"
|
|
3295
|
+
assert_not_contains "$answer" 'the rate we agreed' \
|
|
3296
|
+
"a note body must not reach a $scope answer"
|
|
3297
|
+
# The count is the assertion that bites if the section rule is lost: old-seven
|
|
3298
|
+
# is held for the captain and that list has no section filter of its own.
|
|
3299
|
+
assert_contains "$answer" '"awaiting_captain": 2' \
|
|
3300
|
+
"finished work must not be counted as waiting on the captain at $scope scope"
|
|
3301
|
+
done
|
|
3302
|
+
pass "finished work and note bodies never reach a spoken answer at any scope"
|
|
3303
|
+
|
|
3304
|
+
# The exclusion has to be structural rather than a filter on the way out, so the
|
|
3305
|
+
# count of in-flight work stays honest while the body stays unread.
|
|
3306
|
+
assert_contains "$wide" '"in_flight": 3' \
|
|
3307
|
+
"excluding note bodies must not change the count of in-flight work"
|
|
3308
|
+
pass "excluding a note body does not distort the counts"
|
|
3309
|
+
|
|
3310
|
+
# --- the deny list ----------------------------------------------------------
|
|
3311
|
+
#
|
|
3312
|
+
# Reachable first, suppressed second. Without the first assertion the second
|
|
3313
|
+
# proves nothing.
|
|
3314
|
+
|
|
3315
|
+
assert_contains "$wide" "$DENY_TOKEN" \
|
|
3316
|
+
"fixture is wrong: the deny marker should be reachable before it is denied"
|
|
3317
|
+
|
|
3318
|
+
printf '# one plain substring per line\n%s\n' "$DENY_TOKEN" \
|
|
3319
|
+
> "$HOME_FIXTURE/config/voice-read-deny"
|
|
3320
|
+
denied=$(records_status --scope full) || fail "full scope with a deny list failed"
|
|
3321
|
+
assert_not_contains "$denied" "$DENY_TOKEN" "the deny list must suppress a match"
|
|
3322
|
+
assert_not_contains "$denied" 'gamma-three' \
|
|
3323
|
+
"a denied item must not be named at all"
|
|
3324
|
+
assert_contains "$denied" '"withheld_as_confidential": 1' \
|
|
3325
|
+
"a denied item must still be counted so the captain knows it exists"
|
|
3326
|
+
assert_contains "$denied" '"in_flight": 3' \
|
|
3327
|
+
"denying an item must not change the count of in-flight work"
|
|
3328
|
+
# The other in-flight work is unaffected: this is a substring list, not a switch.
|
|
3329
|
+
assert_contains "$denied" 'alpha-one' "the deny list must not suppress everything"
|
|
3330
|
+
pass "a denied item becomes a withheld count without hiding that work exists"
|
|
3331
|
+
|
|
3332
|
+
# Case-insensitive, because a confidentiality list that depends on the captain
|
|
3333
|
+
# matching the file's capitalisation is a confidentiality list that fails quietly.
|
|
3334
|
+
printf '%s\n' "$(printf '%s' "$DENY_TOKEN" | tr '[:upper:]' '[:lower:]')" \
|
|
3335
|
+
> "$HOME_FIXTURE/config/voice-read-deny"
|
|
3336
|
+
lower=$(records_status --scope full) || fail "lowercase deny list failed"
|
|
3337
|
+
assert_not_contains "$lower" "$DENY_TOKEN" "the deny list must match regardless of case"
|
|
3338
|
+
pass "the deny list matches regardless of case"
|
|
3339
|
+
|
|
3340
|
+
# The withheld figure counts denied items, not refusals, and the lists overlap by
|
|
3341
|
+
# design: alpha-one is in flight AND carries a pull request, beta-two is in flight
|
|
3342
|
+
# AND waiting on the captain. Counting each refusal would tell the captain four
|
|
3343
|
+
# things are being withheld when two are, which is a wrong number spoken
|
|
3344
|
+
# confidently about exactly the subject the captain is most careful with.
|
|
3345
|
+
printf '%s\n%s\n' alpha-one beta-two > "$HOME_FIXTURE/config/voice-read-deny"
|
|
3346
|
+
overlap=$(records_status --scope full) || fail "overlapping deny list failed"
|
|
3347
|
+
assert_contains "$overlap" '"withheld_as_confidential": 2' \
|
|
3348
|
+
"two denied items appearing in two lists each must be withheld twice, not four times"
|
|
3349
|
+
assert_not_contains "$overlap" 'alpha-one' "a denied item must not be named"
|
|
3350
|
+
assert_not_contains "$overlap" 'beta-two' "a denied item must not be named"
|
|
3351
|
+
assert_not_contains "$overlap" 'github.com' \
|
|
3352
|
+
"denying an item must suppress its pull request link too"
|
|
3353
|
+
assert_contains "$overlap" '"in_flight": 3' \
|
|
3354
|
+
"denying items must not change the count of in-flight work"
|
|
3355
|
+
assert_contains "$overlap" '"open_pull_requests": 1' \
|
|
3356
|
+
"denying items must not change the count of open pull requests"
|
|
3357
|
+
pass "an item denied in more than one list is counted as withheld once"
|
|
3358
|
+
|
|
3359
|
+
rm -f "$HOME_FIXTURE/config/voice-read-deny"
|
|
3360
|
+
|
|
3361
|
+
# THE CASE THE DENY LIST EXISTS FOR, and the one a per-list decision gets wrong.
|
|
3362
|
+
# The docstring says the list is for a future open task carrying a customer name,
|
|
3363
|
+
# and a name like that lives in the TITLE or in the HOLD text of an item that is
|
|
3364
|
+
# also in flight, also waiting on the captain, and also carrying a pull request.
|
|
3365
|
+
# A decision taken separately in each list, from whichever fields that list
|
|
3366
|
+
# happens to use, withholds such an item from one list and names it in another.
|
|
3367
|
+
# That is not a narrower answer, it is a leak with a reassuring count beside it.
|
|
3368
|
+
# Both items below sit in all three lists, and each is matched on a field only
|
|
3369
|
+
# one of those lists reads.
|
|
3370
|
+
#
|
|
3371
|
+
# The third item is the one an in-flight-only fixture cannot catch: a QUEUED item
|
|
3372
|
+
# that nothing holds for the captain, so no list iterates it, while its pull
|
|
3373
|
+
# request link still reaches the answer through the worker records. Assembling its
|
|
3374
|
+
# fields only where some list walks past it misses a match on its own title.
|
|
3375
|
+
LEAK_HOME="$TMP_ROOT/deny-every-list"
|
|
3376
|
+
TITLE_TOKEN=LEAKSBYTITLE
|
|
3377
|
+
HOLD_TOKEN=LEAKSBYHOLD
|
|
3378
|
+
QUEUED_TOKEN=LEAKSFROMQUEUED
|
|
3379
|
+
mkdir -p "$LEAK_HOME/data" "$LEAK_HOME/state" "$LEAK_HOME/config"
|
|
3380
|
+
cat > "$LEAK_HOME/data/backlog.md" <<EOF
|
|
3381
|
+
# Backlog
|
|
3382
|
+
|
|
3383
|
+
## In flight
|
|
3384
|
+
- [ ] omega-nine - Renew the $TITLE_TOKEN contract (repo: omega) (kind: captain)
|
|
3385
|
+
- [ ] sigma-ten - Move the account onto the new tier (repo: sigma) (kind: ship) (hold-kind: captain) (hold: waiting on the $HOLD_TOKEN owner)
|
|
3386
|
+
|
|
3387
|
+
## Queued
|
|
3388
|
+
- [ ] zeta-eight - Migrate the $QUEUED_TOKEN estate (repo: zeta) (kind: ship)
|
|
3389
|
+
EOF
|
|
3390
|
+
fm_write_meta "$LEAK_HOME/state/omega-nine.meta" \
|
|
3391
|
+
kind=captain pr=https://github.com/example/omega/pull/11
|
|
3392
|
+
fm_write_meta "$LEAK_HOME/state/sigma-ten.meta" \
|
|
3393
|
+
kind=ship pr=https://github.com/example/sigma/pull/12
|
|
3394
|
+
fm_write_meta "$LEAK_HOME/state/zeta-eight.meta" \
|
|
3395
|
+
kind=ship pr=https://github.com/example/zeta/pull/99
|
|
3396
|
+
|
|
3397
|
+
leak_status() {
|
|
3398
|
+
python3 "$ROOT/bin/fm_voice_records.py" status --home "$LEAK_HOME" --scope full
|
|
3399
|
+
}
|
|
3400
|
+
|
|
3401
|
+
# Reachable in all three lists first, or the suppression below proves nothing.
|
|
3402
|
+
reachable=$(leak_status) || fail "the deny-every-list fixture failed"
|
|
3403
|
+
assert_contains "$reachable" "$TITLE_TOKEN" "fixture: the title marker should be reachable"
|
|
3404
|
+
assert_contains "$reachable" 'omega-nine' "fixture: the item should be named"
|
|
3405
|
+
assert_contains "$reachable" 'pull/11' "fixture: its pull request should be reachable"
|
|
3406
|
+
assert_contains "$reachable" 'sigma-ten' "fixture: the held item should be named"
|
|
3407
|
+
assert_contains "$reachable" 'pull/12' "fixture: its pull request should be reachable"
|
|
3408
|
+
assert_contains "$reachable" '"awaiting_captain": 2' \
|
|
3409
|
+
"fixture: both in-flight items should be waiting on the captain"
|
|
3410
|
+
assert_contains "$reachable" 'pull/99' \
|
|
3411
|
+
"fixture: the queued item should reach the answer through its pull request"
|
|
3412
|
+
assert_contains "$reachable" '"queued": 1' "fixture: the queued item should be counted"
|
|
3413
|
+
|
|
3414
|
+
# Matched on its title, which only the in-flight list reads.
|
|
3415
|
+
printf '%s\n' "$TITLE_TOKEN" > "$LEAK_HOME/config/voice-read-deny"
|
|
3416
|
+
by_title=$(leak_status) || fail "deny by title failed"
|
|
3417
|
+
assert_not_contains "$by_title" "$TITLE_TOKEN" "a title match must be suppressed"
|
|
3418
|
+
assert_not_contains "$by_title" 'omega-nine' \
|
|
3419
|
+
"a denied item must not be named in any list"
|
|
3420
|
+
assert_not_contains "$by_title" 'pull/11' \
|
|
3421
|
+
"a denied item must not surface through its pull request link"
|
|
3422
|
+
assert_contains "$by_title" '"withheld_as_confidential": 1' \
|
|
3423
|
+
"the denied item should be counted once"
|
|
3424
|
+
# The other items are untouched, so this is a substring list and not a switch.
|
|
3425
|
+
assert_contains "$by_title" 'sigma-ten' "the deny list must not suppress everything"
|
|
3426
|
+
assert_contains "$by_title" 'pull/12' "the other pull requests should still be named"
|
|
3427
|
+
assert_contains "$by_title" 'pull/99' "the other pull requests should still be named"
|
|
3428
|
+
assert_contains "$by_title" '"open_pull_requests": 3' \
|
|
3429
|
+
"denying an item must not change the count of open pull requests"
|
|
3430
|
+
|
|
3431
|
+
# Matched on its hold text, which only the captain list reads. The mirror of the
|
|
3432
|
+
# case above: get one list right and this one still leaks.
|
|
3433
|
+
printf '%s\n' "$HOLD_TOKEN" > "$LEAK_HOME/config/voice-read-deny"
|
|
3434
|
+
by_hold=$(leak_status) || fail "deny by hold text failed"
|
|
3435
|
+
assert_not_contains "$by_hold" 'sigma-ten' \
|
|
3436
|
+
"an item matched on its hold text must not be named in the in-flight list"
|
|
3437
|
+
assert_not_contains "$by_hold" 'pull/12' \
|
|
3438
|
+
"an item matched on its hold text must not surface through its pull request"
|
|
3439
|
+
assert_contains "$by_hold" '"withheld_as_confidential": 1' \
|
|
3440
|
+
"the denied item should be counted once"
|
|
3441
|
+
assert_contains "$by_hold" 'omega-nine' "the deny list must not suppress everything"
|
|
3442
|
+
assert_contains "$by_hold" 'pull/11' "the other pull request should still be named"
|
|
3443
|
+
assert_contains "$by_hold" '"in_flight": 2' \
|
|
3444
|
+
"denying an item must not change the count of in-flight work"
|
|
3445
|
+
|
|
3446
|
+
# Matched on the title of a QUEUED item that no list iterates. Its only way into
|
|
3447
|
+
# the answer is its pull request link, and the pull request list knows nothing
|
|
3448
|
+
# about titles, so a field set assembled per list never sees the match at all.
|
|
3449
|
+
printf '%s\n' "$QUEUED_TOKEN" > "$LEAK_HOME/config/voice-read-deny"
|
|
3450
|
+
by_queued=$(leak_status) || fail "deny by queued title failed"
|
|
3451
|
+
assert_not_contains "$by_queued" "$QUEUED_TOKEN" \
|
|
3452
|
+
"a queued item's title match must be suppressed"
|
|
3453
|
+
assert_not_contains "$by_queued" 'zeta-eight' \
|
|
3454
|
+
"a denied queued item must not be named"
|
|
3455
|
+
assert_not_contains "$by_queued" 'pull/99' \
|
|
3456
|
+
"a denied queued item must not surface through its pull request link"
|
|
3457
|
+
assert_contains "$by_queued" '"withheld_as_confidential": 1' \
|
|
3458
|
+
"a denied queued item must be counted, so nothing is hidden silently"
|
|
3459
|
+
assert_contains "$by_queued" '"queued": 1' \
|
|
3460
|
+
"denying it must not change the count of queued work"
|
|
3461
|
+
assert_contains "$by_queued" 'pull/11' "the other pull requests should still be named"
|
|
3462
|
+
assert_contains "$by_queued" 'pull/12' "the other pull requests should still be named"
|
|
3463
|
+
pass "one deny decision per item covers every list that item could appear in"
|
|
3464
|
+
|
|
3465
|
+
# --- what a status line may say ---------------------------------------------
|
|
3466
|
+
#
|
|
3467
|
+
# A status line is free text a crewmate appended, and the verb taken off the
|
|
3468
|
+
# front of it is the ONE record-derived string a counts-scope answer says out
|
|
3469
|
+
# loud. At that scope there is no title and no link, so there is nothing for the
|
|
3470
|
+
# deny list to filter and no scope setting that makes it safe. The vocabulary is
|
|
3471
|
+
# therefore closed to the states bin/fm-brief.sh gives every crewmate plus the two
|
|
3472
|
+
# bin/fm-classify-lib.sh adds when a decision closes, and anything else is a note.
|
|
3473
|
+
VERB_HOME="$TMP_ROOT/status-verbs"
|
|
3474
|
+
# Lowercase on purpose. The reader lowercases a verb before it could ever be
|
|
3475
|
+
# emitted, and assert_not_contains compares case-sensitively, so an uppercase
|
|
3476
|
+
# marker here would make the assertion below unable to fail on leaking code.
|
|
3477
|
+
CUSTOMER_TOKEN=acmecorpmigration
|
|
3478
|
+
mkdir -p "$VERB_HOME/data" "$VERB_HOME/state"
|
|
3479
|
+
cat > "$VERB_HOME/data/backlog.md" <<'EOF'
|
|
3480
|
+
# Backlog
|
|
3481
|
+
|
|
3482
|
+
## In flight
|
|
3483
|
+
- [ ] one - First thing (repo: a) (kind: ship)
|
|
3484
|
+
- [ ] two - Second thing (repo: b) (kind: ship)
|
|
3485
|
+
- [ ] three - Third thing (repo: c) (kind: ship)
|
|
3486
|
+
EOF
|
|
3487
|
+
fm_write_meta "$VERB_HOME/state/one.meta" kind=ship
|
|
3488
|
+
fm_write_meta "$VERB_HOME/state/two.meta" kind=ship
|
|
3489
|
+
fm_write_meta "$VERB_HOME/state/three.meta" kind=ship
|
|
3490
|
+
printf 'needs-decision [key=shape]: which shape\n' > "$VERB_HOME/state/one.status"
|
|
3491
|
+
printf '%s: waiting on their security review\n' "$CUSTOMER_TOKEN" \
|
|
3492
|
+
> "$VERB_HOME/state/two.status"
|
|
3493
|
+
# A log past the tail window, so the read is proven to end at the last line
|
|
3494
|
+
# rather than at the start of whatever window it happened to open.
|
|
3495
|
+
{
|
|
3496
|
+
verb_line=0
|
|
3497
|
+
while [ "$verb_line" -lt 400 ]; do
|
|
3498
|
+
printf 'working: step %s of a long task with a wordy status line\n' "$verb_line"
|
|
3499
|
+
verb_line=$((verb_line + 1))
|
|
3500
|
+
done
|
|
3501
|
+
printf 'done: shipped it\n'
|
|
3502
|
+
} > "$VERB_HOME/state/three.status"
|
|
3503
|
+
[ "$(wc -c < "$VERB_HOME/state/three.status")" -gt 8192 ] \
|
|
3504
|
+
|| fail "fixture: the long status log should exceed the tail window"
|
|
3505
|
+
|
|
3506
|
+
verb_status() {
|
|
3507
|
+
python3 "$ROOT/bin/fm_voice_records.py" status --home "$VERB_HOME" "$@"
|
|
3508
|
+
}
|
|
3509
|
+
|
|
3510
|
+
verbs=$(verb_status --scope counts) || fail "counts scope with odd verbs failed"
|
|
3511
|
+
assert_not_contains "$verbs" "$CUSTOMER_TOKEN" \
|
|
3512
|
+
"a word outside the vocabulary must not be spoken, at the default scope least of all"
|
|
3513
|
+
assert_contains "$verbs" '"note": 1' \
|
|
3514
|
+
"an unrecognised verb should be counted as a note instead"
|
|
3515
|
+
assert_contains "$verbs" '"needs-decision": 1' \
|
|
3516
|
+
"a canonical verb, brackets and all, should survive the fold"
|
|
3517
|
+
assert_contains "$verbs" '"done": 1' \
|
|
3518
|
+
"the last line of a long log is the line that counts"
|
|
3519
|
+
assert_not_contains "$verbs" '"working"' \
|
|
3520
|
+
"an earlier line in the same log must not be reported as the state"
|
|
3521
|
+
pass "the state verb is a closed vocabulary, so free text cannot ride out on it"
|
|
3522
|
+
|
|
3523
|
+
# The two halves of one answer must come from one home. Every script that sets
|
|
3524
|
+
# FM_DATA_OVERRIDE sets FM_STATE_OVERRIDE beside it, so a reader that resolved one
|
|
3525
|
+
# and not the other would count workers and notes from one home while counting
|
|
3526
|
+
# in-flight work from another, which reads exactly like an ordinary answer.
|
|
3527
|
+
alt_data="$TMP_ROOT/data-elsewhere"
|
|
3528
|
+
mkdir -p "$alt_data"
|
|
3529
|
+
cat > "$alt_data/backlog.md" <<'EOF'
|
|
3530
|
+
# Backlog
|
|
3531
|
+
|
|
3532
|
+
## In flight
|
|
3533
|
+
- [ ] moved-one - Work recorded in the overridden data directory (repo: m) (kind: ship)
|
|
3534
|
+
EOF
|
|
3535
|
+
moved=$(FM_DATA_OVERRIDE="$alt_data" verb_status --scope full) \
|
|
3536
|
+
|| fail "status with an overridden data directory failed"
|
|
3537
|
+
assert_contains "$moved" 'moved-one' \
|
|
3538
|
+
"the reader must take the backlog from the overridden data directory"
|
|
3539
|
+
assert_contains "$moved" '"in_flight": 1' "and count only what that backlog holds"
|
|
3540
|
+
inbox_moved=$(FM_HOME="$VERB_HOME" FM_STATE_OVERRIDE="$VERB_HOME/state" \
|
|
3541
|
+
FM_DATA_OVERRIDE="$alt_data" "$ROOT/bin/fm-inbox.sh" status) \
|
|
3542
|
+
|| fail "fm-inbox status with an overridden data directory failed"
|
|
3543
|
+
assert_contains "$inbox_moved" 'moved-one' \
|
|
3544
|
+
"the human rendering of the same records must read the same backlog"
|
|
3545
|
+
pass "the backlog and the state directory always come from the same home"
|
|
3546
|
+
|
|
3547
|
+
# --- pull requests on finished work -----------------------------------------
|
|
3548
|
+
#
|
|
3549
|
+
# A task keeps its state/<id>.meta after its backlog item is marked done, because
|
|
3550
|
+
# removing the record and moving the item are separate steps. So a reader that took
|
|
3551
|
+
# every worker carrying a pull request would count and name finished work, which
|
|
3552
|
+
# this module promises never to read. Worse, the deny list could not reach those
|
|
3553
|
+
# items: with no open item there is no title in the field set, so a captain
|
|
3554
|
+
# substring matching the title silently failed for exactly them while working
|
|
3555
|
+
# everywhere else. Losing the count of a pull request on a finished task is the
|
|
3556
|
+
# accepted cost of that control applying everywhere it appears to.
|
|
3557
|
+
DONE_HOME="$TMP_ROOT/finished-pull-requests"
|
|
3558
|
+
FINISHED_TOKEN=SHIPPEDLASTWEEK
|
|
3559
|
+
mkdir -p "$DONE_HOME/data" "$DONE_HOME/state" "$DONE_HOME/config"
|
|
3560
|
+
cat > "$DONE_HOME/data/backlog.md" <<EOF
|
|
3561
|
+
# Backlog
|
|
3562
|
+
|
|
3563
|
+
## In flight
|
|
3564
|
+
- [ ] still-open - Fix the retry (repo: a) (kind: ship)
|
|
3565
|
+
- [x] ticked-two - Renew the $FINISHED_TOKEN contract (repo: b) (kind: ship)
|
|
3566
|
+
|
|
3567
|
+
## Done
|
|
3568
|
+
- [x] older-three - Migrate the $FINISHED_TOKEN estate (repo: c) (kind: ship)
|
|
3569
|
+
EOF
|
|
3570
|
+
fm_write_meta "$DONE_HOME/state/still-open.meta" \
|
|
3571
|
+
kind=ship pr=https://github.com/example/a/pull/1
|
|
3572
|
+
fm_write_meta "$DONE_HOME/state/ticked-two.meta" \
|
|
3573
|
+
kind=ship pr=https://github.com/example/b/pull/2
|
|
3574
|
+
fm_write_meta "$DONE_HOME/state/older-three.meta" \
|
|
3575
|
+
kind=ship pr=https://github.com/example/c/pull/3
|
|
3576
|
+
|
|
3577
|
+
done_status() {
|
|
3578
|
+
python3 "$ROOT/bin/fm_voice_records.py" status --home "$DONE_HOME" --scope full
|
|
3579
|
+
}
|
|
3580
|
+
|
|
3581
|
+
open_only=$(done_status) || fail "the finished-pull-request fixture failed"
|
|
3582
|
+
assert_contains "$open_only" '"open_pull_requests": 1' \
|
|
3583
|
+
"only open work has an open pull request"
|
|
3584
|
+
assert_contains "$open_only" 'pull/1' "the open task's pull request should be named"
|
|
3585
|
+
assert_not_contains "$open_only" 'ticked-two' \
|
|
3586
|
+
"a ticked item must not be named through its pull request"
|
|
3587
|
+
assert_not_contains "$open_only" 'pull/2' \
|
|
3588
|
+
"a ticked item's pull request must not be named"
|
|
3589
|
+
assert_not_contains "$open_only" 'older-three' \
|
|
3590
|
+
"an item under Done must not be named through its pull request"
|
|
3591
|
+
assert_not_contains "$open_only" 'pull/3' \
|
|
3592
|
+
"an item under Done must not have its pull request named"
|
|
3593
|
+
assert_not_contains "$open_only" "$FINISHED_TOKEN" \
|
|
3594
|
+
"no finished title may reach the answer at any scope"
|
|
3595
|
+
# Excluded by construction, not withheld and counted. A later change that put
|
|
3596
|
+
# finished work back in and leaned on the deny list to hide it would fail here.
|
|
3597
|
+
assert_contains "$open_only" '"withheld_as_confidential": 0' \
|
|
3598
|
+
"finished work is left out rather than counted as withheld"
|
|
3599
|
+
# The worker count is deliberately NOT open-only: a task keeps its runtime record
|
|
3600
|
+
# until teardown removes it, and that record is what "on deck" counts. Asserted in
|
|
3601
|
+
# the same case as the pull request count so the two cannot quietly converge.
|
|
3602
|
+
assert_contains "$open_only" '"workers_on_deck": 3' \
|
|
3603
|
+
"every live runtime record is still on deck, finished or not"
|
|
3604
|
+
pass "a finished task's pull request is neither counted nor named"
|
|
3605
|
+
|
|
3606
|
+
# The deny list, observed doing its job on the one list that still carries links.
|
|
3607
|
+
# A substring matching an OPEN task's title takes that task out of the pull
|
|
3608
|
+
# request detail and says one thing is being withheld, while the count stays
|
|
3609
|
+
# honest: that split is the contract this module states and the earlier cases
|
|
3610
|
+
# pin, so the captain learns how much is waiting without learning what it is.
|
|
3611
|
+
printf '%s\n' 'Fix the retry' > "$DONE_HOME/config/voice-read-deny"
|
|
3612
|
+
denied_open=$(done_status) || fail "deny by an open title failed"
|
|
3613
|
+
assert_not_contains "$denied_open" 'still-open' \
|
|
3614
|
+
"a denied open task must not be named in the pull request detail"
|
|
3615
|
+
assert_not_contains "$denied_open" 'pull/1' \
|
|
3616
|
+
"a denied open task's pull request link must go with it"
|
|
3617
|
+
assert_contains "$denied_open" '"withheld_as_confidential": 1' \
|
|
3618
|
+
"and the captain must be told one thing is being withheld"
|
|
3619
|
+
assert_contains "$denied_open" '"open_pull_requests": 1' \
|
|
3620
|
+
"while the count of open pull requests stays honest"
|
|
3621
|
+
rm -f "$DONE_HOME/config/voice-read-deny"
|
|
3622
|
+
pass "a deny substring on an open title removes its pull request and says so"
|
|
3623
|
+
|
|
3624
|
+
# --- refusals ---------------------------------------------------------------
|
|
3625
|
+
#
|
|
3626
|
+
# A misconfigured read scope must stop rather than fall back to the wider one,
|
|
3627
|
+
# because falling back would widen what is sent on the strength of a typo.
|
|
3628
|
+
|
|
3629
|
+
printf 'everything\n' > "$HOME_FIXTURE/config/voice-read-scope"
|
|
3630
|
+
set +e
|
|
3631
|
+
out=$(records_status 2>&1)
|
|
3632
|
+
code=$?
|
|
3633
|
+
set -e
|
|
3634
|
+
expect_code 2 "$code" "an unknown read scope should refuse"
|
|
3635
|
+
assert_contains "$out" 'voice-read-scope' "the refusal should name the setting"
|
|
3636
|
+
pass "an unknown read scope refuses instead of widening"
|
|
3637
|
+
|
|
3638
|
+
printf 'counts\n' > "$HOME_FIXTURE/config/voice-read-scope"
|
|
3639
|
+
configured=$(records_status) || fail "configured scope failed"
|
|
3640
|
+
assert_contains "$configured" '"scope": "counts"' "the configured scope should be used"
|
|
3641
|
+
rm -f "$HOME_FIXTURE/config/voice-read-scope"
|
|
3642
|
+
pass "the configured read scope is honoured"
|
|
3643
|
+
|
|
3644
|
+
# --- handover ---------------------------------------------------------------
|
|
3645
|
+
#
|
|
3646
|
+
# The point of the boundary: real work is queued for firstmate, not done by the
|
|
3647
|
+
# voice agent. It reuses bin/fm-inbox.sh rather than carrying a second queue.
|
|
3648
|
+
|
|
3649
|
+
before=$(find "$HOME_FIXTURE/state" -maxdepth 2 -name '*.note' | wc -l | tr -d '[:space:]')
|
|
3650
|
+
[ "$before" = 0 ] || fail "fixture should start with an empty inbox"
|
|
3651
|
+
|
|
3652
|
+
handed=$(FM_HOME="$HOME_FIXTURE" python3 "$ROOT/bin/fm_voice_records.py" queue \
|
|
3653
|
+
"Refactor the login module and open a pull request for it" \
|
|
3654
|
+
--home "$HOME_FIXTURE") || fail "handover failed"
|
|
3655
|
+
assert_contains "$handed" '"queued": true' "handover should report the request queued"
|
|
3656
|
+
assert_contains "$handed" 'did not do the work yourself' \
|
|
3657
|
+
"handover should tell the model it handed over rather than acted"
|
|
3658
|
+
|
|
3659
|
+
notes=$(find "$HOME_FIXTURE/state/inbox" -maxdepth 1 -name '*.note' | wc -l | tr -d '[:space:]')
|
|
3660
|
+
[ "$notes" = 1 ] || fail "handover should leave exactly one note, found $notes"
|
|
3661
|
+
note_file=$(find "$HOME_FIXTURE/state/inbox" -maxdepth 1 -name '*.note' | head -1)
|
|
3662
|
+
assert_grep 'Refactor the login module' "$note_file" \
|
|
3663
|
+
"the note should carry the captain's words"
|
|
3664
|
+
|
|
3665
|
+
# Exactly one wake, so a spoken request is presented once at firstmate's next
|
|
3666
|
+
# check rather than queued twice or lost.
|
|
3667
|
+
assert_present "$HOME_FIXTURE/state/.wake-queue" \
|
|
3668
|
+
"handover should wake firstmate"
|
|
3669
|
+
wakes=$(grep -c 'inbox:' "$HOME_FIXTURE/state/.wake-queue")
|
|
3670
|
+
[ "$wakes" = 1 ] || fail "handover should append exactly one wake, found $wakes"
|
|
3671
|
+
|
|
3672
|
+
# The reading half must see what the queueing half just wrote, or the agent says
|
|
3673
|
+
# the request is queued and then, asked what is waiting, says nothing is.
|
|
3674
|
+
paired=$(records_status --scope counts) || fail "status after a handover failed"
|
|
3675
|
+
assert_contains "$paired" '"captain_notes_waiting": 1' \
|
|
3676
|
+
"the reader should count the note the handover just queued"
|
|
3677
|
+
pass "handover queues the request for firstmate and wakes it exactly once"
|
|
3678
|
+
|
|
3679
|
+
# The same pairing when the state directory is moved. bin/fm-inbox.sh resolves
|
|
3680
|
+
# ${FM_STATE_OVERRIDE:-$FM_HOME/state} and the handover queues through it with
|
|
3681
|
+
# the ambient environment, so a reader that ignored the override would count
|
|
3682
|
+
# notes in a directory nothing writes to.
|
|
3683
|
+
alt_state="$TMP_ROOT/state-elsewhere"
|
|
3684
|
+
alt_home="$TMP_ROOT/override-home"
|
|
3685
|
+
mkdir -p "$alt_state" "$alt_home/data" "$alt_home/state"
|
|
3686
|
+
FM_STATE_OVERRIDE="$alt_state" python3 "$ROOT/bin/fm_voice_records.py" queue \
|
|
3687
|
+
"Chase the flaky retry test" --home "$alt_home" >/dev/null \
|
|
3688
|
+
|| fail "handover with an overridden state directory failed"
|
|
3689
|
+
|
|
3690
|
+
moved=$(find "$alt_state/inbox" -maxdepth 1 -name '*.note' | wc -l | tr -d '[:space:]')
|
|
3691
|
+
[ "$moved" = 1 ] || \
|
|
3692
|
+
fail "the queue should write into the overridden state directory, found $moved"
|
|
3693
|
+
[ ! -e "$alt_home/state/inbox" ] || \
|
|
3694
|
+
fail "the queue should not have written under the home when the state is moved"
|
|
3695
|
+
|
|
3696
|
+
overridden=$(FM_STATE_OVERRIDE="$alt_state" python3 \
|
|
3697
|
+
"$ROOT/bin/fm_voice_records.py" status --home "$alt_home") \
|
|
3698
|
+
|| fail "status with an overridden state directory failed"
|
|
3699
|
+
assert_contains "$overridden" '"captain_notes_waiting": 1' \
|
|
3700
|
+
"the reader must count notes where the queue actually wrote them"
|
|
3701
|
+
pass "the reader and the queue resolve the state directory the same way"
|
|
3702
|
+
|
|
3703
|
+
set +e
|
|
3704
|
+
empty_out=$(python3 "$ROOT/bin/fm_voice_records.py" queue " " \
|
|
3705
|
+
--home "$HOME_FIXTURE" 2>&1)
|
|
3706
|
+
empty_code=$?
|
|
3707
|
+
set -e
|
|
3708
|
+
expect_code 2 "$empty_code" "queueing empty text should refuse"
|
|
3709
|
+
assert_contains "$empty_out" 'empty' "the refusal should say the request was empty"
|
|
3710
|
+
pass "an empty request is refused rather than queued as a blank note"
|
|
3711
|
+
|
|
3712
|
+
# --- absent records ---------------------------------------------------------
|
|
3713
|
+
#
|
|
3714
|
+
# A home with no records at all must answer "nothing" rather than fail, because
|
|
3715
|
+
# the agent is spoken to and an exception is not an answer.
|
|
3716
|
+
|
|
3717
|
+
bare="$TMP_ROOT/bare"
|
|
3718
|
+
mkdir -p "$bare"
|
|
3719
|
+
bare_out=$(python3 "$ROOT/bin/fm_voice_records.py" status --home "$bare") \
|
|
3720
|
+
|| fail "an empty home should still answer"
|
|
3721
|
+
assert_contains "$bare_out" '"in_flight": 0' "an empty home should report no work"
|
|
3722
|
+
assert_contains "$bare_out" '"workers_on_deck": 0' "an empty home should report no workers"
|
|
3723
|
+
pass "a home with no records answers nothing rather than failing"
|
|
3724
|
+
|
|
3725
|
+
# --- the whole round trip ----------------------------------------------------
|
|
3726
|
+
#
|
|
3727
|
+
# Every case above holds one piece of the spoken interface still. This one runs
|
|
3728
|
+
# the piece the captain experiences: the laptop client opens the transport, the
|
|
3729
|
+
# relay answers a spoken question from the records and hands a spoken request for
|
|
3730
|
+
# real work to firstmate, and the reply audio and the timing come back down the
|
|
3731
|
+
# same stream. It is the only case that would notice the round trip stopping
|
|
3732
|
+
# working while all of the pieces still passed.
|
|
3733
|
+
#
|
|
3734
|
+
# ONE thing is stood in for: the model. It is a paid service in another region
|
|
3735
|
+
# and no test has a credential for it. The stand-in below speaks the same event
|
|
3736
|
+
# protocol Nova Sonic does and composes what it says out of the tool results the
|
|
3737
|
+
# relay actually hands it, so the words asserted here are the records rather than
|
|
3738
|
+
# a script, and it records what the session was opened with so this case can
|
|
3739
|
+
# check the account and the model the relay chose. Everything else is real: the
|
|
3740
|
+
# client, the frame format, the relay, the reader and bin/fm-inbox.sh.
|
|
3741
|
+
#
|
|
3742
|
+
# What only this case can hold:
|
|
3743
|
+
# the round trip completes at all, in both of its shapes, a status answer and a
|
|
3744
|
+
# handover, and a second turn is not treated as an interruption of the first;
|
|
3745
|
+
# the headline figure is measured from the captain's talk end rather than from
|
|
3746
|
+
# the start of their speech, which on this clip is the difference between half
|
|
3747
|
+
# a second and two and a half;
|
|
3748
|
+
# the talk-end silence padding really is sent, which is trap 2 and the
|
|
3749
|
+
# difference between an answer and no answer;
|
|
3750
|
+
# the laptop needs no AWS credential: the client runs with an environment that
|
|
3751
|
+
# has none, and the session is opened with the key only the desktop side holds.
|
|
3752
|
+
|
|
3753
|
+
E2E="$TMP_ROOT/e2e"
|
|
3754
|
+
E2E_KEY=AKIADESKTOPONLYEXAMPLE
|
|
3755
|
+
E2E_REGION=eu-north-1
|
|
3756
|
+
E2E_MODEL=amazon.nova-2-sonic-v1:0
|
|
3757
|
+
E2E_REQUEST="take the flaky sign-in test on alpha and open a pull request for it"
|
|
3758
|
+
mkdir -p "$E2E/bin" "$E2E/laptop" "$E2E/desktop-home" "$E2E/laptop-home" \
|
|
3759
|
+
"$E2E/fakesdk/aws_sdk_bedrock_runtime" \
|
|
3760
|
+
"$E2E/home/data" "$E2E/home/state" "$E2E/home/config"
|
|
3761
|
+
|
|
3762
|
+
# The laptop holds the two files the guide says to copy, and nothing else.
|
|
3763
|
+
cp "$ROOT/bin/fm-voice-client.py" "$ROOT/bin/fm_voice_frame.py" "$E2E/laptop/"
|
|
3764
|
+
|
|
3765
|
+
cat > "$E2E/home/data/backlog.md" <<EOF
|
|
3766
|
+
# Backlog
|
|
3767
|
+
|
|
3768
|
+
## In flight
|
|
3769
|
+
- [ ] alpha-one - Fix the sign-in redirect (repo: alpha) (kind: ship) (priority: 0)
|
|
3770
|
+
A note body, which is never assembled: $NEVER_TOKEN and the rate we agreed.
|
|
3771
|
+
- [ ] beta-two - Decide the storage shape (repo: beta) (kind: captain)
|
|
3772
|
+
|
|
3773
|
+
## Queued
|
|
3774
|
+
- [ ] delta-four - Add the retry (repo: delta) (kind: ship) (hold-kind: captain)
|
|
3775
|
+
|
|
3776
|
+
## Done
|
|
3777
|
+
- [x] old-six - Shipped the $NEVER_TOKEN integration (repo: alpha) (done 2026-07-01)
|
|
3778
|
+
EOF
|
|
3779
|
+
fm_write_meta "$E2E/home/state/alpha-one.meta" kind=ship mode=no-mistakes \
|
|
3780
|
+
pr=https://github.com/example/alpha/pull/7
|
|
3781
|
+
printf 'working: reading the failing test\n' > "$E2E/home/state/alpha-one.status"
|
|
3782
|
+
printf '%s\n' "$E2E_REGION" > "$E2E/home/config/voice-region"
|
|
3783
|
+
printf '%s\n' "$E2E_MODEL" > "$E2E/home/config/voice-model"
|
|
3784
|
+
printf 'full\n' > "$E2E/home/config/voice-read-scope"
|
|
3785
|
+
|
|
3786
|
+
# The model stand-in, at exactly the import boundary bin/fm-voice-relay.py uses.
|
|
3787
|
+
cat > "$E2E/fakesdk/aws_sdk_bedrock_runtime/__init__.py" <<'PY'
|
|
3788
|
+
"""A scripted stand-in for Nova Sonic's bidirectional stream.
|
|
3789
|
+
|
|
3790
|
+
It answers with what the relay's own tool results contain, so a spoken answer
|
|
3791
|
+
here is derived from firstmate's records rather than from a fixture string, and
|
|
3792
|
+
it appends one JSON line per session describing what that session was opened
|
|
3793
|
+
with and what it was asked. tests/fm-voice-relay.test.sh reads that record.
|
|
3794
|
+
|
|
3795
|
+
FM_FAKE_SCRIPT comma-separated turn kinds: status | handover | clean-end
|
|
3796
|
+
FM_FAKE_THINK seconds before the reply begins, standing in for the model
|
|
3797
|
+
FM_FAKE_STATE file holding the turn counter across the relay's reconnects
|
|
3798
|
+
FM_FAKE_LOG where to append the per-session record
|
|
3799
|
+
FM_FAKE_REQUEST the words the captain uses when asking for real work
|
|
3800
|
+
FM_FAKE_EARLY 1 to answer from the first audio in, not from the talk end
|
|
3801
|
+
|
|
3802
|
+
A clean-end turn is a session the model finishes with while the captain is still
|
|
3803
|
+
speaking: the output stream simply ends, with no error and no answer. That is an
|
|
3804
|
+
ordinary end of a Bedrock session rather than a fault, and the relay has to
|
|
3805
|
+
survive it, so it is a turn kind here rather than a failure injection.
|
|
3806
|
+
|
|
3807
|
+
FM_FAKE_EARLY stands in for the model's own end-of-speech detector firing inside
|
|
3808
|
+
a clip that already ends in silence: the answer begins before this end of the
|
|
3809
|
+
stream has said the turn is over. Nova Sonic really does that, and the relay's
|
|
3810
|
+
own timing figures are negative when it happens, which is the one case where a
|
|
3811
|
+
fast-looking number is meaningless.
|
|
3812
|
+
"""
|
|
3813
|
+
|
|
3814
|
+
import asyncio
|
|
3815
|
+
import base64
|
|
3816
|
+
import json
|
|
3817
|
+
import math
|
|
3818
|
+
import os
|
|
3819
|
+
import struct
|
|
3820
|
+
import sys
|
|
3821
|
+
import types
|
|
3822
|
+
|
|
3823
|
+
OUT_RATE = 24000
|
|
3824
|
+
CHUNK_MS = 100
|
|
3825
|
+
|
|
3826
|
+
THINK = float(os.environ.get("FM_FAKE_THINK", "0.4"))
|
|
3827
|
+
REPLY_SECONDS = float(os.environ.get("FM_FAKE_REPLY_SECONDS", "0.4"))
|
|
3828
|
+
SCRIPT = [s.strip() for s in os.environ.get("FM_FAKE_SCRIPT", "status").split(",")
|
|
3829
|
+
if s.strip()]
|
|
3830
|
+
STATE = os.environ.get("FM_FAKE_STATE", "")
|
|
3831
|
+
LOG = os.environ.get("FM_FAKE_LOG", "")
|
|
3832
|
+
REQUEST = os.environ.get("FM_FAKE_REQUEST", "open a pull request for the retry")
|
|
3833
|
+
EARLY = os.environ.get("FM_FAKE_EARLY", "") == "1"
|
|
3834
|
+
|
|
3835
|
+
HEARD = {"status": "how is the fleet doing right now", "handover": REQUEST}
|
|
3836
|
+
|
|
3837
|
+
# How much of the captain's speech a clean-end session takes before its output
|
|
3838
|
+
# stream ends. Three chunks is 300 ms, so on a two second clip the end lands well
|
|
3839
|
+
# inside the key press and the rest of that press arrives at a session that is
|
|
3840
|
+
# already over.
|
|
3841
|
+
CLEAN_END_AFTER_BYTES = 3200 * 3
|
|
3842
|
+
|
|
3843
|
+
|
|
3844
|
+
def _turn_kind():
|
|
3845
|
+
"""Return this session's turn kind, advancing a counter that lives on disk.
|
|
3846
|
+
|
|
3847
|
+
The relay reconnects per turn on purpose, so the count cannot live in this
|
|
3848
|
+
process: each turn is a new stream in a new session.
|
|
3849
|
+
"""
|
|
3850
|
+
index = 0
|
|
3851
|
+
if STATE:
|
|
3852
|
+
try:
|
|
3853
|
+
with open(STATE, encoding="utf-8") as handle:
|
|
3854
|
+
index = int(handle.read().strip() or "0")
|
|
3855
|
+
except (OSError, ValueError):
|
|
3856
|
+
index = 0
|
|
3857
|
+
try:
|
|
3858
|
+
with open(STATE, "w", encoding="utf-8") as handle:
|
|
3859
|
+
handle.write(str(index + 1))
|
|
3860
|
+
except OSError:
|
|
3861
|
+
pass
|
|
3862
|
+
if not SCRIPT:
|
|
3863
|
+
return "status", index
|
|
3864
|
+
return SCRIPT[index % len(SCRIPT)], index
|
|
3865
|
+
|
|
3866
|
+
|
|
3867
|
+
def _speech(seconds):
|
|
3868
|
+
"""Return reply audio: a quiet tone, so a byte count is a duration."""
|
|
3869
|
+
out = bytearray()
|
|
3870
|
+
for n in range(int(OUT_RATE * seconds)):
|
|
3871
|
+
out += struct.pack("<h", int(6000 * math.sin(2 * math.pi * 220 * n / OUT_RATE)))
|
|
3872
|
+
return bytes(out)
|
|
3873
|
+
|
|
3874
|
+
|
|
3875
|
+
def _status_sentence(result):
|
|
3876
|
+
"""Compose the spoken answer out of what the records reader returned."""
|
|
3877
|
+
if result.get("error"):
|
|
3878
|
+
return "I could not read the records: {}".format(result["error"])
|
|
3879
|
+
said = "Right now, {} in flight, {} waiting on you, {} open pull requests.".format(
|
|
3880
|
+
result.get("in_flight"), result.get("awaiting_captain"),
|
|
3881
|
+
result.get("open_pull_requests"))
|
|
3882
|
+
names = [row.get("id") for row in result.get("in_flight_detail", [])][:2]
|
|
3883
|
+
if names:
|
|
3884
|
+
said += " The ones moving are {}.".format(" and ".join(names))
|
|
3885
|
+
notes = result.get("captain_notes_waiting") or 0
|
|
3886
|
+
if notes:
|
|
3887
|
+
said += " {} note is queued for the first mate.".format(notes)
|
|
3888
|
+
if result.get("scope") == "counts":
|
|
3889
|
+
said += " Identifiers are not available by voice at this read scope."
|
|
3890
|
+
return said
|
|
3891
|
+
|
|
3892
|
+
|
|
3893
|
+
def _queued_sentence(result):
|
|
3894
|
+
if result.get("error"):
|
|
3895
|
+
return "I could not queue that: {}".format(result["error"])
|
|
3896
|
+
return ("That is queued with the first mate as {}. I have not done any of it "
|
|
3897
|
+
"myself.".format(result.get("note_id") or "a note"))
|
|
3898
|
+
|
|
3899
|
+
|
|
3900
|
+
class _Result:
|
|
3901
|
+
def __init__(self, payload):
|
|
3902
|
+
self.value = types.SimpleNamespace(bytes_=payload)
|
|
3903
|
+
|
|
3904
|
+
|
|
3905
|
+
class _OutputReader:
|
|
3906
|
+
def __init__(self, queue):
|
|
3907
|
+
self._queue = queue
|
|
3908
|
+
|
|
3909
|
+
async def receive(self):
|
|
3910
|
+
item = await self._queue.get()
|
|
3911
|
+
return None if item is None else _Result(item)
|
|
3912
|
+
|
|
3913
|
+
|
|
3914
|
+
class _InputStream:
|
|
3915
|
+
def __init__(self, stream):
|
|
3916
|
+
self._stream = stream
|
|
3917
|
+
|
|
3918
|
+
async def send(self, chunk):
|
|
3919
|
+
await self._stream.on_input(chunk.value.bytes_)
|
|
3920
|
+
|
|
3921
|
+
async def close(self):
|
|
3922
|
+
await self._stream.finish()
|
|
3923
|
+
|
|
3924
|
+
|
|
3925
|
+
class _Stream:
|
|
3926
|
+
"""One bidirectional session, which is one turn the way the relay uses it."""
|
|
3927
|
+
|
|
3928
|
+
def __init__(self, model_id, config):
|
|
3929
|
+
self.kind, self.index = _turn_kind()
|
|
3930
|
+
self.out = asyncio.Queue()
|
|
3931
|
+
self.input_stream = _InputStream(self)
|
|
3932
|
+
self._reader = _OutputReader(self.out)
|
|
3933
|
+
self._audio_content = None
|
|
3934
|
+
self._pending_use = None
|
|
3935
|
+
self._tools = {}
|
|
3936
|
+
self._next_tool = 0
|
|
3937
|
+
self._replied = False
|
|
3938
|
+
self._reply_task = None
|
|
3939
|
+
self._logged = False
|
|
3940
|
+
self._ended_early = False
|
|
3941
|
+
self.record = {
|
|
3942
|
+
"turn": self.index + 1,
|
|
3943
|
+
"turn_kind": self.kind,
|
|
3944
|
+
"model_id": model_id,
|
|
3945
|
+
"endpoint": config.endpoint_uri,
|
|
3946
|
+
"region": config.region,
|
|
3947
|
+
"credential_key_id": config.credentials.get("aws_access_key_id"),
|
|
3948
|
+
"tool_names_offered": [],
|
|
3949
|
+
"audio_bytes_in": 0,
|
|
3950
|
+
"tool_calls": [],
|
|
3951
|
+
"heard": "",
|
|
3952
|
+
"said": [],
|
|
3953
|
+
"reply_audio_bytes": 0,
|
|
3954
|
+
}
|
|
3955
|
+
|
|
3956
|
+
async def await_output(self):
|
|
3957
|
+
return (None, self._reader)
|
|
3958
|
+
|
|
3959
|
+
def _emit(self, event):
|
|
3960
|
+
self.out.put_nowait(json.dumps({"event": event}).encode())
|
|
3961
|
+
|
|
3962
|
+
async def on_input(self, raw):
|
|
3963
|
+
event = json.loads(raw.decode()).get("event", {})
|
|
3964
|
+
for name, body in event.items():
|
|
3965
|
+
if name == "promptStart":
|
|
3966
|
+
self.record["tool_names_offered"] = [
|
|
3967
|
+
t.get("toolSpec", {}).get("name")
|
|
3968
|
+
for t in body.get("toolConfiguration", {}).get("tools", [])]
|
|
3969
|
+
elif name == "contentStart" and body.get("type") == "AUDIO":
|
|
3970
|
+
self._audio_content = body.get("contentName")
|
|
3971
|
+
elif name == "contentStart" and body.get("type") == "TOOL":
|
|
3972
|
+
self._pending_use = body.get(
|
|
3973
|
+
"toolResultInputConfiguration", {}).get("toolUseId")
|
|
3974
|
+
elif name == "audioInput":
|
|
3975
|
+
self.record["audio_bytes_in"] += len(
|
|
3976
|
+
base64.b64decode(body.get("content", "")))
|
|
3977
|
+
self._maybe_end_cleanly()
|
|
3978
|
+
if EARLY and not self._replied and not self._ended_early:
|
|
3979
|
+
# Answering while the captain's clip is still arriving, which
|
|
3980
|
+
# is what the model's own endpoint detector does to a clip
|
|
3981
|
+
# that ends in silence.
|
|
3982
|
+
self._replied = True
|
|
3983
|
+
self._reply_task = asyncio.create_task(self._reply())
|
|
3984
|
+
elif name == "toolResult":
|
|
3985
|
+
self._tool_result(body)
|
|
3986
|
+
elif name == "contentEnd":
|
|
3987
|
+
if (body.get("contentName") == self._audio_content
|
|
3988
|
+
and not self._replied and not self._ended_early):
|
|
3989
|
+
# The captain's talk end. Everything measured is measured
|
|
3990
|
+
# from here, so the reply starts no earlier than this.
|
|
3991
|
+
self._replied = True
|
|
3992
|
+
self._audio_content = None
|
|
3993
|
+
self._reply_task = asyncio.create_task(self._reply())
|
|
3994
|
+
|
|
3995
|
+
def _maybe_end_cleanly(self):
|
|
3996
|
+
"""End a clean-end session's output stream, mid-key-press and unannounced.
|
|
3997
|
+
|
|
3998
|
+
None on the output queue is what the SDK gives the reader for a stream
|
|
3999
|
+
that is simply over: no exception, no stop reason, nothing to report. The
|
|
4000
|
+
input half stays open, exactly as it does when the model is the side that
|
|
4001
|
+
finished, so the rest of the captain's key press still arrives here and
|
|
4002
|
+
goes nowhere.
|
|
4003
|
+
"""
|
|
4004
|
+
if (self.kind != "clean-end" or self._ended_early
|
|
4005
|
+
or self.record["audio_bytes_in"] < CLEAN_END_AFTER_BYTES):
|
|
4006
|
+
return
|
|
4007
|
+
self._ended_early = True
|
|
4008
|
+
self.record["ended_early"] = True
|
|
4009
|
+
self._write_log()
|
|
4010
|
+
self.out.put_nowait(None)
|
|
4011
|
+
|
|
4012
|
+
def _tool_result(self, body):
|
|
4013
|
+
try:
|
|
4014
|
+
result = json.loads(body.get("content") or "{}")
|
|
4015
|
+
except ValueError:
|
|
4016
|
+
result = {}
|
|
4017
|
+
if self.record["tool_calls"]:
|
|
4018
|
+
self.record["tool_calls"][-1]["result"] = result
|
|
4019
|
+
future = self._tools.pop(self._pending_use, None)
|
|
4020
|
+
if future is not None and not future.done():
|
|
4021
|
+
future.set_result(result)
|
|
4022
|
+
|
|
4023
|
+
async def _call_tool(self, name, arguments):
|
|
4024
|
+
self._next_tool += 1
|
|
4025
|
+
use_id = "use-{}-{}".format(self.index + 1, self._next_tool)
|
|
4026
|
+
future = asyncio.get_running_loop().create_future()
|
|
4027
|
+
self._tools[use_id] = future
|
|
4028
|
+
self.record["tool_calls"].append({"name": name, "arguments": arguments})
|
|
4029
|
+
self._emit({"toolUse": {"toolName": name, "toolUseId": use_id,
|
|
4030
|
+
"content": json.dumps(arguments)}})
|
|
4031
|
+
try:
|
|
4032
|
+
return await asyncio.wait_for(future, timeout=15)
|
|
4033
|
+
except asyncio.TimeoutError:
|
|
4034
|
+
return {"error": "the relay never answered the tool call"}
|
|
4035
|
+
|
|
4036
|
+
def _say(self, text):
|
|
4037
|
+
self.record["said"].append(text)
|
|
4038
|
+
self._emit({"textOutput": {"role": "ASSISTANT", "content": text}})
|
|
4039
|
+
|
|
4040
|
+
async def _reply(self):
|
|
4041
|
+
await asyncio.sleep(THINK)
|
|
4042
|
+
heard = HEARD.get(self.kind, "how is the fleet doing")
|
|
4043
|
+
self.record["heard"] = heard
|
|
4044
|
+
self._emit({"textOutput": {"role": "USER", "content": heard}})
|
|
4045
|
+
if self.kind == "handover":
|
|
4046
|
+
self._say("I am not the first mate, so I am handing that to it.")
|
|
4047
|
+
result = await self._call_tool("hand_over_to_firstmate",
|
|
4048
|
+
{"request": REQUEST})
|
|
4049
|
+
self._say(_queued_sentence(result))
|
|
4050
|
+
else:
|
|
4051
|
+
result = await self._call_tool("get_fleet_status", {})
|
|
4052
|
+
self._say(_status_sentence(result))
|
|
4053
|
+
pcm = _speech(REPLY_SECONDS)
|
|
4054
|
+
step = OUT_RATE * 2 * CHUNK_MS // 1000
|
|
4055
|
+
for at in range(0, len(pcm), step):
|
|
4056
|
+
block = pcm[at:at + step]
|
|
4057
|
+
self._emit({"audioOutput": {
|
|
4058
|
+
"content": base64.b64encode(block).decode()}})
|
|
4059
|
+
self.record["reply_audio_bytes"] += len(block)
|
|
4060
|
+
await asyncio.sleep(0.01)
|
|
4061
|
+
# Trap 1: this, not completionEnd, is what says the reply ended.
|
|
4062
|
+
self._emit({"contentEnd": {"stopReason": "END_TURN"}})
|
|
4063
|
+
self._write_log()
|
|
4064
|
+
|
|
4065
|
+
def _write_log(self):
|
|
4066
|
+
if self._logged or not LOG:
|
|
4067
|
+
return
|
|
4068
|
+
self._logged = True
|
|
4069
|
+
with open(LOG, "a", encoding="utf-8") as handle:
|
|
4070
|
+
handle.write(json.dumps(self.record) + "\n")
|
|
4071
|
+
|
|
4072
|
+
async def finish(self):
|
|
4073
|
+
self._write_log()
|
|
4074
|
+
self.out.put_nowait(None)
|
|
4075
|
+
|
|
4076
|
+
|
|
4077
|
+
class AsyncBedrockRuntimeConfig:
|
|
4078
|
+
def __init__(self, endpoint_uri, region, credentials):
|
|
4079
|
+
self.endpoint_uri = endpoint_uri
|
|
4080
|
+
self.region = region
|
|
4081
|
+
self.credentials = credentials
|
|
4082
|
+
|
|
4083
|
+
@classmethod
|
|
4084
|
+
async def resolve(cls, endpoint_uri=None, region=None, **credentials):
|
|
4085
|
+
return cls(endpoint_uri, region, credentials)
|
|
4086
|
+
|
|
4087
|
+
|
|
4088
|
+
class AsyncBedrockRuntimeClient:
|
|
4089
|
+
def __init__(self, config=None):
|
|
4090
|
+
self.config = config
|
|
4091
|
+
|
|
4092
|
+
async def invoke_model_with_bidirectional_stream(self, operation):
|
|
4093
|
+
return _Stream(operation.model_id, self.config)
|
|
4094
|
+
|
|
4095
|
+
|
|
4096
|
+
class InvokeModelWithBidirectionalStreamOperationInput:
|
|
4097
|
+
def __init__(self, model_id=None):
|
|
4098
|
+
self.model_id = model_id
|
|
4099
|
+
|
|
4100
|
+
|
|
4101
|
+
class BidirectionalInputPayloadPart:
|
|
4102
|
+
def __init__(self, bytes_=b""):
|
|
4103
|
+
self.bytes_ = bytes_
|
|
4104
|
+
|
|
4105
|
+
|
|
4106
|
+
class InvokeModelWithBidirectionalStreamInputChunk:
|
|
4107
|
+
def __init__(self, value=None):
|
|
4108
|
+
self.value = value
|
|
4109
|
+
|
|
4110
|
+
|
|
4111
|
+
def _submodule(name, **members):
|
|
4112
|
+
module = types.ModuleType(__name__ + "." + name)
|
|
4113
|
+
for key, value in members.items():
|
|
4114
|
+
setattr(module, key, value)
|
|
4115
|
+
sys.modules[module.__name__] = module
|
|
4116
|
+
return module
|
|
4117
|
+
|
|
4118
|
+
|
|
4119
|
+
client = _submodule(
|
|
4120
|
+
"client",
|
|
4121
|
+
AsyncBedrockRuntimeClient=AsyncBedrockRuntimeClient,
|
|
4122
|
+
InvokeModelWithBidirectionalStreamOperationInput=(
|
|
4123
|
+
InvokeModelWithBidirectionalStreamOperationInput))
|
|
4124
|
+
config = _submodule("config", AsyncBedrockRuntimeConfig=AsyncBedrockRuntimeConfig)
|
|
4125
|
+
models = _submodule(
|
|
4126
|
+
"models",
|
|
4127
|
+
BidirectionalInputPayloadPart=BidirectionalInputPayloadPart,
|
|
4128
|
+
InvokeModelWithBidirectionalStreamInputChunk=(
|
|
4129
|
+
InvokeModelWithBidirectionalStreamInputChunk))
|
|
4130
|
+
PY
|
|
4131
|
+
|
|
4132
|
+
# What the desktop side of the connection has, and the laptop side does not. The
|
|
4133
|
+
# AWS credential is here and nowhere else, which is the whole point of the shape.
|
|
4134
|
+
cat > "$E2E/bin/desktop.env" <<EOF
|
|
4135
|
+
PATH=$PATH
|
|
4136
|
+
HOME=$E2E/desktop-home
|
|
4137
|
+
PYTHONPATH=$E2E/fakesdk
|
|
4138
|
+
PYTHONDONTWRITEBYTECODE=1
|
|
4139
|
+
FM_HOME=$E2E/home
|
|
4140
|
+
FM_FAKE_STATE=$E2E/turn-counter
|
|
4141
|
+
FM_FAKE_LOG=$E2E/model-sessions.jsonl
|
|
4142
|
+
FM_FAKE_THINK=0.4
|
|
4143
|
+
FM_FAKE_REPLY_SECONDS=0.4
|
|
4144
|
+
FM_FAKE_SCRIPT=status,handover
|
|
4145
|
+
FM_FAKE_REQUEST=$E2E_REQUEST
|
|
4146
|
+
AWS_ACCESS_KEY_ID=$E2E_KEY
|
|
4147
|
+
AWS_SECRET_ACCESS_KEY=desktop-secret-not-a-real-key
|
|
4148
|
+
EOF
|
|
4149
|
+
|
|
4150
|
+
# Stands in for ssh, so the client takes its real `ssh -T <host> <relay>` path
|
|
4151
|
+
# and the desktop's environment is a boundary rather than an assertion: the relay
|
|
4152
|
+
# starts from env -i and desktop.env, so nothing the laptop holds can reach it.
|
|
4153
|
+
cat > "$E2E/bin/ssh" <<'SH'
|
|
4154
|
+
#!/usr/bin/env bash
|
|
4155
|
+
set -eu
|
|
4156
|
+
DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
4157
|
+
if [ "${1:-}" = "-T" ]; then shift; fi
|
|
4158
|
+
shift # the host, which is this machine
|
|
4159
|
+
desktop_env=()
|
|
4160
|
+
while IFS= read -r line; do desktop_env+=("$line"); done < "$DIR/desktop.env"
|
|
4161
|
+
exec env -i "${desktop_env[@]}" "$@"
|
|
4162
|
+
SH
|
|
4163
|
+
chmod +x "$E2E/bin/ssh"
|
|
4164
|
+
|
|
4165
|
+
# Two seconds of speech-shaped audio ending on speech, not silence: the relay's
|
|
4166
|
+
# own 400 ms of padding is what makes a push-to-talk release answerable, and a
|
|
4167
|
+
# clip this long makes a clock started at the wrong end unmistakable.
|
|
4168
|
+
python3 - "$E2E/clip.pcm" <<'PY' || fail "could not write the e2e clip"
|
|
4169
|
+
import math, struct, sys
|
|
4170
|
+
out = bytearray()
|
|
4171
|
+
for n in range(16000 * 2):
|
|
4172
|
+
swell = 0.5 + 0.5 * math.sin(2 * math.pi * 2.5 * n / 16000)
|
|
4173
|
+
out += struct.pack("<h", int(9000 * swell * math.sin(2 * math.pi * 190 * n / 16000)))
|
|
4174
|
+
open(sys.argv[1], "wb").write(bytes(out))
|
|
4175
|
+
PY
|
|
4176
|
+
|
|
4177
|
+
printf '0\n' > "$E2E/turn-counter"
|
|
4178
|
+
: > "$E2E/model-sessions.jsonl"
|
|
4179
|
+
|
|
4180
|
+
# env -i: the laptop has PATH and HOME and nothing else. No AWS variable, no
|
|
4181
|
+
# interpreter that can reach Bedrock, no firstmate home.
|
|
4182
|
+
laptop_aws=$(env -i PATH="$E2E/bin:$PATH" HOME="$E2E/laptop-home" env \
|
|
4183
|
+
| grep -c '^AWS_' || true)
|
|
4184
|
+
[ "$laptop_aws" = 0 ] || fail "the laptop end should hold no AWS variables"
|
|
4185
|
+
|
|
4186
|
+
set +e
|
|
4187
|
+
env -i PATH="$E2E/bin:$PATH" HOME="$E2E/laptop-home" PYTHONDONTWRITEBYTECODE=1 \
|
|
4188
|
+
python3 "$E2E/laptop/fm-voice-client.py" \
|
|
4189
|
+
--host desktop.example \
|
|
4190
|
+
--relay "$ROOT/bin/fm-voice-relay.py" \
|
|
4191
|
+
--relay-python python3 \
|
|
4192
|
+
--in-file "$E2E/clip.pcm" --out-file "$E2E/reply.pcm" \
|
|
4193
|
+
--runs 2 > "$E2E/runs.jsonl" 2> "$E2E/session.log"
|
|
4194
|
+
e2e_code=$?
|
|
4195
|
+
set -e
|
|
4196
|
+
[ "$e2e_code" = 0 ] || {
|
|
4197
|
+
cat "$E2E/session.log" >&2
|
|
4198
|
+
fail "the spoken round trip exited $e2e_code"
|
|
4199
|
+
}
|
|
4200
|
+
|
|
4201
|
+
# The reader's own answer, taken independently, so the spoken answer is checked
|
|
4202
|
+
# against the records rather than against itself.
|
|
4203
|
+
independent=$(python3 "$ROOT/bin/fm_voice_records.py" status \
|
|
4204
|
+
--home "$E2E/home" --scope full) || fail "independent status read failed"
|
|
4205
|
+
printf '%s' "$independent" > "$E2E/independent.json"
|
|
4206
|
+
|
|
4207
|
+
python3 - "$E2E" "$E2E_KEY" "$E2E_REGION" "$E2E_MODEL" "$E2E_REQUEST" \
|
|
4208
|
+
"$NEVER_TOKEN" <<'PY' || fail "the spoken round trip did not hold"
|
|
4209
|
+
import json, os, sys
|
|
4210
|
+
|
|
4211
|
+
root, key, region, model, request, never = sys.argv[1:7]
|
|
4212
|
+
|
|
4213
|
+
|
|
4214
|
+
def check(cond, label):
|
|
4215
|
+
if not cond:
|
|
4216
|
+
sys.exit("round trip: " + label)
|
|
4217
|
+
|
|
4218
|
+
|
|
4219
|
+
def read(name):
|
|
4220
|
+
with open(os.path.join(root, name), encoding="utf-8") as handle:
|
|
4221
|
+
return [json.loads(line) for line in handle if line.strip()]
|
|
4222
|
+
|
|
4223
|
+
|
|
4224
|
+
runs = read("runs.jsonl")
|
|
4225
|
+
sessions = read("model-sessions.jsonl")
|
|
4226
|
+
records = json.load(open(os.path.join(root, "independent.json"), encoding="utf-8"))
|
|
4227
|
+
transcript = open(os.path.join(root, "session.log"), encoding="utf-8").read()
|
|
4228
|
+
|
|
4229
|
+
# Two turns asked, two turns answered with audio, neither of them lost.
|
|
4230
|
+
check(len(runs) == 2, "expected two turn records, got %d" % len(runs))
|
|
4231
|
+
check(len(sessions) == 2, "expected two model sessions, got %d" % len(sessions))
|
|
4232
|
+
for run in runs:
|
|
4233
|
+
check(run["answered"], "turn %s was not answered" % run["run"])
|
|
4234
|
+
check(run["relay_error"] is None,
|
|
4235
|
+
"turn %s failed: %s" % (run["run"], run["relay_error"]))
|
|
4236
|
+
check(run["reply_audio_seconds"] > 0,
|
|
4237
|
+
"turn %s produced no reply audio" % run["run"])
|
|
4238
|
+
# Push to talk is the default and the only mode that runs, and the transport
|
|
4239
|
+
# is the ssh path rather than a local child.
|
|
4240
|
+
check(run["listen"] == "push-to-talk", "listen mode was %r" % run["listen"])
|
|
4241
|
+
check(run["transport"] == "ssh", "transport was %r" % run["transport"])
|
|
4242
|
+
# The per-turn reconnect exists so a second question is not barge-in.
|
|
4243
|
+
check(not run["interrupted"],
|
|
4244
|
+
"turn %s was treated as an interruption" % run["run"])
|
|
4245
|
+
|
|
4246
|
+
# Whose account and which model. The relay carries no default for either, so
|
|
4247
|
+
# this is the home's configuration reaching Bedrock, and the credential is the
|
|
4248
|
+
# one only the desktop side of the connection holds.
|
|
4249
|
+
for session in sessions:
|
|
4250
|
+
check(session["model_id"] == model, "model was %r" % session["model_id"])
|
|
4251
|
+
check(session["region"] == region, "region was %r" % session["region"])
|
|
4252
|
+
check(session["endpoint"] ==
|
|
4253
|
+
"https://bedrock-runtime.{}.amazonaws.com".format(region),
|
|
4254
|
+
"endpoint was %r" % session["endpoint"])
|
|
4255
|
+
check(session["credential_key_id"] == key,
|
|
4256
|
+
"session opened with %r" % session["credential_key_id"])
|
|
4257
|
+
check(session["tool_names_offered"] ==
|
|
4258
|
+
["get_fleet_status", "hand_over_to_firstmate"],
|
|
4259
|
+
"tools offered were %r" % session["tool_names_offered"])
|
|
4260
|
+
# Trap 2: a push-to-talk release supplies no trailing silence, so the relay
|
|
4261
|
+
# appends its own. Without it the model truncates the turn and never answers.
|
|
4262
|
+
check(session["audio_bytes_in"] == 16000 * 2 * 2 + 400 * 32,
|
|
4263
|
+
"the uplink carried %d bytes, so the talk-end padding is not being "
|
|
4264
|
+
"sent" % session["audio_bytes_in"])
|
|
4265
|
+
|
|
4266
|
+
# The status answer is the records. Every number the agent said aloud came from
|
|
4267
|
+
# the reader, checked against a separate read of the same home.
|
|
4268
|
+
status = sessions[0]
|
|
4269
|
+
check([c["name"] for c in status["tool_calls"]] == ["get_fleet_status"],
|
|
4270
|
+
"the status turn called %r" % [c["name"] for c in status["tool_calls"]])
|
|
4271
|
+
served = status["tool_calls"][0]["result"]
|
|
4272
|
+
for field in ("in_flight", "awaiting_captain", "open_pull_requests", "queued"):
|
|
4273
|
+
check(served[field] == records[field],
|
|
4274
|
+
"the reader served %s=%r but the records say %r"
|
|
4275
|
+
% (field, served[field], records[field]))
|
|
4276
|
+
said = " ".join(status["said"])
|
|
4277
|
+
check("{} in flight".format(records["in_flight"]) in said,
|
|
4278
|
+
"the spoken answer did not carry the count: %r" % said)
|
|
4279
|
+
check(records["in_flight_detail"][0]["id"] in said,
|
|
4280
|
+
"the spoken answer named no open work: %r" % said)
|
|
4281
|
+
check(never not in said and never not in json.dumps(served),
|
|
4282
|
+
"a note body or finished title reached a spoken answer")
|
|
4283
|
+
check(said in transcript, "the captain never saw the answer: %r" % transcript)
|
|
4284
|
+
|
|
4285
|
+
# The handover turn queues real work and says so. The note is firstmate's own
|
|
4286
|
+
# queue, written by bin/fm-inbox.sh, and the agent's confirmation carries the id
|
|
4287
|
+
# that queue gave it, so it cannot be claiming to have queued something it did
|
|
4288
|
+
# not.
|
|
4289
|
+
handover = sessions[1]
|
|
4290
|
+
check([c["name"] for c in handover["tool_calls"]] == ["hand_over_to_firstmate"],
|
|
4291
|
+
"the handover turn called %r" % [c["name"] for c in handover["tool_calls"]])
|
|
4292
|
+
check(handover["tool_calls"][0]["arguments"]["request"] == request,
|
|
4293
|
+
"the captain's words were rewritten: %r"
|
|
4294
|
+
% handover["tool_calls"][0]["arguments"])
|
|
4295
|
+
queued = handover["tool_calls"][0]["result"]
|
|
4296
|
+
check(queued.get("queued") is True, "the request was not queued: %r" % queued)
|
|
4297
|
+
note_id = queued.get("note_id")
|
|
4298
|
+
check(bool(note_id), "the queue returned no note id: %r" % queued)
|
|
4299
|
+
check(runs[1]["queued_note"] == note_id,
|
|
4300
|
+
"the client was told %r, the queue wrote %r"
|
|
4301
|
+
% (runs[1]["queued_note"], note_id))
|
|
4302
|
+
check(note_id in " ".join(handover["said"]),
|
|
4303
|
+
"the agent did not confirm the queued note: %r" % handover["said"])
|
|
4304
|
+
check("handed to the first mate" in transcript,
|
|
4305
|
+
"the captain was never told it was handed over: %r" % transcript)
|
|
4306
|
+
note = os.path.join(root, "home", "state", "inbox", note_id + ".note")
|
|
4307
|
+
check(os.path.exists(note), "no note on disk at %s" % note)
|
|
4308
|
+
check(request in open(note, encoding="utf-8").read(),
|
|
4309
|
+
"the note does not carry the captain's words")
|
|
4310
|
+
|
|
4311
|
+
# THE NUMBER THIS BUILD EXISTS TO PRODUCE, and the instant it is measured from.
|
|
4312
|
+
# The clip is two seconds long and the stand-in waits 0.4 s before speaking, so a
|
|
4313
|
+
# figure measured from the captain's talk end lands near half a second and one
|
|
4314
|
+
# measured from the start of their speech lands near two and a half. The bound is
|
|
4315
|
+
# loose enough for a loaded machine and nowhere near the wrong clock.
|
|
4316
|
+
for run in runs:
|
|
4317
|
+
first = run["first_audio_s"]
|
|
4318
|
+
check(first is not None, "turn %s reported no first audio" % run["run"])
|
|
4319
|
+
check(0.2 < first < 1.6,
|
|
4320
|
+
"turn %s reported first audio at %.3fs, which is not measured from the "
|
|
4321
|
+
"captain's talk end" % (run["run"], first))
|
|
4322
|
+
marks = run["relay_marks_since_talk_end"]
|
|
4323
|
+
for mark in ("tool_use", "tool_answered", "first_audio", "reply_end"):
|
|
4324
|
+
check(mark in marks, "turn %s is missing the %s mark" % (run["run"], mark))
|
|
4325
|
+
check(marks["tool_use"] <= marks["first_audio"] <= marks["reply_end"],
|
|
4326
|
+
"turn %s reports its marks out of order: %r" % (run["run"], marks))
|
|
4327
|
+
check(run["first_frame_s"] is not None and run["first_played_s"] is not None,
|
|
4328
|
+
"turn %s reported no wire or playback figure" % run["run"])
|
|
4329
|
+
|
|
4330
|
+
# The reply audio survived the framing byte for byte.
|
|
4331
|
+
sent = sum(s["reply_audio_bytes"] for s in sessions)
|
|
4332
|
+
got = os.path.getsize(os.path.join(root, "reply.pcm"))
|
|
4333
|
+
check(sent > 0 and sent == got,
|
|
4334
|
+
"the model sent %d bytes of reply audio and the client wrote %d" % (sent, got))
|
|
4335
|
+
|
|
4336
|
+
# AND NOTHING ALARMING WAS SAID, which is the other half of every fault line this
|
|
4337
|
+
# build added. The relay's goodbye arrives on this path too, at the end of every
|
|
4338
|
+
# ordinary session, so a line that reads as a fault would fire here on a session
|
|
4339
|
+
# where all of it worked. An alarm that also goes off on success is worse than no
|
|
4340
|
+
# alarm: the captain learns to skip it, and then the one that means their question
|
|
4341
|
+
# was lost is invisible as well. The session end still speaks, in its own words.
|
|
4342
|
+
for alarming in ("stopped without being asked", "could not handle the relay's",
|
|
4343
|
+
"connection lost", "the connection ended",
|
|
4344
|
+
"were not taken", "no reply within"):
|
|
4345
|
+
check(alarming not in transcript,
|
|
4346
|
+
"a session where every turn was answered said %r: %r"
|
|
4347
|
+
% (alarming, transcript))
|
|
4348
|
+
# And no record named a reason, which is what now decides the exit code as well.
|
|
4349
|
+
# A fault that fires on a session where everything worked would fail every clean
|
|
4350
|
+
# run from here on, so the absence is worth pinning next to the exit code itself.
|
|
4351
|
+
for run in runs:
|
|
4352
|
+
check(run["relay_error"] is None,
|
|
4353
|
+
"turn %s recorded a reason on a session that worked: %r"
|
|
4354
|
+
% (run["run"], run["relay_error"]))
|
|
4355
|
+
check("the relay signed off" in transcript,
|
|
4356
|
+
"but the end of the session should still be said, in wording that cannot be "
|
|
4357
|
+
"read as a fault: %r" % transcript)
|
|
4358
|
+
PY
|
|
4359
|
+
pass "a spoken turn goes out and comes back: the records answer, firstmate gets the work"
|
|
4360
|
+
|
|
4361
|
+
# --- a model session that ends while the captain is still talking ------------
|
|
4362
|
+
#
|
|
4363
|
+
# A Bedrock session ending is not a fault. The stream simply stops: no exception,
|
|
4364
|
+
# no stop reason, nothing to report. It can happen mid-conversation, and when it
|
|
4365
|
+
# does the captain is usually still holding the talk key, because that is when
|
|
4366
|
+
# the relay is talking to the model at all.
|
|
4367
|
+
#
|
|
4368
|
+
# The relay used to treat that as its own reason to stop, which is the worst
|
|
4369
|
+
# available failure shape: the relay dies without saying anything the captain can
|
|
4370
|
+
# act on, and they find out by speaking a whole question into nothing and getting
|
|
4371
|
+
# no answer. Per-turn reconnect already covers this - the next talk key builds a
|
|
4372
|
+
# new session, at a measured cost of 0.02 s - and a reconnect that cannot be made
|
|
4373
|
+
# is spoken to the captain through the turn-failed path. So the session ending
|
|
4374
|
+
# costs them the remainder of one key press, and nothing else.
|
|
4375
|
+
#
|
|
4376
|
+
# This case is the round trip above with one difference: the model finishes with
|
|
4377
|
+
# the first session 300 ms into a two second key press. What it holds is that the
|
|
4378
|
+
# relay is still serving afterwards and that the NEXT talk key gets a working
|
|
4379
|
+
# session rather than a closed pipe - a real answer, out of the real records, over
|
|
4380
|
+
# the same connection. The relay may exit for three reasons and this is not one of
|
|
4381
|
+
# them.
|
|
4382
|
+
|
|
4383
|
+
SURVIVE="$E2E/survive"
|
|
4384
|
+
mkdir -p "$SURVIVE/bin"
|
|
4385
|
+
|
|
4386
|
+
# The same desktop, with its own turn script and its own record of what the model
|
|
4387
|
+
# was asked, so neither run can read the other's sessions.
|
|
4388
|
+
grep -v '^FM_FAKE_' "$E2E/bin/desktop.env" > "$SURVIVE/bin/desktop.env"
|
|
4389
|
+
cat >> "$SURVIVE/bin/desktop.env" <<EOF
|
|
4390
|
+
FM_FAKE_STATE=$SURVIVE/turn-counter
|
|
4391
|
+
FM_FAKE_LOG=$SURVIVE/model-sessions.jsonl
|
|
4392
|
+
FM_FAKE_THINK=0.4
|
|
4393
|
+
FM_FAKE_REPLY_SECONDS=0.4
|
|
4394
|
+
FM_FAKE_SCRIPT=clean-end,status
|
|
4395
|
+
EOF
|
|
4396
|
+
cp "$E2E/bin/ssh" "$SURVIVE/bin/ssh"
|
|
4397
|
+
printf '0\n' > "$SURVIVE/turn-counter"
|
|
4398
|
+
: > "$SURVIVE/model-sessions.jsonl"
|
|
4399
|
+
|
|
4400
|
+
# Exit 1 is the honest outcome and what is asserted: one of the two turns really
|
|
4401
|
+
# was lost, because the model stopped listening part way through it.
|
|
4402
|
+
set +e
|
|
4403
|
+
env -i PATH="$SURVIVE/bin:$PATH" HOME="$E2E/laptop-home" PYTHONDONTWRITEBYTECODE=1 \
|
|
4404
|
+
python3 "$E2E/laptop/fm-voice-client.py" \
|
|
4405
|
+
--host desktop.example \
|
|
4406
|
+
--relay "$ROOT/bin/fm-voice-relay.py" \
|
|
4407
|
+
--relay-python python3 \
|
|
4408
|
+
--in-file "$E2E/clip.pcm" --out-file "$SURVIVE/reply.pcm" \
|
|
4409
|
+
--timeout 12 --runs 2 > "$SURVIVE/runs.jsonl" 2> "$SURVIVE/session.log"
|
|
4410
|
+
survive_code=$?
|
|
4411
|
+
set -e
|
|
4412
|
+
[ "$survive_code" = 1 ] || {
|
|
4413
|
+
cat "$SURVIVE/session.log" >&2
|
|
4414
|
+
fail "a lost turn and a good one should exit 1, not $survive_code"
|
|
4415
|
+
}
|
|
4416
|
+
|
|
4417
|
+
python3 - "$SURVIVE" "$E2E/independent.json" <<'PY' \
|
|
4418
|
+
|| fail "a model session ending did not leave the relay serving"
|
|
4419
|
+
import json, os, sys
|
|
4420
|
+
|
|
4421
|
+
root, records_path = sys.argv[1:3]
|
|
4422
|
+
|
|
4423
|
+
CLIP_BYTES = 16000 * 2 * 2
|
|
4424
|
+
|
|
4425
|
+
|
|
4426
|
+
def check(cond, label):
|
|
4427
|
+
if not cond:
|
|
4428
|
+
sys.exit("session ended: " + label)
|
|
4429
|
+
|
|
4430
|
+
|
|
4431
|
+
def read(name):
|
|
4432
|
+
with open(os.path.join(root, name), encoding="utf-8") as handle:
|
|
4433
|
+
return [json.loads(line) for line in handle if line.strip()]
|
|
4434
|
+
|
|
4435
|
+
|
|
4436
|
+
runs = read("runs.jsonl")
|
|
4437
|
+
sessions = read("model-sessions.jsonl")
|
|
4438
|
+
records = json.load(open(records_path, encoding="utf-8"))
|
|
4439
|
+
transcript = open(os.path.join(root, "session.log"), encoding="utf-8").read()
|
|
4440
|
+
|
|
4441
|
+
# The first session really did end part way through the captain's key press,
|
|
4442
|
+
# rather than after answering: it took some of the clip and not all of it.
|
|
4443
|
+
check(len(sessions) >= 1, "the model was never asked anything")
|
|
4444
|
+
first = sessions[0]
|
|
4445
|
+
check(first["turn_kind"] == "clean-end" and first.get("ended_early"),
|
|
4446
|
+
"the first session did not end early: %r" % first)
|
|
4447
|
+
check(0 < first["audio_bytes_in"] < CLIP_BYTES,
|
|
4448
|
+
"the session ended after %d of %d bytes, so it did not end mid-press"
|
|
4449
|
+
% (first["audio_bytes_in"], CLIP_BYTES))
|
|
4450
|
+
check(not first["said"] and not first["reply_audio_bytes"],
|
|
4451
|
+
"the lost turn was answered after all: %r" % first)
|
|
4452
|
+
|
|
4453
|
+
# THE POINT. The relay was still there for the next talk key, so two turns were
|
|
4454
|
+
# taken over the one connection and the second one was a whole session of its own.
|
|
4455
|
+
check(len(runs) == 2,
|
|
4456
|
+
"the relay stopped serving when the model ended its session: %d turn(s) "
|
|
4457
|
+
"taken, %r" % (len(runs), transcript))
|
|
4458
|
+
check(len(sessions) == 2,
|
|
4459
|
+
"the next talk key did not get a session: %d opened" % len(sessions))
|
|
4460
|
+
check(not runs[0]["answered"], "the lost turn should be the first one: %r" % runs[0])
|
|
4461
|
+
# An ordinary session end is not a turn FAILURE - the relay names none, marks the
|
|
4462
|
+
# session spent for nobody, and the next talk key gets a working one - but the
|
|
4463
|
+
# turn it landed in still has no answer, and the record has to say why. Left null
|
|
4464
|
+
# there, the only turn in this whole file that went unanswered for a knowable
|
|
4465
|
+
# reason reads in runs.jsonl exactly like a turn the model declined, and that file
|
|
4466
|
+
# is where the published latency spread comes from.
|
|
4467
|
+
check(runs[0]["relay_error"],
|
|
4468
|
+
"a turn released by the session ending must name why it has no answer: %r"
|
|
4469
|
+
% runs[0])
|
|
4470
|
+
check("session" in runs[0]["relay_error"],
|
|
4471
|
+
"and it should name the session ending rather than some other fault: %r"
|
|
4472
|
+
% runs[0]["relay_error"])
|
|
4473
|
+
check("could not finish that turn" not in transcript,
|
|
4474
|
+
"the relay must not have called this a failed turn: %r" % transcript)
|
|
4475
|
+
|
|
4476
|
+
# And it was a working session rather than a closed pipe: a real answer, composed
|
|
4477
|
+
# from a real read of the records, spoken to the captain over the same connection.
|
|
4478
|
+
good = runs[1]
|
|
4479
|
+
check(good["answered"] and good["reply_audio_seconds"] > 0,
|
|
4480
|
+
"the next talk key got no answer: %r" % good)
|
|
4481
|
+
check(good["relay_error"] is None,
|
|
4482
|
+
"the replacement session failed: %r" % good["relay_error"])
|
|
4483
|
+
check([c["name"] for c in sessions[1]["tool_calls"]] == ["get_fleet_status"],
|
|
4484
|
+
"the replacement turn called %r"
|
|
4485
|
+
% [c["name"] for c in sessions[1]["tool_calls"]])
|
|
4486
|
+
# The whole question, not an answer to nothing: every byte of the clip and the
|
|
4487
|
+
# talk-end padding reached the replacement session.
|
|
4488
|
+
check(sessions[1]["audio_bytes_in"] == CLIP_BYTES + 400 * 32,
|
|
4489
|
+
"the replacement session heard %d bytes of a %d byte question"
|
|
4490
|
+
% (sessions[1]["audio_bytes_in"], CLIP_BYTES + 400 * 32))
|
|
4491
|
+
served = sessions[1]["tool_calls"][0]["result"]
|
|
4492
|
+
for field in ("in_flight", "open_pull_requests"):
|
|
4493
|
+
check(served[field] == records[field],
|
|
4494
|
+
"the replacement session served %s=%r but the records say %r"
|
|
4495
|
+
% (field, served[field], records[field]))
|
|
4496
|
+
said = " ".join(sessions[1]["said"])
|
|
4497
|
+
check("{} in flight".format(records["in_flight"]) in said,
|
|
4498
|
+
"the answer did not carry the count: %r" % said)
|
|
4499
|
+
check(said in transcript, "the captain never heard the answer: %r" % transcript)
|
|
4500
|
+
check(good["first_audio_s"] is not None and 0.2 < good["first_audio_s"] < 1.6,
|
|
4501
|
+
"the recovered turn reported first audio at %r, which is not measured from "
|
|
4502
|
+
"the captain's talk end" % good["first_audio_s"])
|
|
4503
|
+
check(os.path.getsize(os.path.join(root, "reply.pcm"))
|
|
4504
|
+
== sessions[1]["reply_audio_bytes"],
|
|
4505
|
+
"the reply audio the client wrote is not what the good session sent")
|
|
4506
|
+
|
|
4507
|
+
# Said once. The rest of that key press is another seventeen audio frames, and
|
|
4508
|
+
# the flag saying the session is over stays set for every one of them, so a
|
|
4509
|
+
# notice sent from the frame loop instead of from the end itself would put this
|
|
4510
|
+
# line in front of the captain ten times a second while they were still speaking.
|
|
4511
|
+
check(transcript.count("the relay ended the session") == 1,
|
|
4512
|
+
"the session ending was announced %d times: %r"
|
|
4513
|
+
% (transcript.count("the relay ended the session"), transcript))
|
|
4514
|
+
check("connection lost" not in transcript,
|
|
4515
|
+
"the connection should have outlived the session: %r" % transcript)
|
|
4516
|
+
PY
|
|
4517
|
+
pass "a model session that ends mid-conversation costs one turn, not the relay"
|
|
4518
|
+
|
|
4519
|
+
# --- an uplink that stops being a frame stream --------------------------------
|
|
4520
|
+
#
|
|
4521
|
+
# One of the three things that ends the relay, and the only one it diagnoses: the
|
|
4522
|
+
# captain's uplink desynchronises, so the relay can no longer tell a header from
|
|
4523
|
+
# audio. It writes what it saw on its own stderr and exits 2, which is honest,
|
|
4524
|
+
# and it used to tell the client only goodbye. Everything the relay knew stayed
|
|
4525
|
+
# on a stderr no run file quotes, while the turn the captain was in the middle of
|
|
4526
|
+
# came back as answered: false with nothing in relay_error - the reason existed
|
|
4527
|
+
# and was thrown away.
|
|
4528
|
+
#
|
|
4529
|
+
# Worse, the teardown closes the model session first, and that sets the flag which
|
|
4530
|
+
# silences the session reader's own notice, so this path really did send the client
|
|
4531
|
+
# nothing at all but the goodbye.
|
|
4532
|
+
#
|
|
4533
|
+
# The relay is driven directly here rather than through the client, because no
|
|
4534
|
+
# client sends a bad frame: the desync is the transport corrupting one, and the
|
|
4535
|
+
# only way to put one on the wire is to be the other end. The model is the same
|
|
4536
|
+
# stand-in the round trip above uses, and it is never asked to answer.
|
|
4537
|
+
|
|
4538
|
+
set +e
|
|
4539
|
+
env -i PATH="$PATH" HOME="$E2E/desktop-home" PYTHONPATH="$E2E/fakesdk" \
|
|
4540
|
+
PYTHONDONTWRITEBYTECODE=1 FM_HOME="$E2E/home" \
|
|
4541
|
+
AWS_ACCESS_KEY_ID="$E2E_KEY" \
|
|
4542
|
+
AWS_SECRET_ACCESS_KEY=desktop-secret-not-a-real-key \
|
|
4543
|
+
python3 - "$ROOT/bin" <<'PY'
|
|
4544
|
+
import os, subprocess, sys
|
|
4545
|
+
sys.path.insert(0, sys.argv[1])
|
|
4546
|
+
import fm_voice_frame as frame
|
|
4547
|
+
|
|
4548
|
+
relay = os.path.join(sys.argv[1], "fm-voice-relay.py")
|
|
4549
|
+
|
|
4550
|
+
|
|
4551
|
+
def check(cond, label):
|
|
4552
|
+
if not cond:
|
|
4553
|
+
sys.exit("desync: " + label)
|
|
4554
|
+
|
|
4555
|
+
|
|
4556
|
+
proc = subprocess.Popen(
|
|
4557
|
+
[sys.executable, relay, "--serve"],
|
|
4558
|
+
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
4559
|
+
try:
|
|
4560
|
+
magic = proc.stdout.read(len(frame.MAGIC))
|
|
4561
|
+
check(magic == frame.MAGIC, "the relay did not open the stream: %r" % magic)
|
|
4562
|
+
reader = frame.Reader(proc.stdout)
|
|
4563
|
+
kind, payload = reader.read()
|
|
4564
|
+
check(kind == frame.NOTICE
|
|
4565
|
+
and frame.decode_json(payload).get("event") == "ready",
|
|
4566
|
+
"the relay was never ready: %r %r" % (kind, payload))
|
|
4567
|
+
|
|
4568
|
+
# A turn is open when the uplink goes, which is when it goes in practice: the
|
|
4569
|
+
# captain is holding the talk key, so that is when there are frames on the
|
|
4570
|
+
# wire to be corrupted at all.
|
|
4571
|
+
proc.stdin.write(frame.encode(frame.TALK_START))
|
|
4572
|
+
proc.stdin.write(b"\xff\x00\x00\x00\x00")
|
|
4573
|
+
proc.stdin.flush()
|
|
4574
|
+
|
|
4575
|
+
frames = []
|
|
4576
|
+
while True:
|
|
4577
|
+
got = reader.read()
|
|
4578
|
+
if got is None:
|
|
4579
|
+
break
|
|
4580
|
+
frames.append(got)
|
|
4581
|
+
if got[0] == frame.BYE:
|
|
4582
|
+
break
|
|
4583
|
+
finally:
|
|
4584
|
+
proc.stdin.close()
|
|
4585
|
+
said = proc.stderr.read().decode("utf-8", "replace")
|
|
4586
|
+
code = proc.wait(timeout=30)
|
|
4587
|
+
|
|
4588
|
+
kinds = [k for k, _ in frames]
|
|
4589
|
+
check(frame.BYE in kinds, "the relay never said goodbye: %r" % kinds)
|
|
4590
|
+
named = [n for n in (frame.decode_json(p) for k, p in frames if k == frame.NOTICE)
|
|
4591
|
+
if n.get("event") == "turn-failed"]
|
|
4592
|
+
check(named,
|
|
4593
|
+
"the relay diagnosed the fault and told the client only goodbye: %r" % kinds)
|
|
4594
|
+
check("FrameError" in named[0].get("error", ""),
|
|
4595
|
+
"the notice should carry what the relay saw: %r" % named[0])
|
|
4596
|
+
# Before the goodbye, and before the model session is closed: closing it awaits
|
|
4597
|
+
# the model stream, which can be slow or raise, and the client must learn the
|
|
4598
|
+
# reason either way.
|
|
4599
|
+
failed_at = next(i for i, (k, p) in enumerate(frames)
|
|
4600
|
+
if k == frame.NOTICE
|
|
4601
|
+
and frame.decode_json(p).get("event") == "turn-failed")
|
|
4602
|
+
check(failed_at < kinds.index(frame.BYE),
|
|
4603
|
+
"the reason must reach the client ahead of the goodbye: %r" % kinds)
|
|
4604
|
+
|
|
4605
|
+
# Still said on the relay's own stderr, which the captain reads over SSH, and
|
|
4606
|
+
# still the unhappy exit code: naming the fault down the connection does not make
|
|
4607
|
+
# it a turn the relay recovered from.
|
|
4608
|
+
check("not a frame stream any more" in said,
|
|
4609
|
+
"the relay should still name the desync on its stderr: %r" % said)
|
|
4610
|
+
check(code == 2, "a desynchronised uplink must exit 2, got %r" % code)
|
|
4611
|
+
PY
|
|
4612
|
+
desync_code=$?
|
|
4613
|
+
set -e
|
|
4614
|
+
[ "$desync_code" = 0 ] || fail "a desynchronised uplink was not named to the client"
|
|
4615
|
+
pass "a desynchronised uplink is named to the client before the goodbye, not only to stderr"
|
|
4616
|
+
|
|
4617
|
+
# --- the desktop's own check, and the clock it refuses to lie about ----------
|
|
4618
|
+
#
|
|
4619
|
+
# `fm-voice-relay.py --self-test <clip.pcm>` is what docs/voice-relay.md tells the
|
|
4620
|
+
# captain to run before they touch the laptop, and it is also the instrument the
|
|
4621
|
+
# direct column of the latency table was measured with. Everything above drives
|
|
4622
|
+
# the relay through the client; this drives the desktop check itself, because a
|
|
4623
|
+
# broken --self-test is a captain who cannot tell a configured desktop from an
|
|
4624
|
+
# unconfigured one, and a number in a table that nobody can reproduce.
|
|
4625
|
+
#
|
|
4626
|
+
# The second case is the one worth having. A clip that already ends in silence
|
|
4627
|
+
# makes the model answer before this end of the stream has said the turn is over,
|
|
4628
|
+
# so every figure is measured from the wrong instant and comes out negative. The
|
|
4629
|
+
# reply really is fast and the number really is meaningless, which is the worst
|
|
4630
|
+
# combination to leave in a results file for someone who was not here. The guard
|
|
4631
|
+
# has to name the marks and say why, not print the figure.
|
|
4632
|
+
|
|
4633
|
+
SELFTEST="$E2E/self-test"
|
|
4634
|
+
mkdir -p "$SELFTEST"
|
|
4635
|
+
printf '0\n' > "$SELFTEST/turn-counter"
|
|
4636
|
+
|
|
4637
|
+
# The same two seconds of speech, with a second of silence glued on the end: the
|
|
4638
|
+
# shape the docs warn against, and the only way to reach the guard.
|
|
4639
|
+
cat "$E2E/clip.pcm" > "$SELFTEST/ends-in-silence.pcm"
|
|
4640
|
+
python3 - "$SELFTEST/ends-in-silence.pcm" <<'PY' \
|
|
4641
|
+
|| fail "could not write the silence-tailed clip"
|
|
4642
|
+
import sys
|
|
4643
|
+
with open(sys.argv[1], "ab") as handle:
|
|
4644
|
+
handle.write(b"\x00\x00" * 16000)
|
|
4645
|
+
PY
|
|
4646
|
+
|
|
4647
|
+
# The desktop, and only the desktop: the AWS credential and the SDK live here.
|
|
4648
|
+
relay_self_test() {
|
|
4649
|
+
local clip=$1
|
|
4650
|
+
shift
|
|
4651
|
+
env -i PATH="$PATH" HOME="$E2E/desktop-home" PYTHONPATH="$E2E/fakesdk" \
|
|
4652
|
+
PYTHONDONTWRITEBYTECODE=1 FM_HOME="$E2E/home" \
|
|
4653
|
+
FM_FAKE_STATE="$SELFTEST/turn-counter" FM_FAKE_LOG="$SELFTEST/sessions.jsonl" \
|
|
4654
|
+
FM_FAKE_THINK=0.4 FM_FAKE_REPLY_SECONDS=0.4 FM_FAKE_SCRIPT=status \
|
|
4655
|
+
AWS_ACCESS_KEY_ID="$E2E_KEY" \
|
|
4656
|
+
AWS_SECRET_ACCESS_KEY=desktop-secret-not-a-real-key \
|
|
4657
|
+
"$@" python3 "$ROOT/bin/fm-voice-relay.py" --self-test "$clip"
|
|
4658
|
+
}
|
|
4659
|
+
|
|
4660
|
+
set +e
|
|
4661
|
+
ok_out=$(relay_self_test "$E2E/clip.pcm" 2> "$SELFTEST/ok.err")
|
|
4662
|
+
ok_code=$?
|
|
4663
|
+
set -e
|
|
4664
|
+
expect_code 0 "$ok_code" "the documented desktop check should answer"
|
|
4665
|
+
printf '%s\n' "$ok_out" > "$SELFTEST/ok.json"
|
|
4666
|
+
|
|
4667
|
+
python3 - "$SELFTEST/ok.json" "$E2E/independent.json" "$E2E_REGION" "$E2E_MODEL" \
|
|
4668
|
+
<<'PY' || fail "the desktop check did not report a usable measurement"
|
|
4669
|
+
import json, sys
|
|
4670
|
+
|
|
4671
|
+
report = json.load(open(sys.argv[1], encoding="utf-8"))
|
|
4672
|
+
records = json.load(open(sys.argv[2], encoding="utf-8"))
|
|
4673
|
+
region, model = sys.argv[3:5]
|
|
4674
|
+
|
|
4675
|
+
|
|
4676
|
+
def check(cond, label):
|
|
4677
|
+
if not cond:
|
|
4678
|
+
sys.exit("self-test: %s -- %r" % (label, report))
|
|
4679
|
+
|
|
4680
|
+
|
|
4681
|
+
check(report["mode"] == "self-test", "not a self-test report")
|
|
4682
|
+
check(report["answered"] and report["reply_audio_seconds"] > 0, "it did not answer")
|
|
4683
|
+
check(report["relay_error"] is None, "it reported an error")
|
|
4684
|
+
check(report["region"] == region and report["model"] == model,
|
|
4685
|
+
"it used the wrong account's model")
|
|
4686
|
+
check(report["tool_names"] == ["get_fleet_status"], "it called the wrong tool")
|
|
4687
|
+
# The words are the records, the same as over the relay.
|
|
4688
|
+
check("{} in flight".format(records["in_flight"]) in report["said"],
|
|
4689
|
+
"the spoken answer did not carry the count")
|
|
4690
|
+
check(report["heard"], "it reported nothing heard")
|
|
4691
|
+
# The figure the direct column of the latency table is made of, measured from the
|
|
4692
|
+
# talk end: the clip is two seconds and the stand-in thinks for 0.4 s, so a clock
|
|
4693
|
+
# started at the wrong end lands near 2.4.
|
|
4694
|
+
check(report["clock_unusable"] == [], "it flagged a clip that ends on speech")
|
|
4695
|
+
for mark in ("tool_use_s", "first_audio_s", "reply_end_s"):
|
|
4696
|
+
check(report[mark] is not None, "no %s figure" % mark)
|
|
4697
|
+
check(0.2 < report["first_audio_s"] < 1.6,
|
|
4698
|
+
"first audio at %r is not measured from the talk end" % report["first_audio_s"])
|
|
4699
|
+
check(report["tool_use_s"] <= report["first_audio_s"] <= report["reply_end_s"],
|
|
4700
|
+
"the figures are out of order")
|
|
4701
|
+
PY
|
|
4702
|
+
pass "the desktop's own check answers from the records and times it from the talk end"
|
|
4703
|
+
|
|
4704
|
+
set +e
|
|
4705
|
+
early_out=$(relay_self_test "$SELFTEST/ends-in-silence.pcm" FM_FAKE_EARLY=1 \
|
|
4706
|
+
2> "$SELFTEST/early.err")
|
|
4707
|
+
early_code=$?
|
|
4708
|
+
set -e
|
|
4709
|
+
expect_code 0 "$early_code" "an answered turn is still an answered turn"
|
|
4710
|
+
printf '%s\n' "$early_out" > "$SELFTEST/early.json"
|
|
4711
|
+
|
|
4712
|
+
early_err=$(cat "$SELFTEST/early.err")
|
|
4713
|
+
assert_contains "$early_err" "measured from the wrong instant" \
|
|
4714
|
+
"the guard must say why the timings cannot be used"
|
|
4715
|
+
assert_contains "$early_err" "ends on speech" \
|
|
4716
|
+
"the guard must say what clip to pass instead"
|
|
4717
|
+
|
|
4718
|
+
python3 - "$SELFTEST/early.json" <<'PY' \
|
|
4719
|
+
|| fail "a reply that beat the end of the clip was recorded as a good measurement"
|
|
4720
|
+
import json, sys
|
|
4721
|
+
|
|
4722
|
+
report = json.load(open(sys.argv[1], encoding="utf-8"))
|
|
4723
|
+
|
|
4724
|
+
|
|
4725
|
+
def check(cond, label):
|
|
4726
|
+
if not cond:
|
|
4727
|
+
sys.exit("wrong clock: %s -- %r" % (label, report))
|
|
4728
|
+
|
|
4729
|
+
|
|
4730
|
+
# It answered. That is exactly why the figure is dangerous rather than obviously
|
|
4731
|
+
# broken: a reader sees answered: true and a fast number.
|
|
4732
|
+
check(report["answered"] and report["reply_audio_seconds"] > 0,
|
|
4733
|
+
"the turn was not answered at all, so this is not the case under test")
|
|
4734
|
+
check(report["first_audio_s"] < 0,
|
|
4735
|
+
"the reply did not beat the end of the clip, so the guard was not reached")
|
|
4736
|
+
for mark in ("tool_use", "first_audio", "reply_end"):
|
|
4737
|
+
check(mark in report["clock_unusable"], "%s is not named as unusable" % mark)
|
|
4738
|
+
PY
|
|
4739
|
+
pass "a reply that arrives before the end of the clip is named as an unusable clock"
|
|
4740
|
+
|
|
4741
|
+
printf 'all voice relay cases passed\n'
|