@vellumai/assistant 0.10.12 → 0.11.0-staging.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +1 -1
- package/ARCHITECTURE.md +4 -5
- package/README.md +1 -1
- package/docs/architecture/memory.md +60 -23
- package/docs/error-handling.md +3 -3
- package/docs/guardian-request-flow.md +112 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/package.json +1 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/__tests__/channels.test.ts +9 -1
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +449 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/channels.ts +1 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/secret-detection.ts +468 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@slack/types/dist/events/app.d.ts +28 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@slack/types/dist/events/app.d.ts.map +1 -1
- package/node_modules/@vellumai/gateway-client/node_modules/@slack/types/dist/events/index.d.ts +2 -2
- package/node_modules/@vellumai/gateway-client/node_modules/@slack/types/dist/events/index.d.ts.map +1 -1
- package/node_modules/@vellumai/gateway-client/node_modules/@slack/types/dist/events/index.js.map +1 -1
- package/node_modules/@vellumai/gateway-client/node_modules/@slack/types/dist/events/message.d.ts +2 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@slack/types/dist/events/message.d.ts.map +1 -1
- package/node_modules/@vellumai/gateway-client/node_modules/@slack/types/package.json +1 -1
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/package.json +1 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/__tests__/channels.test.ts +9 -1
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +449 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/channels.ts +1 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/secret-detection.ts +468 -0
- package/node_modules/@vellumai/gateway-client/package.json +1 -1
- package/node_modules/@vellumai/gateway-client/src/admission-policy-contract.ts +6 -0
- package/node_modules/@vellumai/gateway-client/src/inbound-contract.ts +34 -0
- package/node_modules/@vellumai/service-contracts/package.json +1 -0
- package/node_modules/@vellumai/service-contracts/src/__tests__/channels.test.ts +9 -1
- package/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +449 -0
- package/node_modules/@vellumai/service-contracts/src/channels.ts +1 -0
- package/node_modules/@vellumai/service-contracts/src/secret-detection.ts +468 -0
- package/openapi.yaml +264 -14
- package/package.json +2 -2
- package/src/__tests__/acp-session.test.ts +3 -3
- package/src/__tests__/agent-loop-mutable-latest-user-message.test.ts +3 -0
- package/src/__tests__/anthropic-provider.test.ts +40 -0
- package/src/__tests__/app-conversation-ids-backfill.test.ts +33 -2
- package/src/__tests__/app-conversation-ids.test.ts +204 -0
- package/src/__tests__/approval-cascade.test.ts +5 -5
- package/src/__tests__/approval-routes-http.test.ts +3 -3
- package/src/__tests__/ask-question-resolver.test.ts +207 -0
- package/src/__tests__/assistant-event-hub-self-exclusion.test.ts +16 -16
- package/src/__tests__/assistant-event-hub-targeted.test.ts +13 -11
- package/src/__tests__/assistant-event-hub.test.ts +13 -11
- package/src/__tests__/assistant-event.test.ts +12 -9
- package/src/__tests__/assistant-feature-flag-guard.test.ts +3 -3
- package/src/__tests__/assistant-feature-flag-guardrails.test.ts +1 -1
- package/src/__tests__/assistant-id-boundary-guard.test.ts +0 -58
- package/src/__tests__/assistant-stream-state.test.ts +17 -7
- package/src/__tests__/avatar-identity-sync.test.ts +3 -3
- package/src/__tests__/background-workers-disk-pressure.test.ts +1 -0
- package/src/__tests__/call-conversation-messages.test.ts +134 -18
- package/src/__tests__/call-start-guardian-guard.test.ts +27 -0
- package/src/__tests__/channel-approvals.test.ts +0 -27
- package/src/__tests__/channel-guardian.test.ts +85 -512
- package/src/__tests__/channel-readiness-slack-remote.test.ts +151 -3
- package/src/__tests__/channel-retry-sweep.test.ts +88 -1
- package/src/__tests__/channel-setup-panel-ack.test.ts +12 -9
- package/src/__tests__/chat-reveal-guard-priming.test.ts +31 -34
- package/src/__tests__/cli-memory-v2-reembed-skills.test.ts +2 -2
- package/src/__tests__/compaction-events.test.ts +8 -8
- package/src/__tests__/config-loader-platform-defaults.test.ts +54 -28
- package/src/__tests__/config-model-image-provider.test.ts +20 -0
- package/src/__tests__/config-schema-memory-substrate.test.ts +306 -0
- package/src/__tests__/config-schema.test.ts +14 -2
- package/src/__tests__/config-set-platform-guard.test.ts +11 -12
- package/src/__tests__/config-sounds-sync.test.ts +4 -4
- package/src/__tests__/confirmation-request-guardian-bridge.test.ts +5 -0
- package/src/__tests__/context-search-memory-source.test.ts +1 -1
- package/src/__tests__/context-search-memory-v2-source.test.ts +6 -6
- package/src/__tests__/context-token-estimator.test.ts +54 -1
- package/src/__tests__/conversation-agent-loop-disk-pressure.test.ts +12 -3
- package/src/__tests__/conversation-agent-loop-fatal-cleanup.test.ts +13 -4
- package/src/__tests__/conversation-agent-loop-handlers-max-tokens.test.ts +4 -4
- package/src/__tests__/conversation-agent-loop-inference-profile.test.ts +10 -1
- package/src/__tests__/conversation-agent-loop-overflow.test.ts +22 -13
- package/src/__tests__/conversation-agent-loop.test.ts +39 -30
- package/src/__tests__/conversation-confirmation-signals.test.ts +14 -14
- package/src/__tests__/conversation-evictor.test.ts +49 -0
- package/src/__tests__/conversation-fork-crud.test.ts +1 -1
- package/src/__tests__/conversation-group-tools.test.ts +279 -0
- package/src/__tests__/conversation-history-web-search.test.ts +1 -1
- package/src/__tests__/conversation-key-store-disk-view.test.ts +5 -5
- package/src/__tests__/conversation-lineage.test.ts +235 -0
- package/src/__tests__/conversation-loop-db-lock-resilience.test.ts +5 -5
- package/src/__tests__/conversation-message-sync-tags.test.ts +4 -4
- package/src/__tests__/conversation-queue.test.ts +81 -81
- package/src/__tests__/conversation-routes-enabled-plugins.test.ts +8 -1
- package/src/__tests__/conversation-routes-hidden-queue.test.ts +8 -1
- package/src/__tests__/conversation-routes-slash-commands.test.ts +10 -1
- package/src/__tests__/conversation-runtime-assembly.test.ts +29 -2
- package/src/__tests__/conversation-slash-queue.test.ts +11 -11
- package/src/__tests__/conversation-slash-unknown.test.ts +2 -2
- package/src/__tests__/conversation-speed-override.test.ts +2 -2
- package/src/__tests__/conversation-summarize-route.test.ts +9 -0
- package/src/__tests__/conversation-surfaces-action-delivery.test.ts +18 -16
- package/src/__tests__/conversation-surfaces-activation-emit.test.ts +17 -17
- package/src/__tests__/conversation-surfaces-app-open.test.ts +7 -4
- package/src/__tests__/conversation-surfaces-data-persist.test.ts +7 -7
- package/src/__tests__/conversation-surfaces-standalone-payloads.test.ts +9 -9
- package/src/__tests__/conversation-surfaces-standalone.test.ts +12 -12
- package/src/__tests__/conversation-surfaces-state-update.test.ts +8 -5
- package/src/__tests__/conversation-surfaces-table-action.test.ts +6 -3
- package/src/__tests__/conversation-surfaces-task-progress.test.ts +46 -46
- package/src/__tests__/conversation-sync-tags.test.ts +5 -5
- package/src/__tests__/conversation-wait-for-idle.test.ts +2 -2
- package/src/__tests__/conversation-workspace-cache-state.test.ts +0 -4
- package/src/__tests__/conversation-workspace-injection.test.ts +0 -3
- package/src/__tests__/conversation-workspace-tool-tracking.test.ts +0 -3
- package/src/__tests__/copy-composer-tc-templates.test.ts +3 -131
- package/src/__tests__/credential-security-invariants.test.ts +1 -0
- package/src/__tests__/daemon-assistant-events.test.ts +14 -15
- package/src/__tests__/db-test-helpers.ts +13 -27
- package/src/__tests__/disk-pressure-routes.test.ts +12 -3
- package/src/__tests__/document-tool-security.test.ts +95 -0
- package/src/__tests__/drain-kick-guard.test.ts +232 -0
- package/src/__tests__/dynamic-skill-background-guard.test.ts +19 -6
- package/src/__tests__/emit-event-signal.test.ts +5 -6
- package/src/__tests__/encrypted-store-test-helpers.ts +10 -20
- package/src/__tests__/events-tail-route.test.ts +6 -4
- package/src/__tests__/feature-flag-test-helpers.ts +10 -20
- package/src/__tests__/file-edit-tool.test.ts +24 -4
- package/src/__tests__/file-list-tool.test.ts +2 -1
- package/src/__tests__/file-read-tool.test.ts +62 -10
- package/src/__tests__/file-write-tool.test.ts +24 -5
- package/src/__tests__/followup-tools.test.ts +58 -0
- package/src/__tests__/gateway-threshold-reader-mock.ts +99 -0
- package/src/__tests__/graph-extraction-event-date.test.ts +1 -1
- package/src/__tests__/guardian-question-mode.test.ts +42 -3
- package/src/__tests__/guardian-reply-router-answer-mode.test.ts +283 -0
- package/src/__tests__/guardian-vellum-destination-pinning.test.ts +221 -0
- package/src/__tests__/heartbeat-disk-pressure.test.ts +1 -0
- package/src/__tests__/heartbeat-service.test.ts +1 -0
- package/src/__tests__/helpers/native-web-search-harness.ts +10 -10
- package/src/__tests__/host-browser-proxy.test.ts +1 -1
- package/src/__tests__/host-file-edit-tool.test.ts +3 -3
- package/src/__tests__/host-file-read-tool.test.ts +2 -2
- package/src/__tests__/host-file-write-tool.test.ts +7 -3
- package/src/__tests__/host-shell-tool.test.ts +17 -5
- package/src/__tests__/host-transfer-pending-interactions.test.ts +8 -8
- package/src/__tests__/image-credentials.test.ts +65 -8
- package/src/__tests__/injection-block.test.ts +4 -4
- package/src/__tests__/injector-chain.test.ts +1 -1
- package/src/__tests__/injector-pkb-v2-silenced.test.ts +2 -2
- package/src/__tests__/injector-registry-order-guard.test.ts +4 -9
- package/src/__tests__/injector-v3-suppression.test.ts +6 -4
- package/src/__tests__/lifecycle-memory-v2-seed.test.ts +27 -28
- package/src/__tests__/list-messages-page-latest.test.ts +2 -2
- package/src/__tests__/llm-usage-store.test.ts +67 -4
- package/src/__tests__/log-redact.test.ts +18 -0
- package/src/__tests__/managed-profile-guard.test.ts +27 -0
- package/src/__tests__/managed-skill-lifecycle.test.ts +2 -2
- package/src/__tests__/media-generate-image.test.ts +62 -11
- package/src/__tests__/memory-identity-context-parity.test.ts +1 -1
- package/src/__tests__/memory-jobs-worker-lanes.test.ts +9 -1
- package/src/__tests__/memory-retrieval-hook.test.ts +4 -4
- package/src/__tests__/notification-decision-strategy.test.ts +115 -11
- package/src/__tests__/notification-deep-link.test.ts +12 -12
- package/src/__tests__/notification-vellum-adapter.test.ts +8 -8
- package/src/__tests__/outbound-slack-persistence.test.ts +8 -8
- package/src/__tests__/path-policy.test.ts +315 -0
- package/src/__tests__/pending-interactions-resolved-event.test.ts +6 -6
- package/src/__tests__/persistence-layering-guard.test.ts +2 -2
- package/src/__tests__/pkb-autoinject.test.ts +1 -1
- package/src/__tests__/platform-bash-auto-approve.test.ts +1 -0
- package/src/__tests__/platform-callback-registration.test.ts +10 -10
- package/src/__tests__/plugin-event-hub-facade.test.ts +22 -15
- package/src/__tests__/plugin-import-boundary-guard.test.ts +15 -15
- package/src/__tests__/processing-flag-persist-failure.test.ts +0 -4
- package/src/__tests__/provider-platform-proxy-integration.test.ts +0 -1
- package/src/__tests__/rebuild-index-graph-nodes.test.ts +1 -1
- package/src/__tests__/request-secret-standalone-channel-gate.test.ts +5 -5
- package/src/__tests__/require-fresh-approval.test.ts +38 -38
- package/src/__tests__/resolve-app-id.test.ts +32 -0
- package/src/__tests__/run-due-schedules.test.ts +72 -1
- package/src/__tests__/runtime-events-sse-parity.test.ts +11 -10
- package/src/__tests__/runtime-events-sse.test.ts +1 -1
- package/src/__tests__/schedule-routes.test.ts +44 -5
- package/src/__tests__/schedule-store.test.ts +3 -3
- package/src/__tests__/schedule-tools.test.ts +176 -0
- package/src/__tests__/secret-prompt-log-hygiene.test.ts +3 -3
- package/src/__tests__/secret-prompter-channel-fallback.test.ts +3 -3
- package/src/__tests__/secret-response-routing.test.ts +3 -3
- package/src/__tests__/secret-routes-platform-proxy.test.ts +4 -7
- package/src/__tests__/secret-scanner.test.ts +36 -0
- package/src/__tests__/send-endpoint-busy.test.ts +7 -7
- package/src/__tests__/slack-required-scopes-mirror.test.ts +81 -0
- package/src/__tests__/starter-task-flow.test.ts +26 -3
- package/src/__tests__/subagent-call-site-routing.test.ts +2 -2
- package/src/__tests__/subagent-disposal.test.ts +40 -11
- package/src/__tests__/subagent-fork-notifications.test.ts +4 -4
- package/src/__tests__/subagent-fork-prompt-role.test.ts +3 -3
- package/src/__tests__/subagent-fork-spawn.test.ts +4 -4
- package/src/__tests__/subagent-manager-notify.test.ts +73 -14
- package/src/__tests__/subagent-notify-parent.test.ts +48 -4
- package/src/__tests__/subagent-spawn-and-await.test.ts +62 -14
- package/src/__tests__/subagent-tools.test.ts +64 -0
- package/src/__tests__/sync-message-contract.test.ts +3 -3
- package/src/__tests__/terminal-tools.test.ts +12 -3
- package/src/__tests__/test-preload.ts +9 -0
- package/src/__tests__/tool-approval-handler.test.ts +587 -43
- package/src/__tests__/tool-execution-pipeline.benchmark.test.ts +7 -4
- package/src/__tests__/tool-executor.test.ts +116 -0
- package/src/__tests__/tool-grant-request-escalation.test.ts +5 -0
- package/src/__tests__/tool-preview-lifecycle.test.ts +32 -29
- package/src/__tests__/tool-result-metadata-plumbing.test.ts +5 -5
- package/src/__tests__/tool-side-effects-slack-dm.test.ts +1 -0
- package/src/__tests__/tool-start-timestamp.test.ts +4 -4
- package/src/__tests__/trusted-contact-lifecycle-notifications.test.ts +9 -22
- package/src/__tests__/ui-choice-copy-surfaces.test.ts +13 -13
- package/src/__tests__/ui-work-result-surface.test.ts +3 -3
- package/src/__tests__/voice-scoped-grant-consumer.test.ts +6 -6
- package/src/__tests__/voice-session-bridge.test.ts +44 -44
- package/src/__tests__/workspace-migration-134-image-generation-mode-to-provider.test.ts +284 -0
- package/src/__tests__/workspace-policy.test.ts +482 -2
- package/src/acp/__tests__/client-handler.test.ts +3 -3
- package/src/acp/__tests__/session-manager-persistence.test.ts +10 -10
- package/src/acp/__tests__/session-manager-resume.test.ts +7 -7
- package/src/acp/client-handler.ts +3 -3
- package/src/acp/session-manager.ts +8 -8
- package/src/api/events/subagent-event.ts +2 -2
- package/src/api/responses/memory-v2-activation-log.ts +2 -2
- package/src/api/responses/subagent-detail.ts +13 -0
- package/src/approvals/AGENTS.md +2 -0
- package/src/approvals/guardian-request-resolvers.ts +118 -24
- package/src/apps/app-store.ts +95 -4
- package/src/calls/__tests__/voice-control-protocol.test.ts +57 -0
- package/src/calls/__tests__/voice-session-bridge.test.ts +229 -7
- package/src/calls/__tests__/voice-triage-escalate.test.ts +3 -4
- package/src/calls/call-controller.ts +27 -27
- package/src/calls/call-conversation-messages.ts +58 -18
- package/src/calls/voice-control-protocol.ts +56 -0
- package/src/calls/voice-session-bridge.ts +178 -65
- package/src/calls/voice-triage-escalate.ts +2 -3
- package/src/channels/config.ts +8 -0
- package/src/channels/types.ts +1 -0
- package/src/cli/commands/config.ts +49 -13
- package/src/cli/commands/memory/index.help.ts +4 -2
- package/src/cli/commands/memory/memory-retrospective.ts +5 -0
- package/src/cli/commands/webhooks.help.ts +9 -4
- package/src/cli/lib/bundled-marketplace.json +12 -0
- package/src/cli.ts +2 -109
- package/src/config/__tests__/deployment-context-defaults.test.ts +9 -0
- package/src/config/__tests__/substrate-twin-shadowing.test.ts +191 -0
- package/src/config/assistant-feature-flags.ts +1 -1
- package/src/config/bundled-skills/conversation-groups/SKILL.md +29 -0
- package/src/config/bundled-skills/conversation-groups/TOOLS.json +58 -0
- package/src/config/bundled-skills/conversation-groups/tools/group-create.ts +12 -0
- package/src/config/bundled-skills/conversation-groups/tools/group-list.ts +12 -0
- package/src/config/bundled-skills/conversation-groups/tools/move-to-group.ts +12 -0
- package/src/config/bundled-skills/image-studio/tools/media-generate-image.ts +13 -7
- package/src/config/bundled-skills/schedule/SKILL.md +2 -0
- package/src/config/feature-flag-cache.ts +6 -16
- package/src/config/feature-flag-registry.json +14 -38
- package/src/config/loader.ts +7 -1
- package/src/config/memory-tier.ts +15 -10
- package/src/config/memory-v3-gate.test.ts +278 -1
- package/src/config/memory-v3-gate.ts +88 -26
- package/src/config/raw-config-utils.ts +1 -1
- package/src/config/schemas/__tests__/live-voice.test.ts +27 -16
- package/src/config/schemas/__tests__/memory-v3.test.ts +1 -0
- package/src/config/schemas/live-voice.ts +33 -7
- package/src/config/schemas/llm.ts +1 -0
- package/src/config/schemas/memory-retrospective.ts +13 -0
- package/src/config/schemas/memory-substrate.ts +183 -0
- package/src/config/schemas/memory-v2.ts +3 -2
- package/src/config/schemas/memory-v3.ts +11 -4
- package/src/config/schemas/memory.ts +35 -1
- package/src/config/schemas/services.ts +10 -2
- package/src/config/substrate-twin-shadowing.ts +195 -0
- package/src/config/webhook-routing.ts +67 -0
- package/src/context/compactor.ts +10 -3
- package/src/context/token-estimator.ts +21 -0
- package/src/daemon/__tests__/channel-ui-capability.test.ts +85 -0
- package/src/daemon/__tests__/conversation-surfaces-launch.test.ts +3 -3
- package/src/daemon/__tests__/embedding-reconcile.test.ts +3 -3
- package/src/daemon/__tests__/inference-profile-notification.test.ts +3 -6
- package/src/daemon/__tests__/native-web-search-metadata.test.ts +5 -5
- package/src/daemon/__tests__/web-search-status-text.test.ts +2 -2
- package/src/daemon/channel-ui-capability.ts +70 -1
- package/src/daemon/conversation-agent-loop-handlers.ts +4 -4
- package/src/daemon/conversation-agent-loop.ts +12 -5
- package/src/daemon/conversation-evictor.ts +26 -4
- package/src/daemon/conversation-launch.ts +11 -14
- package/src/daemon/conversation-lineage.ts +76 -0
- package/src/daemon/conversation-messaging.ts +14 -14
- package/src/daemon/conversation-notifiers.ts +2 -2
- package/src/daemon/conversation-process.ts +133 -21
- package/src/daemon/conversation-queue-manager.ts +7 -7
- package/src/daemon/conversation-runtime-assembly.ts +16 -0
- package/src/daemon/conversation-store.ts +11 -3
- package/src/daemon/conversation-surfaces.ts +8 -14
- package/src/daemon/conversation-tool-setup.ts +13 -7
- package/src/daemon/conversation-usage.ts +12 -11
- package/src/daemon/conversation.ts +38 -15
- package/src/daemon/disk-pressure-guard.ts +5 -12
- package/src/daemon/embedding-reconcile.ts +14 -9
- package/src/daemon/handlers/config-channels.ts +40 -134
- package/src/daemon/handlers/config-model.ts +15 -7
- package/src/daemon/handlers/conversations.ts +4 -2
- package/src/daemon/handlers/shared.ts +29 -0
- package/src/daemon/host-proxy-base.ts +4 -4
- package/src/daemon/lifecycle.ts +13 -0
- package/src/daemon/message-protocol.ts +8 -144
- package/src/daemon/message-types/computer-use.ts +0 -12
- package/src/daemon/message-types/conversations.ts +9 -345
- package/src/daemon/message-types/host-app-control.ts +0 -9
- package/src/daemon/message-types/host-browser.ts +0 -7
- package/src/daemon/message-types/host-ui-snapshot.ts +0 -7
- package/src/daemon/message-types/messages.ts +0 -48
- package/src/daemon/message-types/notifications.ts +0 -7
- package/src/daemon/message-types/skills.ts +0 -5
- package/src/daemon/message-types/surfaces.ts +0 -9
- package/src/daemon/message-types/sync.ts +0 -2
- package/src/daemon/process-message.ts +5 -5
- package/src/daemon/skill-memory-refresh.ts +5 -5
- package/src/daemon/tool-side-effects.ts +32 -16
- package/src/daemon/trust-context.ts +1 -1
- package/src/daemon/wake-conversation-ops.ts +4 -4
- package/src/heartbeat/__tests__/heartbeat-run-store.test.ts +100 -7
- package/src/heartbeat/__tests__/heartbeat-service.test.ts +67 -0
- package/src/heartbeat/heartbeat-run-store.ts +81 -4
- package/src/heartbeat/heartbeat-service.ts +32 -1
- package/src/home/__tests__/feed-writer.test.ts +7 -7
- package/src/home/__tests__/home-content-refresh.test.ts +3 -0
- package/src/home/__tests__/suggested-prompts.test.ts +1 -0
- package/src/home/feed-writer.ts +6 -13
- package/src/home/home-content-refresh.ts +6 -9
- package/src/home/relationship-state-writer.ts +5 -12
- package/src/home/suggested-prompts-cache.ts +6 -19
- package/src/ipc/__tests__/events-publish-client.test.ts +67 -0
- package/src/ipc/events-publish-client.ts +44 -0
- package/src/ipc/routes/__tests__/conversation-sync-ipc-routes.test.ts +2 -2
- package/src/ipc/routes/__tests__/events-ipc-routes.test.ts +38 -9
- package/src/ipc/routes/events-ipc-routes.ts +11 -31
- package/src/live-voice/__tests__/front-decision.test.ts +4 -176
- package/src/live-voice/__tests__/live-voice-agent-turn.test.ts +276 -68
- package/src/live-voice/__tests__/live-voice-continuation-fork-hydration.test.ts +286 -0
- package/src/live-voice/__tests__/live-voice-events.test.ts +6 -3
- package/src/live-voice/__tests__/live-voice-progress.test.ts +185 -46
- package/src/live-voice/__tests__/live-voice-triage-escalate.test.ts +24 -59
- package/src/live-voice/__tests__/live-voice-tts-session.test.ts +38 -54
- package/src/live-voice/__tests__/live-voice-vad.test.ts +1154 -636
- package/src/live-voice/front-decision.ts +5 -155
- package/src/live-voice/live-voice-metrics.ts +4 -1
- package/src/live-voice/live-voice-session.ts +1288 -439
- package/src/live-voice/progress-phrases.ts +17 -0
- package/src/live-voice/protocol.ts +15 -0
- package/src/media/app-icon-generator.ts +15 -8
- package/src/media/avatar-router.ts +9 -5
- package/src/media/image-credentials.ts +42 -10
- package/src/media/image-service.ts +1 -13
- package/src/media/types.ts +26 -3
- package/src/notifications/AGENTS.md +2 -0
- package/src/notifications/__tests__/broadcaster.test.ts +95 -0
- package/src/notifications/adapters/macos.ts +3 -3
- package/src/notifications/broadcaster.ts +57 -0
- package/src/notifications/copy-composer.ts +0 -15
- package/src/notifications/decision-engine.ts +21 -22
- package/src/notifications/guardian-question-mode.ts +57 -2
- package/src/notifications/home-feed-side-effect.ts +4 -0
- package/src/notifications/signal.ts +6 -4
- package/src/notifications/trusted-contact-payloads.ts +3 -4
- package/src/notifications/vellum-card-affinity.ts +39 -0
- package/src/oauth/AGENTS.md +1 -1
- package/src/permissions/channel-permission-query.test.ts +69 -0
- package/src/permissions/channel-permission-query.ts +73 -1
- package/src/permissions/checker.test.ts +298 -15
- package/src/permissions/checker.ts +102 -36
- package/src/permissions/confirmation-guardian-request.test.ts +2 -2
- package/src/permissions/confirmation-guardian-request.ts +2 -2
- package/src/permissions/gateway-threshold-reader.test.ts +224 -17
- package/src/permissions/gateway-threshold-reader.ts +82 -10
- package/src/permissions/ipc-risk-types.ts +8 -0
- package/src/permissions/prompter.ts +5 -5
- package/src/permissions/question-guardian-request.test.ts +222 -0
- package/src/permissions/question-guardian-request.ts +179 -0
- package/src/permissions/question-prompter.test.ts +8 -8
- package/src/permissions/question-prompter.ts +18 -0
- package/src/permissions/secret-prompter.ts +2 -4
- package/src/permissions/workspace-policy.ts +269 -34
- package/src/persistence/__tests__/lifecycle-quiesce.test.ts +82 -0
- package/src/persistence/__tests__/message-content-file.test.ts +58 -3
- package/src/persistence/conversation-crud.ts +93 -0
- package/src/persistence/conversation-key-store.ts +63 -0
- package/src/persistence/conversation-parent.ts +27 -0
- package/src/persistence/db-async-query.ts +1 -1
- package/src/persistence/db-singleton.ts +14 -27
- package/src/persistence/delivery-crud.ts +45 -0
- package/src/persistence/embeddings/embedding-backend.ts +2 -1
- package/src/persistence/embeddings/embedding-identity.ts +7 -3
- package/src/persistence/job-utils.ts +32 -27
- package/src/persistence/jobs-store.ts +92 -3
- package/src/persistence/lifecycle-quiesce.ts +80 -0
- package/src/persistence/llm-usage-store.ts +45 -8
- package/src/persistence/message-content-file.ts +4 -4
- package/src/persistence/migrations/336-move-memory-v2-activation-logs-to-memory-db.ts +1 -1
- package/src/persistence/migrations/351-schedule-skill-script-handoff.ts +39 -0
- package/src/persistence/migrations/352-drop-schedule-skill-script-handoff.test.ts +83 -0
- package/src/persistence/migrations/352-drop-schedule-skill-script-handoff.ts +36 -0
- package/src/persistence/migrations/353-add-llm-usage-conversation-type.test.ts +110 -0
- package/src/persistence/migrations/353-add-llm-usage-conversation-type.ts +49 -0
- package/src/persistence/migrations/354-backfill-app-conversation-lineage.ts +41 -0
- package/src/persistence/schema/infrastructure.ts +11 -0
- package/src/persistence/steps.ts +9 -1
- package/src/plugin-api/__tests__/publish-event-forwarding.test.ts +113 -0
- package/src/plugin-api/__tests__/publish-event.test.ts +57 -0
- package/src/plugin-api/conversation-turn.ts +4 -4
- package/src/plugin-api/event-hub-facade.ts +15 -6
- package/src/plugin-api/index.ts +10 -4
- package/src/plugin-api/publish-event.ts +36 -0
- package/src/plugins/AGENTS.md +2 -0
- package/src/plugins/defaults/compaction/window-manager.ts +9 -0
- package/src/plugins/defaults/index.ts +9 -13
- package/src/plugins/defaults/memory/AGENTS.md +775 -0
- package/src/plugins/defaults/memory/__tests__/capability-seed-embed-gate.test.ts +186 -0
- package/src/plugins/defaults/memory/__tests__/fixtures/memory-v2-activation-fixtures.ts +1 -1
- package/src/plugins/defaults/memory/__tests__/indexer-tier-dispatch.test.ts +129 -0
- package/src/plugins/defaults/memory/__tests__/jobs-worker-quiesce-gate.test.ts +91 -0
- package/src/plugins/defaults/memory/__tests__/jobs-worker-v1-graph-substrate-noop.test.ts +342 -0
- package/src/plugins/defaults/memory/__tests__/jobs-worker-v2-graph-trigger-embed.test.ts +9 -1
- package/src/plugins/defaults/memory/__tests__/jobs-worker-v2-schedule.test.ts +36 -1
- package/src/plugins/defaults/memory/__tests__/jobs-worker-v2-sweep-defer.test.ts +9 -1
- package/src/plugins/defaults/memory/__tests__/memory-log-stores-degraded.test.ts +2 -2
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-job.test.ts +44 -10
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-sweep.test.ts +151 -10
- package/src/plugins/defaults/memory/__tests__/memory-tier-boundary-guard.test.ts +1155 -0
- package/src/plugins/defaults/memory/__tests__/memory-v2-activation-log-store.test.ts +1 -1
- package/src/plugins/defaults/memory/__tests__/memory-v2-concept-frequency.test.ts +3 -3
- package/src/plugins/defaults/memory/__tests__/tools-delete-memory-page.test.ts +1 -1
- package/src/plugins/defaults/memory/context-search/sources/memory-v2.ts +71 -35
- package/src/plugins/defaults/memory/context-search/sources/memory.ts +5 -1
- package/src/plugins/defaults/memory/fork-conversation-memory.ts +4 -4
- package/src/plugins/defaults/memory/graph/__tests__/conversation-graph-memory-v2-routing.test.ts +80 -10
- package/src/plugins/defaults/memory/graph/__tests__/handle-remember-v2.test.ts +1 -1
- package/src/plugins/defaults/memory/graph/capability-seed.ts +24 -10
- package/src/plugins/defaults/memory/graph/conversation-graph-memory.ts +123 -49
- package/src/plugins/defaults/memory/graph/graph-search.ts +6 -118
- package/src/plugins/defaults/memory/graph/in-context-tracker.test.ts +173 -0
- package/src/plugins/defaults/memory/graph/in-context-tracker.ts +109 -0
- package/src/plugins/defaults/memory/graph/store.ts +1 -0
- package/src/plugins/defaults/memory/graph/tool-handlers.test.ts +1 -1
- package/src/plugins/defaults/memory/graph/tool-handlers.ts +5 -2
- package/src/plugins/defaults/memory/graph-topology/__tests__/build-memory-graph.test.ts +1 -1
- package/src/plugins/defaults/memory/graph-topology/build-memory-graph.ts +6 -9
- package/src/plugins/defaults/memory/hooks/__tests__/user-prompt-submit.test.ts +30 -16
- package/src/plugins/defaults/memory/hooks/user-prompt-submit.ts +18 -14
- package/src/plugins/defaults/memory/indexer.ts +146 -89
- package/src/plugins/defaults/memory/injectors.ts +86 -41
- package/src/plugins/defaults/memory/job-handlers.ts +130 -39
- package/src/plugins/defaults/memory/jobs/__tests__/embed-concept-page.test.ts +2 -2
- package/src/plugins/defaults/memory/jobs/embed-concept-page.ts +22 -10
- package/src/plugins/defaults/memory/jobs-worker.ts +279 -208
- package/src/plugins/defaults/memory/memory-retrospective-enqueue.ts +4 -2
- package/src/plugins/defaults/memory/memory-retrospective-job.ts +40 -3
- package/src/plugins/defaults/memory/memory-retrospective-sweep.ts +99 -11
- package/src/plugins/defaults/memory/src/__tests__/memory-v2-routes.test.ts +5 -3
- package/src/plugins/defaults/memory/src/__tests__/memory-v2-simulate-route.test.ts +3 -4
- package/src/plugins/defaults/memory/src/memory-graph-routes.ts +2 -1
- package/src/plugins/defaults/memory/src/memory-item-routes.test.ts +7 -5
- package/src/plugins/defaults/memory/src/memory-item-routes.ts +51 -19
- package/src/plugins/defaults/memory/src/memory-v2-routes.ts +31 -20
- package/src/plugins/defaults/memory/startup.ts +58 -21
- package/src/plugins/defaults/memory/substrate/__tests__/boot-maintenance.test.ts +335 -0
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/cli-command-store.test.ts +14 -8
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/consolidation-job.test.ts +51 -21
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/edge-index.test.ts +1 -1
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/frontmatter-sweep.test.ts +3 -3
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/page-index.test.ts +1 -1
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/page-store.test.ts +1 -1
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/prompts-consolidation.test.ts +2 -2
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/qdrant.test.ts +23 -14
- package/src/plugins/defaults/memory/substrate/__tests__/reembed-job.test.ts +262 -0
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/sim.test.ts +9 -12
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/skill-content.test.ts +2 -2
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/skill-store.test.ts +9 -9
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/sparse-bm25.test.ts +12 -5
- package/src/plugins/defaults/memory/substrate/__tests__/spread.test.ts +276 -0
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/static-context.test.ts +17 -8
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/sweep-job.test.ts +9 -9
- package/src/plugins/defaults/memory/{v3/substrate/memory-v2-startup.ts → substrate/boot-maintenance.ts} +43 -24
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/cli-command-store.ts +13 -8
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/consolidation-job.ts +51 -27
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/constants.ts +1 -0
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/edge-index.ts +28 -9
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/frontmatter-sweep.ts +14 -7
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/injected-block-slugs.ts +9 -3
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/page-index.ts +29 -10
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/page-store.ts +35 -12
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/prompts/consolidation.ts +14 -8
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/prompts/sweep.ts +4 -1
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/qdrant.ts +96 -36
- package/src/plugins/defaults/memory/substrate/reembed-job.ts +54 -0
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/sim.ts +26 -15
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/skill-content.ts +1 -1
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/skill-store.ts +13 -8
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/sparse-bm25.ts +27 -10
- package/src/plugins/defaults/memory/substrate/spread.ts +165 -0
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/static-context.ts +12 -7
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/sweep-job.ts +20 -15
- package/src/plugins/defaults/memory/substrate/tuning.ts +93 -0
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/types.ts +7 -4
- package/src/plugins/defaults/memory/tools.test.ts +1 -1
- package/src/plugins/defaults/memory/tools.ts +1 -1
- package/src/plugins/defaults/memory/v1/README.md +5 -0
- package/src/plugins/defaults/memory/{__tests__ → v1/__tests__}/filing-jobs.test.ts +4 -4
- package/src/plugins/defaults/memory/{filing-jobs.ts → v1/filing-jobs.ts} +3 -3
- package/src/plugins/defaults/memory/{graph → v1/graph}/bootstrap.test.ts +68 -6
- package/src/plugins/defaults/memory/{graph → v1/graph}/bootstrap.ts +40 -21
- package/src/plugins/defaults/memory/{graph → v1/graph}/consolidation.ts +14 -14
- package/src/plugins/defaults/memory/{graph → v1/graph}/decay.test.ts +1 -1
- package/src/plugins/defaults/memory/{graph → v1/graph}/decay.ts +3 -3
- package/src/plugins/defaults/memory/{graph → v1/graph}/extraction-job.ts +5 -5
- package/src/plugins/defaults/memory/{graph → v1/graph}/extraction.test.ts +1 -1
- package/src/plugins/defaults/memory/{graph → v1/graph}/extraction.ts +20 -14
- package/src/plugins/defaults/memory/{graph → v1/graph}/graph-search.test.ts +3 -3
- package/src/plugins/defaults/memory/v1/graph/graph-search.ts +123 -0
- package/src/plugins/defaults/memory/{graph → v1/graph}/injection.test.ts +1 -118
- package/src/plugins/defaults/memory/{graph → v1/graph}/injection.ts +9 -111
- package/src/plugins/defaults/memory/{graph → v1/graph}/narrative.ts +6 -6
- package/src/plugins/defaults/memory/{graph → v1/graph}/pattern-scan.ts +5 -5
- package/src/plugins/defaults/memory/{graph → v1/graph}/retriever.test.ts +135 -42
- package/src/plugins/defaults/memory/{graph → v1/graph}/retriever.ts +40 -22
- package/src/plugins/defaults/memory/{graph → v1/graph}/scoring.test.ts +1 -1
- package/src/plugins/defaults/memory/{graph → v1/graph}/scoring.ts +1 -1
- package/src/plugins/defaults/memory/{graph → v1/graph}/serendipity.ts +1 -1
- package/src/plugins/defaults/memory/{graph → v1/graph}/triggers.test.ts +1 -1
- package/src/plugins/defaults/memory/{graph → v1/graph}/triggers.ts +1 -1
- package/src/plugins/defaults/memory/{identity-context.ts → v1/identity-context.ts} +4 -4
- package/src/plugins/defaults/memory/{job-handlers → v1/job-handlers}/backfill.ts +6 -6
- package/src/plugins/defaults/memory/{job-handlers → v1/job-handlers}/embedding.test.ts +6 -6
- package/src/plugins/defaults/memory/{job-handlers → v1/job-handlers}/embedding.ts +6 -6
- package/src/plugins/defaults/memory/{job-handlers → v1/job-handlers}/index-maintenance.ts +8 -8
- package/src/plugins/defaults/memory/{jobs → v1/jobs}/embed-pkb-file.test.ts +7 -7
- package/src/plugins/defaults/memory/{jobs → v1/jobs}/embed-pkb-file.ts +5 -5
- package/src/plugins/defaults/memory/{pkb → v1/pkb}/autoinject.ts +1 -1
- package/src/plugins/defaults/memory/{pkb → v1/pkb}/context.ts +2 -2
- package/src/plugins/defaults/memory/{pkb → v1/pkb}/pkb-index.test.ts +3 -3
- package/src/plugins/defaults/memory/{pkb → v1/pkb}/pkb-index.ts +2 -2
- package/src/plugins/defaults/memory/{pkb → v1/pkb}/pkb-reconcile.test.ts +3 -3
- package/src/plugins/defaults/memory/{pkb → v1/pkb}/pkb-reconcile.ts +2 -2
- package/src/plugins/defaults/memory/{pkb → v1/pkb}/pkb-search.test.ts +3 -3
- package/src/plugins/defaults/memory/{pkb → v1/pkb}/pkb-search.ts +6 -6
- package/src/plugins/defaults/memory/{pkb → v1/pkb}/types.ts +3 -5
- package/src/plugins/defaults/memory/{pkb-schedule.ts → v1/pkb-schedule.ts} +2 -2
- package/src/plugins/defaults/memory/{search/semantic.ts → v1/semantic-search.ts} +1 -1
- package/src/plugins/defaults/memory/v2/__tests__/activation-store.test.ts +1 -1
- package/src/plugins/defaults/memory/v2/__tests__/activation.test.ts +11 -274
- package/src/plugins/defaults/memory/v2/__tests__/backfill-jobs.test.ts +9 -131
- package/src/plugins/defaults/memory/v2/__tests__/harness-compare.test.ts +1 -1
- package/src/plugins/defaults/memory/v2/__tests__/harness-oracle.test.ts +1 -1
- package/src/plugins/defaults/memory/v2/__tests__/harness-replay-input.test.ts +1 -1
- package/src/plugins/defaults/memory/v2/__tests__/injection.test.ts +14 -10
- package/src/plugins/defaults/memory/v2/__tests__/reranker.test.ts +7 -5
- package/src/plugins/defaults/memory/v2/__tests__/router.test.ts +12 -6
- package/src/plugins/defaults/memory/{memory-v2-activation-log-store.ts → v2/activation-log-store.ts} +2 -2
- package/src/plugins/defaults/memory/v2/activation-store.ts +22 -8
- package/src/plugins/defaults/memory/v2/activation.ts +30 -163
- package/src/plugins/defaults/memory/v2/backfill-jobs.ts +7 -54
- package/src/plugins/defaults/memory/{memory-v2-concept-frequency.ts → v2/concept-frequency.ts} +3 -3
- package/src/plugins/defaults/memory/v2/harness/oracle.ts +1 -1
- package/src/plugins/defaults/memory/v2/harness/replay-input.ts +2 -2
- package/src/plugins/defaults/memory/v2/harness/retriever.ts +1 -1
- package/src/plugins/defaults/memory/v2/injection.ts +36 -26
- package/src/plugins/defaults/memory/v2/migration.ts +2 -2
- package/src/plugins/defaults/memory/{rerank-local.ts → v2/rerank-local.ts} +44 -18
- package/src/plugins/defaults/memory/v2/reranker.ts +29 -11
- package/src/plugins/defaults/memory/v2/router.ts +38 -15
- package/src/plugins/defaults/memory/v3/__tests__/candidate-match.test.ts +5 -3
- package/src/plugins/defaults/memory/v3/__tests__/carry-integration.test.ts +21 -8
- package/src/plugins/defaults/memory/v3/__tests__/edge.test.ts +1 -1
- package/src/plugins/defaults/memory/v3/__tests__/live-integration.test.ts +1 -1
- package/src/plugins/defaults/memory/v3/__tests__/maintain-job.test.ts +4 -1
- package/src/plugins/defaults/memory/v3/__tests__/orchestrate.test.ts +181 -3
- package/src/plugins/defaults/memory/v3/__tests__/selection-log-store.test.ts +3 -0
- package/src/plugins/defaults/memory/v3/__tests__/shadow-integration.test.ts +14 -5
- package/src/plugins/defaults/memory/v3/__tests__/shadow-plugin.test.ts +40 -50
- package/src/plugins/defaults/memory/v3/__tests__/tuning-profile.test.ts +1 -0
- package/src/plugins/defaults/memory/v3/candidate-match.ts +2 -2
- package/src/plugins/defaults/memory/v3/capabilities.ts +6 -4
- package/src/plugins/defaults/memory/v3/card.ts +25 -9
- package/src/plugins/defaults/memory/v3/edge.ts +52 -18
- package/src/plugins/defaults/memory/v3/gate.ts +4 -4
- package/src/plugins/defaults/memory/v3/injector.ts +10 -3
- package/src/plugins/defaults/memory/v3/maintain-job.ts +3 -3
- package/src/plugins/defaults/memory/v3/orchestrate.ts +100 -18
- package/src/plugins/defaults/memory/v3/page-content.ts +26 -10
- package/src/plugins/defaults/memory/v3/prune.ts +40 -16
- package/src/plugins/defaults/memory/v3/selection-log-store.ts +34 -12
- package/src/plugins/defaults/memory/v3/shadow-plugin.ts +9 -13
- package/src/plugins/defaults/memory/v3/span-query.test.ts +86 -0
- package/src/plugins/defaults/memory/v3/span-query.ts +71 -0
- package/src/plugins/defaults/memory/v3/tuning-profile.ts +3 -0
- package/src/plugins/defaults/memory/v3/types.ts +4 -1
- package/src/plugins/defaults/memory/v3-eval/eval-packets.ts +2 -2
- package/src/plugins/defaults/memory/validation.ts +1 -1
- package/src/plugins/defaults/memory/worker.ts +2 -1
- package/src/plugins/defaults/platform-hosted/__tests__/reengage.test.ts +104 -12
- package/src/plugins/defaults/platform-hosted/routes/reengage.ts +104 -12
- package/src/plugins/types.ts +2 -2
- package/src/prompts/sections.ts +4 -16
- package/src/providers/__tests__/registry-native-web-search.test.ts +47 -2
- package/src/providers/anthropic/client.ts +15 -6
- package/src/providers/content-block-schema.ts +26 -0
- package/src/providers/inference/__tests__/connections-credential-normalize.test.ts +87 -0
- package/src/providers/inference/adapter-factory.ts +6 -0
- package/src/providers/inference/connections.ts +30 -14
- package/src/providers/model-catalog.ts +77 -0
- package/src/providers/openai/__tests__/chat-completions-input-audio.test.ts +260 -0
- package/src/providers/openai/chat-completions-provider.ts +65 -17
- package/src/providers/openai/input-audio.ts +76 -0
- package/src/providers/openrouter/client.ts +0 -2
- package/src/providers/platform-proxy/constants.ts +4 -0
- package/src/providers/poolside/client.ts +42 -0
- package/src/providers/registry.ts +40 -8
- package/src/providers/retry.ts +2 -0
- package/src/providers/types.ts +39 -1
- package/src/providers/vercel-ai-gateway/client.ts +0 -2
- package/src/runtime/AGENTS.md +7 -5
- package/src/runtime/__tests__/question-request-guardian-bridge.test.ts +173 -0
- package/src/runtime/__tests__/question-resolution.test.ts +111 -0
- package/src/runtime/access-request-helper.ts +7 -15
- package/src/runtime/assistant-event-hub.ts +43 -31
- package/src/runtime/assistant-event-publish-options.ts +31 -0
- package/src/runtime/assistant-event.ts +19 -30
- package/src/runtime/assistant-stream-state.ts +4 -4
- package/src/runtime/channel-approvals.ts +0 -25
- package/src/runtime/channel-readiness-service.ts +99 -40
- package/src/runtime/channel-retry-sweep.ts +38 -15
- package/src/runtime/confirmation-request-guardian-bridge.ts +4 -0
- package/src/runtime/guardian-reply-router.ts +108 -0
- package/src/runtime/http-server.ts +8 -0
- package/src/runtime/http-types.ts +2 -2
- package/src/runtime/process-role.ts +42 -0
- package/src/runtime/question-request-guardian-bridge.ts +184 -0
- package/src/runtime/question-resolution.ts +153 -0
- package/src/runtime/routes/__tests__/bookmark-routes.test.ts +6 -2
- package/src/runtime/routes/__tests__/conversation-query-routes.test.ts +1 -1
- package/src/runtime/routes/__tests__/subagents-routes.test.ts +250 -0
- package/src/runtime/routes/__tests__/webhook-routes.test.ts +201 -0
- package/src/runtime/routes/acp-routes.ts +2 -2
- package/src/runtime/routes/bookmark-routes.ts +3 -15
- package/src/runtime/routes/canned-message-complete.ts +2 -2
- package/src/runtime/routes/consolidation-routes.ts +8 -2
- package/src/runtime/routes/conversation-list-routes.ts +3 -19
- package/src/runtime/routes/conversation-management-routes.ts +1 -5
- package/src/runtime/routes/conversation-query-routes.ts +52 -8
- package/src/runtime/routes/conversation-routes.ts +36 -22
- package/src/runtime/routes/credential-request-routes.ts +0 -1
- package/src/runtime/routes/filing-routes.ts +7 -7
- package/src/runtime/routes/global-search-routes.ts +1 -1
- package/src/runtime/routes/guardian-approval-prompt.ts +2 -2
- package/src/runtime/routes/image-generation-routes.ts +12 -9
- package/src/runtime/routes/inbound-message-handler.ts +17 -0
- package/src/runtime/routes/inbound-stages/acl-enforcement.test.ts +50 -0
- package/src/runtime/routes/inbound-stages/acl-enforcement.ts +19 -0
- package/src/runtime/routes/inbound-stages/background-dispatch.ts +4 -4
- package/src/runtime/routes/inbound-stages/inbound-content-prep.test.ts +129 -0
- package/src/runtime/routes/inbound-stages/inbound-content-prep.ts +141 -9
- package/src/runtime/routes/index.ts +2 -0
- package/src/runtime/routes/lifecycle-routes.ts +199 -0
- package/src/runtime/routes/platform-routes.ts +3 -8
- package/src/runtime/routes/playground/__tests__/inject-failures.test.ts +7 -7
- package/src/runtime/routes/playground/__tests__/reset-circuit.test.ts +5 -5
- package/src/runtime/routes/question-routes.ts +34 -87
- package/src/runtime/routes/secret-routes.ts +11 -1
- package/src/runtime/routes/settings-routes.ts +8 -18
- package/src/runtime/routes/subagents-routes.ts +35 -8
- package/src/runtime/routes/upgrade-broadcast-routes.ts +4 -5
- package/src/runtime/routes/webhook-routes.ts +89 -30
- package/src/runtime/routes/workspace-routes.test.ts +7 -4
- package/src/runtime/slack-reply-session.test.ts +12 -12
- package/src/runtime/slack-reply-session.ts +17 -17
- package/src/runtime/sync/sync-publisher.test.ts +7 -5
- package/src/runtime/tool-grant-request-helper.ts +4 -0
- package/src/schedule/schedule-store.ts +70 -0
- package/src/schedule/scheduler.ts +27 -0
- package/src/security/AGENTS.md +1 -1
- package/src/security/credential-key.ts +17 -0
- package/src/security/secret-ingress.ts +14 -105
- package/src/security/secret-patterns.ts +8 -139
- package/src/security/secret-scanner.ts +8 -3
- package/src/security/store-path-override.ts +6 -16
- package/src/sequence/store.ts +8 -0
- package/src/signals/emit-event.ts +5 -10
- package/src/signals/event-stream.ts +20 -10
- package/src/subagent/manager.ts +76 -25
- package/src/subagent/notify.ts +8 -0
- package/src/subagent/types.ts +13 -3
- package/src/telegram/__tests__/webhook-health.test.ts +442 -0
- package/src/telegram/webhook-health.ts +478 -0
- package/src/telemetry/AGENTS.md +6 -0
- package/src/telemetry/memory-corpus-size.test.ts +129 -0
- package/src/telemetry/memory-corpus-size.ts +184 -0
- package/src/telemetry/memory-tier-reporter.test.ts +67 -2
- package/src/telemetry/memory-tier-reporter.ts +30 -3
- package/src/tools/__tests__/conversation-group-tool-input-schemas.test.ts +61 -0
- package/src/tools/__tests__/document-tool-input-schemas.test.ts +103 -0
- package/src/tools/__tests__/followup-call-acp-tool-input-schemas.test.ts +125 -0
- package/src/tools/__tests__/schedule-tool-input-schemas.test.ts +83 -0
- package/src/tools/__tests__/subagent-workflow-tool-input-schemas.test.ts +116 -0
- package/src/tools/__tests__/tool-input-schemas.test.ts +192 -0
- package/src/tools/__tests__/tool-schema-drift-harness.ts +142 -0
- package/src/tools/acp/abort.ts +21 -1
- package/src/tools/acp/context.ts +5 -5
- package/src/tools/acp/spawn.test.ts +15 -5
- package/src/tools/acp/spawn.ts +26 -3
- package/src/tools/acp/status.ts +20 -1
- package/src/tools/acp/steer.test.ts +20 -0
- package/src/tools/acp/steer.ts +23 -2
- package/src/tools/apps/resolve-app-id.ts +15 -3
- package/src/tools/ask-question/ask-question-tool.test.ts +65 -0
- package/src/tools/ask-question/ask-question-tool.ts +68 -92
- package/src/tools/browser/browser-screencast.ts +6 -6
- package/src/tools/calls/call-end.ts +24 -3
- package/src/tools/calls/call-start.ts +35 -11
- package/src/tools/calls/call-status.ts +20 -1
- package/src/tools/conversation-groups/group_create.ts +63 -0
- package/src/tools/conversation-groups/group_list.ts +20 -0
- package/src/tools/conversation-groups/group_shared.ts +52 -0
- package/src/tools/conversation-groups/move_to_group.ts +106 -0
- package/src/tools/document/document-comment-tool.test.ts +49 -0
- package/src/tools/document/document-comment-tool.ts +45 -6
- package/src/tools/document/document-tool.ts +142 -24
- package/src/tools/executor.ts +27 -13
- package/src/tools/filesystem/edit.ts +62 -60
- package/src/tools/filesystem/list.ts +36 -29
- package/src/tools/filesystem/read.ts +56 -39
- package/src/tools/filesystem/write.ts +44 -38
- package/src/tools/followups/followup_create.ts +41 -12
- package/src/tools/followups/followup_list.ts +34 -6
- package/src/tools/followups/followup_resolve.ts +26 -4
- package/src/tools/host-filesystem/edit.ts +50 -60
- package/src/tools/host-filesystem/read.ts +45 -39
- package/src/tools/host-filesystem/write.ts +31 -36
- package/src/tools/host-terminal/host-shell.ts +76 -62
- package/src/tools/policy-context.ts +27 -10
- package/src/tools/schedule/create.ts +76 -33
- package/src/tools/schedule/delete.ts +21 -2
- package/src/tools/schedule/list.ts +25 -3
- package/src/tools/schedule/update.ts +105 -48
- package/src/tools/shared/filesystem/path-policy.ts +246 -40
- package/src/tools/shared/zod-tool-schema.ts +117 -0
- package/src/tools/subagent/abort.ts +12 -4
- package/src/tools/subagent/consult-deadline.ts +9 -3
- package/src/tools/subagent/message.ts +21 -5
- package/src/tools/subagent/notify-parent.ts +44 -23
- package/src/tools/subagent/read.ts +15 -4
- package/src/tools/subagent/resolve.ts +26 -4
- package/src/tools/subagent/spawn.ts +37 -13
- package/src/tools/subagent/status.ts +12 -4
- package/src/tools/system/request-permission.ts +35 -26
- package/src/tools/terminal/shell.ts +68 -55
- package/src/tools/tool-approval-handler.ts +327 -53
- package/src/tools/tool-input-schemas.ts +85 -0
- package/src/tools/tool-types.ts +7 -0
- package/src/tools/types.ts +8 -0
- package/src/tools/workflows/manage-workflows.ts +30 -2
- package/src/tools/workflows/run-workflow.test.ts +54 -2
- package/src/tools/workflows/run-workflow.ts +32 -4
- package/src/util/fs-symlinks.ts +30 -0
- package/src/util/log-redact.ts +7 -3
- package/src/util/platform.ts +15 -0
- package/src/vellum-assistant-namespace.d.ts +80 -0
- package/src/watcher/watcher-store.ts +8 -0
- package/src/workflows/leaf-runner.test.ts +104 -2
- package/src/workflows/leaf-runner.ts +31 -6
- package/src/workspace/migrations/070-memory-v2-summary-schema-rebuild.ts +2 -2
- package/src/workspace/migrations/134-image-generation-mode-to-provider.ts +128 -0
- package/src/workspace/migrations/135-copy-substrate-tunables.ts +248 -0
- package/src/workspace/migrations/__tests__/135-copy-substrate-tunables.test.ts +446 -0
- package/src/workspace/migrations/__tests__/backfill-v3-leaves.test.ts +2 -2
- package/src/workspace/migrations/registry.ts +4 -0
- package/tsconfig.plugin-api.json +1 -1
- package/src/__tests__/memory-retrieval-budget.test.ts +0 -49
- package/src/__tests__/secret-patterns.test.ts +0 -27
- package/src/config/__tests__/proc-to-skills.test.ts +0 -21
- package/src/config/memory-tier.test.ts +0 -44
- package/src/daemon/message-types/acp.ts +0 -20
- package/src/daemon/message-types/apps.ts +0 -11
- package/src/daemon/message-types/background-tools.ts +0 -14
- package/src/daemon/message-types/bookmarks.ts +0 -13
- package/src/daemon/message-types/contacts.ts +0 -13
- package/src/daemon/message-types/document-comments.ts +0 -14
- package/src/daemon/message-types/documents.ts +0 -13
- package/src/daemon/message-types/home.ts +0 -15
- package/src/daemon/message-types/host-bash.ts +0 -12
- package/src/daemon/message-types/host-cu.ts +0 -10
- package/src/daemon/message-types/host-file.ts +0 -12
- package/src/daemon/message-types/host-transfer.ts +0 -12
- package/src/daemon/message-types/integrations.ts +0 -90
- package/src/daemon/message-types/memory.ts +0 -10
- package/src/daemon/message-types/schedules.ts +0 -14
- package/src/daemon/message-types/settings.ts +0 -17
- package/src/daemon/message-types/subagents.ts +0 -63
- package/src/daemon/message-types/upgrades.ts +0 -14
- package/src/daemon/message-types/workflows.ts +0 -25
- package/src/daemon/message-types/workspace.ts +0 -137
- package/src/live-voice/ack-phrases.ts +0 -58
- package/src/plugins/defaults/memory/retrieval-budget.ts +0 -29
- package/src/plugins/defaults/memory/v3/__tests__/gate-flag.test.ts +0 -69
- package/src/plugins/defaults/memory/v3/gate-flag.ts +0 -20
- package/src/plugins/defaults/memory/v3/substrate/__tests__/memory-v2-startup.test.ts +0 -86
- package/src/plugins/defaults/memory/v3/substrate/memory-v2-startup.test.ts +0 -205
- /package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/concept-page-frontmatter-schema.test.ts +0 -0
- /package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/consolidation-prompt-flag-gating-guard.test.ts +0 -0
- /package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/injected-block-slugs.test.ts +0 -0
- /package/src/plugins/defaults/memory/{v3/substrate → substrate}/cli-command-content.ts +0 -0
|
@@ -1,14 +1,12 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { mkdirSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { beforeAll, describe, expect, mock, test } from "bun:test";
|
|
2
4
|
|
|
3
5
|
import type { TurnDetectorConfig } from "../../calls/media-turn-detector.js";
|
|
4
6
|
import type {
|
|
5
7
|
VoiceTurnCallbacks,
|
|
6
8
|
VoiceTurnOptions,
|
|
7
9
|
} from "../../calls/voice-session-bridge.js";
|
|
8
|
-
import {
|
|
9
|
-
clearCachedOverrides,
|
|
10
|
-
setCachedOverrides,
|
|
11
|
-
} from "../../config/feature-flag-cache.js";
|
|
12
10
|
import {
|
|
13
11
|
getConfig,
|
|
14
12
|
loadRawConfig,
|
|
@@ -19,12 +17,12 @@ import type {
|
|
|
19
17
|
StreamingTranscriber,
|
|
20
18
|
SttStreamServerEvent,
|
|
21
19
|
} from "../../stt/types.js";
|
|
22
|
-
import
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
} from "../front-decision.js";
|
|
20
|
+
import { __resetRegistryForTesting } from "../../tools/registry.js";
|
|
21
|
+
import { getWorkspaceSkillsDir } from "../../util/platform.js";
|
|
22
|
+
import type { VoiceFrontDecider } from "../front-decision.js";
|
|
26
23
|
import type { LiveVoiceAudioArchiveResult } from "../live-voice-archive.js";
|
|
27
24
|
import {
|
|
25
|
+
CONTINUATION_DELIVERY_CONTENT,
|
|
28
26
|
createLiveVoiceSession,
|
|
29
27
|
type LiveVoiceBackgroundContinuationSpawner,
|
|
30
28
|
LiveVoiceSession,
|
|
@@ -44,6 +42,20 @@ import {
|
|
|
44
42
|
type LiveVoiceServerFrame,
|
|
45
43
|
} from "../protocol.js";
|
|
46
44
|
|
|
45
|
+
// The closed-session delivery route writes into the parent conversation, which
|
|
46
|
+
// needs a live conversation store. Stub the seam so the assertion is about
|
|
47
|
+
// where the answer went, not about conversation plumbing.
|
|
48
|
+
//
|
|
49
|
+
// `mock.module` is process-global in Bun and leaks into sibling files that run
|
|
50
|
+
// in the same process — run this file on its own (`bun test <path>`).
|
|
51
|
+
const injectMessageIntoParentMock = mock(
|
|
52
|
+
(_parentConversationId: string, _message: string) => {},
|
|
53
|
+
);
|
|
54
|
+
mock.module("../../subagent/notify.js", () => ({
|
|
55
|
+
injectMessageIntoParent: injectMessageIntoParentMock,
|
|
56
|
+
notifyParentFromChild: mock(async () => {}),
|
|
57
|
+
}));
|
|
58
|
+
|
|
47
59
|
const SAMPLE_RATE = 24_000;
|
|
48
60
|
|
|
49
61
|
const VAD_START_FRAME = {
|
|
@@ -154,6 +166,7 @@ function createHarness(options: {
|
|
|
154
166
|
spawnBackgroundContinuation?: LiveVoiceBackgroundContinuationSpawner;
|
|
155
167
|
getTurnTeardown?: (conversationId: string) => Promise<void> | undefined;
|
|
156
168
|
detachTeardownSettleTimeoutMs?: number;
|
|
169
|
+
continuationAnnounceSilenceMs?: number;
|
|
157
170
|
}) {
|
|
158
171
|
const sequencer = createLiveVoiceServerFrameSequencer();
|
|
159
172
|
const frames: LiveVoiceServerFrame[] = [];
|
|
@@ -217,6 +230,11 @@ function createHarness(options: {
|
|
|
217
230
|
...(options.detachTeardownSettleTimeoutMs !== undefined
|
|
218
231
|
? { detachTeardownSettleTimeoutMs: options.detachTeardownSettleTimeoutMs }
|
|
219
232
|
: {}),
|
|
233
|
+
...(options.continuationAnnounceSilenceMs !== undefined
|
|
234
|
+
? {
|
|
235
|
+
continuationAnnounceSilenceMs: options.continuationAnnounceSilenceMs,
|
|
236
|
+
}
|
|
237
|
+
: {}),
|
|
220
238
|
};
|
|
221
239
|
const session = options.viaFactory
|
|
222
240
|
? createLiveVoiceSession(context, {
|
|
@@ -313,10 +331,79 @@ async function flushAsyncCallbacks(): Promise<void> {
|
|
|
313
331
|
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
314
332
|
}
|
|
315
333
|
|
|
334
|
+
/**
|
|
335
|
+
* Write a skill into the (per-process temp) workspace skills dir so the
|
|
336
|
+
* `skill_load` contention gate resolves it from the real on-disk catalog.
|
|
337
|
+
*/
|
|
338
|
+
function installSkillFixture(id: string, body: string): void {
|
|
339
|
+
const directory = join(getWorkspaceSkillsDir(), id);
|
|
340
|
+
mkdirSync(directory, { recursive: true });
|
|
341
|
+
writeFileSync(
|
|
342
|
+
join(directory, "SKILL.md"),
|
|
343
|
+
`---\nname: ${id}\ndescription: Fixture skill for the contention gate\n---\n\n${body}\n`,
|
|
344
|
+
);
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
/**
|
|
348
|
+
* Barge in during thinking so the interrupted turn detaches a continuation,
|
|
349
|
+
* then hand back the follow-up turn's options plus the continuation's abort
|
|
350
|
+
* signal — the two handles every foreground-wins assertion needs. The
|
|
351
|
+
* continuation hangs until its signal aborts, so it is still in flight while
|
|
352
|
+
* the follow-up turn starts tools.
|
|
353
|
+
*/
|
|
354
|
+
async function startForegroundWinsScenario(): Promise<{
|
|
355
|
+
followUp: VoiceTurnOptions | undefined;
|
|
356
|
+
signal: AbortSignal | undefined;
|
|
357
|
+
}> {
|
|
358
|
+
const spawnBackgroundContinuation = mock(
|
|
359
|
+
(args: {
|
|
360
|
+
parentConversationId: string;
|
|
361
|
+
objective: string;
|
|
362
|
+
label: string;
|
|
363
|
+
signal: AbortSignal;
|
|
364
|
+
}): Promise<string> =>
|
|
365
|
+
new Promise<string>((resolve) => {
|
|
366
|
+
args.signal.addEventListener("abort", () => resolve(""), {
|
|
367
|
+
once: true,
|
|
368
|
+
});
|
|
369
|
+
}),
|
|
370
|
+
);
|
|
371
|
+
const calls: VoiceTurnOptions[] = [];
|
|
372
|
+
const startVoiceTurn = mock(async (options: VoiceTurnOptions) => {
|
|
373
|
+
calls.push(options);
|
|
374
|
+
return { turnId: `bridge-turn-${calls.length}`, abort: mock() };
|
|
375
|
+
});
|
|
376
|
+
const streamTtsAudio = mock(async (options: LiveVoiceTtsOptions) => {
|
|
377
|
+
options.onAudioChunk(makeTtsChunk("assistant audio"));
|
|
378
|
+
return makeTtsResult("assistant audio");
|
|
379
|
+
});
|
|
380
|
+
const { frames, session } = createHarness({
|
|
381
|
+
finals: ["first question", "second question"],
|
|
382
|
+
startVoiceTurn,
|
|
383
|
+
streamTtsAudio,
|
|
384
|
+
spawnBackgroundContinuation,
|
|
385
|
+
});
|
|
386
|
+
|
|
387
|
+
await session.start();
|
|
388
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
389
|
+
await waitFor(() => frames.some((frame) => frame.type === "thinking"));
|
|
390
|
+
await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
|
|
391
|
+
await waitFor(() => spawnBackgroundContinuation.mock.calls.length === 1);
|
|
392
|
+
await waitFor(() => calls.some((c) => c.content === "second question"));
|
|
393
|
+
|
|
394
|
+
const signal = spawnBackgroundContinuation.mock.calls[0]?.[0]?.signal;
|
|
395
|
+
expect(signal?.aborted).toBe(false);
|
|
396
|
+
return {
|
|
397
|
+
followUp: calls.find((c) => c.content === "second question"),
|
|
398
|
+
signal,
|
|
399
|
+
};
|
|
400
|
+
}
|
|
401
|
+
|
|
316
402
|
describe("LiveVoiceSession server VAD", () => {
|
|
317
|
-
// The
|
|
318
|
-
//
|
|
319
|
-
|
|
403
|
+
// The foreground-wins classification consults the tool registry's owner map
|
|
404
|
+
// (a tool is only provably read-only when it is the trusted built-in);
|
|
405
|
+
// register the core baseline so built-in names resolve like in the daemon.
|
|
406
|
+
beforeAll(() => __resetRegistryForTesting());
|
|
320
407
|
|
|
321
408
|
test("ready echoes turnDetection server_vad", async () => {
|
|
322
409
|
const { frames, session } = createHarness({});
|
|
@@ -679,8 +766,7 @@ describe("LiveVoiceSession server VAD", () => {
|
|
|
679
766
|
expect(laterTurn?.voiceControlPrompt).not.toContain("first question");
|
|
680
767
|
});
|
|
681
768
|
|
|
682
|
-
test("
|
|
683
|
-
setCachedOverrides({ "voice-duplex-handoff": true }, { fromGateway: true });
|
|
769
|
+
test("a thinking barge-in spawns a background continuation", async () => {
|
|
684
770
|
const spawnBackgroundContinuation = mock(
|
|
685
771
|
async (_args: {
|
|
686
772
|
parentConversationId: string;
|
|
@@ -716,8 +802,7 @@ describe("LiveVoiceSession server VAD", () => {
|
|
|
716
802
|
expect(spawnArgs?.objective).toContain("first question");
|
|
717
803
|
});
|
|
718
804
|
|
|
719
|
-
test("
|
|
720
|
-
setCachedOverrides({ "voice-duplex-handoff": true }, { fromGateway: true });
|
|
805
|
+
test("a client interrupt aborts an in-flight continuation", async () => {
|
|
721
806
|
// Hang the continuation so it is still in flight when the interrupt lands;
|
|
722
807
|
// resolve it when its signal aborts.
|
|
723
808
|
const spawnBackgroundContinuation = mock(
|
|
@@ -758,8 +843,7 @@ describe("LiveVoiceSession server VAD", () => {
|
|
|
758
843
|
expect(signal?.aborted).toBe(true);
|
|
759
844
|
});
|
|
760
845
|
|
|
761
|
-
test("
|
|
762
|
-
setCachedOverrides({ "voice-duplex-handoff": true }, { fromGateway: true });
|
|
846
|
+
test("a client interrupt during barge-in cleanup skips the continuation", async () => {
|
|
763
847
|
const spawnBackgroundContinuation = mock(
|
|
764
848
|
async (_args: {
|
|
765
849
|
parentConversationId: string;
|
|
@@ -809,38 +893,7 @@ describe("LiveVoiceSession server VAD", () => {
|
|
|
809
893
|
expect(spawnBackgroundContinuation).not.toHaveBeenCalled();
|
|
810
894
|
});
|
|
811
895
|
|
|
812
|
-
test("
|
|
813
|
-
const spawnBackgroundContinuation = mock(
|
|
814
|
-
async (_args: {
|
|
815
|
-
parentConversationId: string;
|
|
816
|
-
objective: string;
|
|
817
|
-
label: string;
|
|
818
|
-
signal: AbortSignal;
|
|
819
|
-
}): Promise<string> => "",
|
|
820
|
-
);
|
|
821
|
-
const streamTtsAudio = mock(async (options: LiveVoiceTtsOptions) => {
|
|
822
|
-
options.onAudioChunk(makeTtsChunk("assistant audio"));
|
|
823
|
-
return makeTtsResult("assistant audio");
|
|
824
|
-
});
|
|
825
|
-
const { frames, session } = createHarness({
|
|
826
|
-
finals: ["first question", "second question"],
|
|
827
|
-
streamTtsAudio,
|
|
828
|
-
spawnBackgroundContinuation,
|
|
829
|
-
});
|
|
830
|
-
|
|
831
|
-
await session.start();
|
|
832
|
-
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
833
|
-
await waitFor(() => frames.some((frame) => frame.type === "thinking"));
|
|
834
|
-
await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
|
|
835
|
-
await waitFor(() =>
|
|
836
|
-
frames.some((frame) => frame.type === "turn_cancelled"),
|
|
837
|
-
);
|
|
838
|
-
await flushAsyncCallbacks();
|
|
839
|
-
expect(spawnBackgroundContinuation).not.toHaveBeenCalled();
|
|
840
|
-
});
|
|
841
|
-
|
|
842
|
-
test("voice-duplex-handoff on: no continuation when the model already completed", async () => {
|
|
843
|
-
setCachedOverrides({ "voice-duplex-handoff": true }, { fromGateway: true });
|
|
896
|
+
test("no continuation when the model already completed", async () => {
|
|
844
897
|
let callbacks: VoiceTurnCallbacks | undefined;
|
|
845
898
|
const spawnBackgroundContinuation = mock(
|
|
846
899
|
async (_args: {
|
|
@@ -892,8 +945,7 @@ describe("LiveVoiceSession server VAD", () => {
|
|
|
892
945
|
releaseTts?.();
|
|
893
946
|
});
|
|
894
947
|
|
|
895
|
-
test("
|
|
896
|
-
setCachedOverrides({ "voice-duplex-handoff": true }, { fromGateway: true });
|
|
948
|
+
test("the continuation fork waits for the interrupted turn's teardown to settle", async () => {
|
|
897
949
|
const spawnBackgroundContinuation = mock(
|
|
898
950
|
async (_args: {
|
|
899
951
|
parentConversationId: string;
|
|
@@ -941,8 +993,7 @@ describe("LiveVoiceSession server VAD", () => {
|
|
|
941
993
|
await waitFor(() => spawnBackgroundContinuation.mock.calls.length === 1);
|
|
942
994
|
});
|
|
943
995
|
|
|
944
|
-
test("
|
|
945
|
-
setCachedOverrides({ "voice-duplex-handoff": true }, { fromGateway: true });
|
|
996
|
+
test("the continuation is skipped when the teardown wait times out", async () => {
|
|
946
997
|
const spawnBackgroundContinuation = mock(
|
|
947
998
|
async (_args: {
|
|
948
999
|
parentConversationId: string;
|
|
@@ -983,8 +1034,7 @@ describe("LiveVoiceSession server VAD", () => {
|
|
|
983
1034
|
expect(spawnBackgroundContinuation).not.toHaveBeenCalled();
|
|
984
1035
|
});
|
|
985
1036
|
|
|
986
|
-
test("
|
|
987
|
-
setCachedOverrides({ "voice-duplex-handoff": true }, { fromGateway: true });
|
|
1037
|
+
test("a client interrupt during the teardown wait skips the continuation", async () => {
|
|
988
1038
|
const spawnBackgroundContinuation = mock(
|
|
989
1039
|
async (_args: {
|
|
990
1040
|
parentConversationId: string;
|
|
@@ -1048,8 +1098,21 @@ describe("LiveVoiceSession server VAD", () => {
|
|
|
1048
1098
|
return { startVoiceTurn, calls };
|
|
1049
1099
|
}
|
|
1050
1100
|
|
|
1051
|
-
|
|
1052
|
-
|
|
1101
|
+
// Records every turn's options and completes none of them, so every turn
|
|
1102
|
+
// stays "thinking" and a barge-in always lands on a live one.
|
|
1103
|
+
function makeNeverCompletingTurnStarter(): {
|
|
1104
|
+
startVoiceTurn: LiveVoiceTurnStarter;
|
|
1105
|
+
calls: VoiceTurnOptions[];
|
|
1106
|
+
} {
|
|
1107
|
+
const calls: VoiceTurnOptions[] = [];
|
|
1108
|
+
const startVoiceTurn = mock(async (options: VoiceTurnOptions) => {
|
|
1109
|
+
calls.push(options);
|
|
1110
|
+
return { turnId: `bridge-turn-${calls.length}`, abort: mock() };
|
|
1111
|
+
});
|
|
1112
|
+
return { startVoiceTurn, calls };
|
|
1113
|
+
}
|
|
1114
|
+
|
|
1115
|
+
test("a completed continuation's result folds into the next turn's control prompt", async () => {
|
|
1053
1116
|
// Control the continuation's resolution so we know exactly when its result
|
|
1054
1117
|
// is stashed relative to the turns we inspect.
|
|
1055
1118
|
let resolveContinuation: ((result: string) => void) | undefined;
|
|
@@ -1114,8 +1177,7 @@ describe("LiveVoiceSession server VAD", () => {
|
|
|
1114
1177
|
expect(later?.voiceControlPrompt).not.toContain("THE_RESULT");
|
|
1115
1178
|
});
|
|
1116
1179
|
|
|
1117
|
-
test("
|
|
1118
|
-
setCachedOverrides({ "voice-duplex-handoff": true }, { fromGateway: true });
|
|
1180
|
+
test("a client interrupt drops a stashed continuation result", async () => {
|
|
1119
1181
|
let resolveContinuation: ((result: string) => void) | undefined;
|
|
1120
1182
|
const spawnBackgroundContinuation = mock(
|
|
1121
1183
|
(_args: {
|
|
@@ -1163,8 +1225,7 @@ describe("LiveVoiceSession server VAD", () => {
|
|
|
1163
1225
|
expect(later?.voiceControlPrompt).not.toContain("THE_RESULT");
|
|
1164
1226
|
});
|
|
1165
1227
|
|
|
1166
|
-
test("
|
|
1167
|
-
setCachedOverrides({ "voice-duplex-handoff": true }, { fromGateway: true });
|
|
1228
|
+
test("an empty continuation result adds no context to the next turn", async () => {
|
|
1168
1229
|
// Continuation ends with no answer text (e.g. it stopped on a tool call):
|
|
1169
1230
|
// there is nothing to fold in.
|
|
1170
1231
|
const spawnBackgroundContinuation = mock(
|
|
@@ -1203,8 +1264,7 @@ describe("LiveVoiceSession server VAD", () => {
|
|
|
1203
1264
|
expect(later?.voiceControlPrompt).not.toContain("background you finished");
|
|
1204
1265
|
});
|
|
1205
1266
|
|
|
1206
|
-
test("
|
|
1207
|
-
setCachedOverrides({ "voice-duplex-handoff": true }, { fromGateway: true });
|
|
1267
|
+
test("a newer continuation's result survives an older one finishing later", async () => {
|
|
1208
1268
|
// Capture each continuation's resolver in spawn (detach) order.
|
|
1209
1269
|
const resolvers: Array<(result: string) => void> = [];
|
|
1210
1270
|
const spawnBackgroundContinuation = mock(
|
|
@@ -1273,8 +1333,7 @@ describe("LiveVoiceSession server VAD", () => {
|
|
|
1273
1333
|
expect(resurfaced?.voiceControlPrompt).not.toContain("OLDER_RESULT");
|
|
1274
1334
|
});
|
|
1275
1335
|
|
|
1276
|
-
test("
|
|
1277
|
-
setCachedOverrides({ "voice-duplex-handoff": true }, { fromGateway: true });
|
|
1336
|
+
test("a rapid second barge-in invalidates a first continuation that finishes after it", async () => {
|
|
1278
1337
|
const resolvers: Array<(result: string) => void> = [];
|
|
1279
1338
|
const spawnBackgroundContinuation = mock(
|
|
1280
1339
|
(_args: {
|
|
@@ -1325,8 +1384,13 @@ describe("LiveVoiceSession server VAD", () => {
|
|
|
1325
1384
|
await waitFor(() => calls.some((c) => c.content === "second question"));
|
|
1326
1385
|
// Barge #2 is a rapid second interruption: it bumps the sequence
|
|
1327
1386
|
// synchronously, so A is invalidated even before barge #2's own async detach
|
|
1328
|
-
// runs
|
|
1387
|
+
// runs — and A's run itself is ABORTED, so a still-writing older
|
|
1388
|
+
// continuation can never overlap the newer full-ability one.
|
|
1329
1389
|
await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
|
|
1390
|
+
await waitFor(
|
|
1391
|
+
() =>
|
|
1392
|
+
spawnBackgroundContinuation.mock.calls[0]?.[0]?.signal.aborted === true,
|
|
1393
|
+
);
|
|
1330
1394
|
// A finishes AFTER barge #2 and must be rejected. The newer continuation is
|
|
1331
1395
|
// left pending so it can't mask the bug by superseding A itself.
|
|
1332
1396
|
resolvers[0]?.("OLDER_RESULT");
|
|
@@ -1342,13 +1406,16 @@ describe("LiveVoiceSession server VAD", () => {
|
|
|
1342
1406
|
"background you finished",
|
|
1343
1407
|
);
|
|
1344
1408
|
|
|
1345
|
-
//
|
|
1409
|
+
// The newer continuation was not collateral damage of superseding A.
|
|
1346
1410
|
await waitFor(() => resolvers.length === 2);
|
|
1411
|
+
expect(spawnBackgroundContinuation.mock.calls[1]?.[0]?.signal.aborted).toBe(
|
|
1412
|
+
false,
|
|
1413
|
+
);
|
|
1414
|
+
// Cleanup the still-pending newer continuation.
|
|
1347
1415
|
resolvers[1]?.("");
|
|
1348
1416
|
});
|
|
1349
1417
|
|
|
1350
|
-
test("
|
|
1351
|
-
setCachedOverrides({ "voice-duplex-handoff": true }, { fromGateway: true });
|
|
1418
|
+
test("a new barge-in drops an already-stashed continuation result", async () => {
|
|
1352
1419
|
const resolvers: Array<(result: string) => void> = [];
|
|
1353
1420
|
const spawnBackgroundContinuation = mock(
|
|
1354
1421
|
(_args: {
|
|
@@ -1415,191 +1482,1050 @@ describe("LiveVoiceSession server VAD", () => {
|
|
|
1415
1482
|
resolvers[1]?.("");
|
|
1416
1483
|
});
|
|
1417
1484
|
|
|
1418
|
-
test("a
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
startVoiceTurn,
|
|
1429
|
-
// Suspend the async barge-in teardown at the turn_cancelled send so the
|
|
1430
|
-
// aborted turn stays non-finalized — the exact window a late model delta
|
|
1431
|
-
// could race into before cancelAssistantTurn finishes.
|
|
1432
|
-
holdSendFrame: (payload) => {
|
|
1433
|
-
if (payload.type !== "turn_cancelled" || releaseTurnCancelled) {
|
|
1434
|
-
return null;
|
|
1435
|
-
}
|
|
1436
|
-
return new Promise<void>((resolve) => {
|
|
1437
|
-
releaseTurnCancelled = resolve;
|
|
1438
|
-
});
|
|
1439
|
-
},
|
|
1485
|
+
test("a foreground consequential tool start aborts an in-flight continuation", async () => {
|
|
1486
|
+
const { followUp, signal } = await startForegroundWinsScenario();
|
|
1487
|
+
|
|
1488
|
+
// The follow-up turn starts a consequential tool: foreground wins the
|
|
1489
|
+
// workspace, so the still-running continuation is aborted before the two
|
|
1490
|
+
// can race on writes. `remember` is deliberately NOT on the core
|
|
1491
|
+
// side-effect name list — the classification must fail closed on any
|
|
1492
|
+
// tool that is not provably read-only, not just named writers.
|
|
1493
|
+
followUp?.callbacks?.tool_use_start?.("remember", {
|
|
1494
|
+
toolUseId: "tool-1",
|
|
1440
1495
|
});
|
|
1496
|
+
expect(signal?.aborted).toBe(true);
|
|
1497
|
+
});
|
|
1441
1498
|
|
|
1442
|
-
|
|
1443
|
-
await
|
|
1444
|
-
await waitFor(() => callbacks !== undefined);
|
|
1499
|
+
test("a read-only built-in leaves the continuation running; an unclassified tool fails closed", async () => {
|
|
1500
|
+
const { followUp, signal } = await startForegroundWinsScenario();
|
|
1445
1501
|
|
|
1446
|
-
//
|
|
1447
|
-
//
|
|
1448
|
-
|
|
1449
|
-
|
|
1502
|
+
// A provably read-only built-in cannot contend on the workspace: the
|
|
1503
|
+
// continuation survives (this is the topic-change case the handoff
|
|
1504
|
+
// exists for).
|
|
1505
|
+
followUp?.callbacks?.tool_use_start?.("file_read", { toolUseId: "tool-1" });
|
|
1506
|
+
expect(signal?.aborted).toBe(false);
|
|
1450
1507
|
|
|
1451
|
-
//
|
|
1452
|
-
//
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1508
|
+
// web_fetch is a network read: it cannot corrupt local state, so it does
|
|
1509
|
+
// not contend even though it is on the core SIDE_EFFECT_TOOLS list (which
|
|
1510
|
+
// exists for a permission question, not a contention one). This is the
|
|
1511
|
+
// canonical topic-change case — "what's the weather?" over a running
|
|
1512
|
+
// build — and it is exactly what the handoff exists to keep alive.
|
|
1513
|
+
followUp?.callbacks?.tool_use_start?.("web_fetch", {
|
|
1514
|
+
toolUseId: "tool-3",
|
|
1515
|
+
});
|
|
1516
|
+
expect(signal?.aborted).toBe(false);
|
|
1456
1517
|
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1518
|
+
// A tool the registry does not know (e.g. an MCP or plugin tool) is not
|
|
1519
|
+
// provably non-contending: fail closed and abort.
|
|
1520
|
+
followUp?.callbacks?.tool_use_start?.("calendar_lookup", {
|
|
1521
|
+
toolUseId: "tool-4",
|
|
1522
|
+
});
|
|
1523
|
+
expect(signal?.aborted).toBe(true);
|
|
1462
1524
|
});
|
|
1463
1525
|
|
|
1464
|
-
test("
|
|
1465
|
-
|
|
1466
|
-
const
|
|
1467
|
-
const startVoiceTurn = mock(async (options: VoiceTurnOptions) => {
|
|
1468
|
-
callbacks ??= options.callbacks;
|
|
1469
|
-
return { turnId: "bridge-turn", abort };
|
|
1470
|
-
});
|
|
1471
|
-
let releaseFirstDelta: (() => void) | undefined;
|
|
1472
|
-
// No TTS streamer: the turn emits text but never leaves the pre-TTS phase.
|
|
1473
|
-
const { frames, session } = createHarness({
|
|
1474
|
-
finals: ["first question", "second question"],
|
|
1475
|
-
startVoiceTurn,
|
|
1476
|
-
streamTtsAudio: null,
|
|
1477
|
-
// Hold the first assistant_text_delta's transport write so the second
|
|
1478
|
-
// one sits queued behind it (a backed-up outbound queue).
|
|
1479
|
-
holdSendFrame: (payload) => {
|
|
1480
|
-
if (
|
|
1481
|
-
payload.type !== "assistant_text_delta" ||
|
|
1482
|
-
releaseFirstDelta !== undefined
|
|
1483
|
-
) {
|
|
1484
|
-
return null;
|
|
1485
|
-
}
|
|
1486
|
-
return new Promise<void>((resolve) => {
|
|
1487
|
-
releaseFirstDelta = resolve;
|
|
1488
|
-
});
|
|
1489
|
-
},
|
|
1490
|
-
});
|
|
1526
|
+
test("an installed static skill_load leaves the continuation running", async () => {
|
|
1527
|
+
installSkillFixture("static-fixture-skill", "Plain instructions.");
|
|
1528
|
+
const { followUp, signal } = await startForegroundWinsScenario();
|
|
1491
1529
|
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1530
|
+
// Re-entering a skill that is installed locally and renders no inline
|
|
1531
|
+
// commands is a pure read — and it is the first call of nearly every
|
|
1532
|
+
// barge-in follow-up, so it must not kill the continuation.
|
|
1533
|
+
followUp?.callbacks?.tool_use_start?.("skill_load", {
|
|
1534
|
+
toolUseId: "tool-1",
|
|
1535
|
+
input: { skill: "static-fixture-skill" },
|
|
1536
|
+
});
|
|
1537
|
+
expect(signal?.aborted).toBe(false);
|
|
1495
1538
|
|
|
1496
|
-
//
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1539
|
+
// web_fetch is exempt by name regardless of input.
|
|
1540
|
+
followUp?.callbacks?.tool_use_start?.("web_fetch", {
|
|
1541
|
+
toolUseId: "tool-2",
|
|
1542
|
+
input: { url: "https://example.com" },
|
|
1543
|
+
});
|
|
1544
|
+
expect(signal?.aborted).toBe(false);
|
|
1501
1545
|
|
|
1502
|
-
//
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1546
|
+
// skill_execute is a dispatcher whose resolved inner tool can mutate.
|
|
1547
|
+
followUp?.callbacks?.tool_use_start?.("skill_execute", {
|
|
1548
|
+
toolUseId: "tool-3",
|
|
1549
|
+
input: { skill: "static-fixture-skill" },
|
|
1550
|
+
});
|
|
1551
|
+
expect(signal?.aborted).toBe(true);
|
|
1552
|
+
});
|
|
1506
1553
|
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
await waitFor(() =>
|
|
1512
|
-
frames.some((frame) => frame.type === "turn_cancelled"),
|
|
1554
|
+
test("a skill_load with inline command expansions aborts the continuation", async () => {
|
|
1555
|
+
installSkillFixture(
|
|
1556
|
+
"dynamic-fixture-skill",
|
|
1557
|
+
"Current status: !`git status --short`",
|
|
1513
1558
|
);
|
|
1514
|
-
|
|
1515
|
-
expect(
|
|
1516
|
-
frames.some(
|
|
1517
|
-
(frame) =>
|
|
1518
|
-
frame.type === "assistant_text_delta" && frame.text === "leaked tail",
|
|
1519
|
-
),
|
|
1520
|
-
).toBe(false);
|
|
1521
|
-
});
|
|
1559
|
+
const { followUp, signal } = await startForegroundWinsScenario();
|
|
1522
1560
|
|
|
1523
|
-
|
|
1524
|
-
//
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
await new Promise<void>((_resolve, reject) => {
|
|
1529
|
-
const fail = () =>
|
|
1530
|
-
reject(new Error("turn aborted while waiting for the lock"));
|
|
1531
|
-
if (options.signal?.aborted) {
|
|
1532
|
-
fail();
|
|
1533
|
-
return;
|
|
1534
|
-
}
|
|
1535
|
-
options.signal?.addEventListener("abort", fail, { once: true });
|
|
1536
|
-
});
|
|
1537
|
-
return { turnId: "bridge-turn", abort: mock() };
|
|
1538
|
-
};
|
|
1539
|
-
let releaseTurnCancelled: (() => void) | undefined;
|
|
1540
|
-
const { frames, session } = createHarness({
|
|
1541
|
-
finals: ["first question", "second question"],
|
|
1542
|
-
startVoiceTurn,
|
|
1543
|
-
holdSendFrame: (payload) => {
|
|
1544
|
-
if (payload.type !== "turn_cancelled" || releaseTurnCancelled) {
|
|
1545
|
-
return null;
|
|
1546
|
-
}
|
|
1547
|
-
return new Promise<void>((resolve) => {
|
|
1548
|
-
releaseTurnCancelled = resolve;
|
|
1549
|
-
});
|
|
1550
|
-
},
|
|
1561
|
+
// Inline expansions run shell at load time, so this load writes to the
|
|
1562
|
+
// host and contends like any other mutator.
|
|
1563
|
+
followUp?.callbacks?.tool_use_start?.("skill_load", {
|
|
1564
|
+
toolUseId: "tool-1",
|
|
1565
|
+
input: { skill: "dynamic-fixture-skill" },
|
|
1551
1566
|
});
|
|
1567
|
+
expect(signal?.aborted).toBe(true);
|
|
1568
|
+
});
|
|
1552
1569
|
|
|
1553
|
-
|
|
1554
|
-
await
|
|
1555
|
-
await waitFor(() => frames.some((frame) => frame.type === "thinking"));
|
|
1556
|
-
|
|
1557
|
-
// Barge in while the turn's start is still pending on the lock: the abort
|
|
1558
|
-
// rejects startVoiceTurn, whose catch must treat the aborted turn as dead
|
|
1559
|
-
// rather than surface a stray error frame while teardown is in flight.
|
|
1560
|
-
await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
|
|
1561
|
-
await waitFor(() => releaseTurnCancelled !== undefined);
|
|
1562
|
-
await flushAsyncCallbacks();
|
|
1563
|
-
expect(countType(frames, "error")).toBe(0);
|
|
1570
|
+
test("a skill_load with no input aborts the continuation", async () => {
|
|
1571
|
+
const { followUp, signal } = await startForegroundWinsScenario();
|
|
1564
1572
|
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1573
|
+
// Without the input there is no way to tell a pure read from an
|
|
1574
|
+
// auto-installing load: fail closed.
|
|
1575
|
+
followUp?.callbacks?.tool_use_start?.("skill_load", {
|
|
1576
|
+
toolUseId: "tool-1",
|
|
1577
|
+
});
|
|
1578
|
+
expect(signal?.aborted).toBe(true);
|
|
1570
1579
|
});
|
|
1571
1580
|
|
|
1572
|
-
test("a
|
|
1573
|
-
let
|
|
1574
|
-
const
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
|
|
1581
|
+
test("a foreground-wins abort keeps an already-stashed continuation result", async () => {
|
|
1582
|
+
let resolveContinuation: ((result: string) => void) | undefined;
|
|
1583
|
+
const spawnBackgroundContinuation = mock(
|
|
1584
|
+
(_args: {
|
|
1585
|
+
parentConversationId: string;
|
|
1586
|
+
objective: string;
|
|
1587
|
+
label: string;
|
|
1588
|
+
signal: AbortSignal;
|
|
1589
|
+
}): Promise<string> =>
|
|
1590
|
+
new Promise<string>((resolve) => {
|
|
1591
|
+
resolveContinuation = resolve;
|
|
1592
|
+
}),
|
|
1593
|
+
);
|
|
1594
|
+
const { startVoiceTurn, calls } = makeNeverCompletingTurnStarter();
|
|
1579
1595
|
const streamTtsAudio = mock(async (options: LiveVoiceTtsOptions) => {
|
|
1580
1596
|
options.onAudioChunk(makeTtsChunk("assistant audio"));
|
|
1581
1597
|
return makeTtsResult("assistant audio");
|
|
1582
1598
|
});
|
|
1583
1599
|
const { frames, session } = createHarness({
|
|
1584
|
-
finals: ["first question", "second question"],
|
|
1600
|
+
finals: ["first question", "second question", "third question"],
|
|
1585
1601
|
startVoiceTurn,
|
|
1586
1602
|
streamTtsAudio,
|
|
1603
|
+
spawnBackgroundContinuation,
|
|
1587
1604
|
});
|
|
1588
1605
|
|
|
1589
1606
|
await session.start();
|
|
1590
1607
|
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
1591
1608
|
await waitFor(() => frames.some((frame) => frame.type === "thinking"));
|
|
1592
1609
|
|
|
1593
|
-
//
|
|
1594
|
-
//
|
|
1595
|
-
|
|
1596
|
-
await
|
|
1610
|
+
// Barge in; the follow-up turn launches, then the continuation finishes
|
|
1611
|
+
// and stashes its answer for the NEXT turn while the follow-up still runs.
|
|
1612
|
+
await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
|
|
1613
|
+
await waitFor(() => resolveContinuation !== undefined);
|
|
1614
|
+
await waitFor(() => calls.some((c) => c.content === "second question"));
|
|
1615
|
+
resolveContinuation?.("THE_RESULT");
|
|
1597
1616
|
await flushAsyncCallbacks();
|
|
1598
1617
|
|
|
1599
|
-
|
|
1600
|
-
|
|
1618
|
+
// The follow-up turn starts writing: the foreground-wins abort fires, but
|
|
1619
|
+
// a completed continuation's stashed answer cannot race anything and is
|
|
1620
|
+
// kept.
|
|
1621
|
+
const followUp = calls.find((c) => c.content === "second question");
|
|
1622
|
+
followUp?.callbacks?.tool_use_start?.("file_write", {
|
|
1623
|
+
toolUseId: "tool-1",
|
|
1624
|
+
});
|
|
1625
|
+
followUp?.callbacks?.assistant_text_delta?.(makeTextDelta("ok"));
|
|
1626
|
+
followUp?.callbacks?.message_complete?.(makeMessageComplete());
|
|
1627
|
+
await waitFor(() => frames.some((frame) => frame.type === "tts_done"));
|
|
1601
1628
|
|
|
1602
|
-
// The
|
|
1629
|
+
// The next turn still folds the stashed answer in as context.
|
|
1630
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
1631
|
+
await waitFor(() => calls.some((c) => c.content === "third question"));
|
|
1632
|
+
const resurfaced = calls.find((c) => c.content === "third question");
|
|
1633
|
+
expect(resurfaced?.voiceControlPrompt).toContain("THE_RESULT");
|
|
1634
|
+
});
|
|
1635
|
+
|
|
1636
|
+
// A continuation spawner whose run resolves only when the test says so, so
|
|
1637
|
+
// the finished-continuation moment is exact.
|
|
1638
|
+
function makeControlledContinuation() {
|
|
1639
|
+
let resolveContinuation: ((result: string) => void) | undefined;
|
|
1640
|
+
const spawnBackgroundContinuation = mock(
|
|
1641
|
+
(_args: {
|
|
1642
|
+
parentConversationId: string;
|
|
1643
|
+
objective: string;
|
|
1644
|
+
label: string;
|
|
1645
|
+
signal: AbortSignal;
|
|
1646
|
+
}): Promise<string> =>
|
|
1647
|
+
new Promise<string>((resolve) => {
|
|
1648
|
+
resolveContinuation = resolve;
|
|
1649
|
+
}),
|
|
1650
|
+
);
|
|
1651
|
+
return {
|
|
1652
|
+
spawnBackgroundContinuation,
|
|
1653
|
+
finish: (result: string) => resolveContinuation?.(result),
|
|
1654
|
+
};
|
|
1655
|
+
}
|
|
1656
|
+
|
|
1657
|
+
function makeImmediateTts(): LiveVoiceTtsStreamer {
|
|
1658
|
+
return mock(async (options: LiveVoiceTtsOptions) => {
|
|
1659
|
+
options.onAudioChunk(makeTtsChunk("assistant audio"));
|
|
1660
|
+
return makeTtsResult("assistant audio");
|
|
1661
|
+
});
|
|
1662
|
+
}
|
|
1663
|
+
|
|
1664
|
+
// ~600 ms of 24 kHz mono PCM in a single chunk, so the client-side playback
|
|
1665
|
+
// tail estimate outlives the turn that produced it by a wide margin.
|
|
1666
|
+
function makeLongTailTts(): LiveVoiceTtsStreamer {
|
|
1667
|
+
const audio = "x".repeat(28_800);
|
|
1668
|
+
return mock(async (options: LiveVoiceTtsOptions) => {
|
|
1669
|
+
options.onAudioChunk(makeTtsChunk(audio));
|
|
1670
|
+
return makeTtsResult(audio);
|
|
1671
|
+
});
|
|
1672
|
+
}
|
|
1673
|
+
|
|
1674
|
+
function announcementOf(
|
|
1675
|
+
calls: VoiceTurnOptions[],
|
|
1676
|
+
): VoiceTurnOptions | undefined {
|
|
1677
|
+
return calls.find((c) => c.content === CONTINUATION_DELIVERY_CONTENT);
|
|
1678
|
+
}
|
|
1679
|
+
|
|
1680
|
+
function announcementCount(calls: VoiceTurnOptions[]): number {
|
|
1681
|
+
return calls.filter((c) => c.content === CONTINUATION_DELIVERY_CONTENT)
|
|
1682
|
+
.length;
|
|
1683
|
+
}
|
|
1684
|
+
|
|
1685
|
+
test("a continuation finishing on an idle live call is announced", async () => {
|
|
1686
|
+
const continuation = makeControlledContinuation();
|
|
1687
|
+
const { startVoiceTurn, calls } = makeResurfaceTurnStarter();
|
|
1688
|
+
const { frames, session } = createHarness({
|
|
1689
|
+
// The barge-in utterance transcribes to nothing, so no follow-up turn
|
|
1690
|
+
// runs and the call goes quiet.
|
|
1691
|
+
finals: ["first question", ""],
|
|
1692
|
+
startVoiceTurn,
|
|
1693
|
+
streamTtsAudio: makeImmediateTts(),
|
|
1694
|
+
spawnBackgroundContinuation: continuation.spawnBackgroundContinuation,
|
|
1695
|
+
continuationAnnounceSilenceMs: 20,
|
|
1696
|
+
});
|
|
1697
|
+
|
|
1698
|
+
await session.start();
|
|
1699
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
1700
|
+
await waitFor(() => frames.some((frame) => frame.type === "thinking"));
|
|
1701
|
+
await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
|
|
1702
|
+
await waitFor(
|
|
1703
|
+
() => continuation.spawnBackgroundContinuation.mock.calls.length === 1,
|
|
1704
|
+
);
|
|
1705
|
+
await waitFor(() =>
|
|
1706
|
+
frames.some((frame) => frame.type === "utterance_discarded"),
|
|
1707
|
+
);
|
|
1708
|
+
|
|
1709
|
+
// Nobody is speaking, so the session starts the delivery turn itself.
|
|
1710
|
+
continuation.finish("THE_RESULT");
|
|
1711
|
+
await waitFor(() => announcementOf(calls) !== undefined);
|
|
1712
|
+
const announcement = announcementOf(calls);
|
|
1713
|
+
expect(announcement?.voiceControlPrompt).toContain("THE_RESULT");
|
|
1714
|
+
expect(announcement?.voiceControlPrompt).toContain("first question");
|
|
1715
|
+
});
|
|
1716
|
+
|
|
1717
|
+
test("an announcement persists hidden and is never delivered twice", async () => {
|
|
1718
|
+
const continuation = makeControlledContinuation();
|
|
1719
|
+
const { startVoiceTurn, calls } = makeResurfaceTurnStarter();
|
|
1720
|
+
const { frames, session } = createHarness({
|
|
1721
|
+
finals: ["first question", "", "third question"],
|
|
1722
|
+
startVoiceTurn,
|
|
1723
|
+
streamTtsAudio: makeImmediateTts(),
|
|
1724
|
+
spawnBackgroundContinuation: continuation.spawnBackgroundContinuation,
|
|
1725
|
+
continuationAnnounceSilenceMs: 20,
|
|
1726
|
+
});
|
|
1727
|
+
|
|
1728
|
+
await session.start();
|
|
1729
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
1730
|
+
await waitFor(() => frames.some((frame) => frame.type === "thinking"));
|
|
1731
|
+
await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
|
|
1732
|
+
await waitFor(
|
|
1733
|
+
() => continuation.spawnBackgroundContinuation.mock.calls.length === 1,
|
|
1734
|
+
);
|
|
1735
|
+
await waitFor(() =>
|
|
1736
|
+
frames.some((frame) => frame.type === "utterance_discarded"),
|
|
1737
|
+
);
|
|
1738
|
+
|
|
1739
|
+
continuation.finish("THE_RESULT");
|
|
1740
|
+
await waitFor(() => announcementOf(calls) !== undefined);
|
|
1741
|
+
const announcement = announcementOf(calls);
|
|
1742
|
+
// The turn the user never started persists as an internal instruction, so
|
|
1743
|
+
// nothing renders as a user bubble; the answer rides the control prompt.
|
|
1744
|
+
expect(announcement?.content).toBe(CONTINUATION_DELIVERY_CONTENT);
|
|
1745
|
+
expect(announcement?.hiddenSyntheticPrompt).toBe(true);
|
|
1746
|
+
|
|
1747
|
+
// Announced, so the stash must not repeat it on the user's next turn.
|
|
1748
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
1749
|
+
await waitFor(() => calls.some((c) => c.content === "third question"));
|
|
1750
|
+
const later = calls.find((c) => c.content === "third question");
|
|
1751
|
+
expect(later?.voiceControlPrompt).not.toContain("THE_RESULT");
|
|
1752
|
+
});
|
|
1753
|
+
|
|
1754
|
+
test("a user utterance before the silence elapses cancels the announcement", async () => {
|
|
1755
|
+
const continuation = makeControlledContinuation();
|
|
1756
|
+
const { startVoiceTurn, calls } = makeResurfaceTurnStarter();
|
|
1757
|
+
const { frames, session } = createHarness({
|
|
1758
|
+
finals: ["first question", "", "third question"],
|
|
1759
|
+
startVoiceTurn,
|
|
1760
|
+
streamTtsAudio: makeImmediateTts(),
|
|
1761
|
+
spawnBackgroundContinuation: continuation.spawnBackgroundContinuation,
|
|
1762
|
+
// Long enough that the user always speaks first.
|
|
1763
|
+
continuationAnnounceSilenceMs: 5_000,
|
|
1764
|
+
});
|
|
1765
|
+
|
|
1766
|
+
await session.start();
|
|
1767
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
1768
|
+
await waitFor(() => frames.some((frame) => frame.type === "thinking"));
|
|
1769
|
+
await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
|
|
1770
|
+
await waitFor(
|
|
1771
|
+
() => continuation.spawnBackgroundContinuation.mock.calls.length === 1,
|
|
1772
|
+
);
|
|
1773
|
+
await waitFor(() =>
|
|
1774
|
+
frames.some((frame) => frame.type === "utterance_discarded"),
|
|
1775
|
+
);
|
|
1776
|
+
|
|
1777
|
+
continuation.finish("THE_RESULT");
|
|
1778
|
+
await flushAsyncCallbacks();
|
|
1779
|
+
|
|
1780
|
+
// The user speaks first: their turn delivers the answer through the stash,
|
|
1781
|
+
// and the announcement is dropped rather than repeating it after.
|
|
1782
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
1783
|
+
await waitFor(() => calls.some((c) => c.content === "third question"));
|
|
1784
|
+
const spoken = calls.find((c) => c.content === "third question");
|
|
1785
|
+
expect(spoken?.voiceControlPrompt).toContain("THE_RESULT");
|
|
1786
|
+
await flushAsyncCallbacks();
|
|
1787
|
+
expect(announcementOf(calls)).toBeUndefined();
|
|
1788
|
+
});
|
|
1789
|
+
|
|
1790
|
+
test("an active turn at fire time announces after the turn becomes idle", async () => {
|
|
1791
|
+
const continuation = makeControlledContinuation();
|
|
1792
|
+
const { startVoiceTurn, calls } = makeNeverCompletingTurnStarter();
|
|
1793
|
+
const { frames, session } = createHarness({
|
|
1794
|
+
// The barge-in utterance transcribes, so a follow-up turn holds the floor
|
|
1795
|
+
// for the whole announcement window.
|
|
1796
|
+
finals: ["first question", "second question", "third question"],
|
|
1797
|
+
startVoiceTurn,
|
|
1798
|
+
streamTtsAudio: makeImmediateTts(),
|
|
1799
|
+
spawnBackgroundContinuation: continuation.spawnBackgroundContinuation,
|
|
1800
|
+
continuationAnnounceSilenceMs: 20,
|
|
1801
|
+
});
|
|
1802
|
+
|
|
1803
|
+
await session.start();
|
|
1804
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
1805
|
+
await waitFor(() => frames.some((frame) => frame.type === "thinking"));
|
|
1806
|
+
await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
|
|
1807
|
+
await waitFor(
|
|
1808
|
+
() => continuation.spawnBackgroundContinuation.mock.calls.length === 1,
|
|
1809
|
+
);
|
|
1810
|
+
await waitFor(() => calls.some((c) => c.content === "second question"));
|
|
1811
|
+
|
|
1812
|
+
// The continuation finishes while the follow-up turn is still running, so
|
|
1813
|
+
// it remains queued instead of speaking over the assistant.
|
|
1814
|
+
continuation.finish("THE_RESULT");
|
|
1815
|
+
await new Promise((resolve) => setTimeout(resolve, 300));
|
|
1816
|
+
expect(announcementOf(calls)).toBeUndefined();
|
|
1817
|
+
|
|
1818
|
+
// Completing the active turn re-arms the silence window and announces the
|
|
1819
|
+
// result without requiring another user turn.
|
|
1820
|
+
const followUp = calls.find((c) => c.content === "second question");
|
|
1821
|
+
followUp?.callbacks?.assistant_text_delta?.(makeTextDelta("ok"));
|
|
1822
|
+
followUp?.callbacks?.message_complete?.(makeMessageComplete());
|
|
1823
|
+
await waitFor(() => frames.some((frame) => frame.type === "tts_done"));
|
|
1824
|
+
await waitFor(() => announcementOf(calls) !== undefined);
|
|
1825
|
+
expect(announcementOf(calls)?.voiceControlPrompt).toContain("THE_RESULT");
|
|
1826
|
+
});
|
|
1827
|
+
|
|
1828
|
+
test("a client interrupt between finish and fire cancels the announcement", async () => {
|
|
1829
|
+
const continuation = makeControlledContinuation();
|
|
1830
|
+
const { startVoiceTurn, calls } = makeResurfaceTurnStarter();
|
|
1831
|
+
const { frames, session } = createHarness({
|
|
1832
|
+
finals: ["first question", ""],
|
|
1833
|
+
startVoiceTurn,
|
|
1834
|
+
streamTtsAudio: makeImmediateTts(),
|
|
1835
|
+
spawnBackgroundContinuation: continuation.spawnBackgroundContinuation,
|
|
1836
|
+
continuationAnnounceSilenceMs: 300,
|
|
1837
|
+
});
|
|
1838
|
+
|
|
1839
|
+
await session.start();
|
|
1840
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
1841
|
+
await waitFor(() => frames.some((frame) => frame.type === "thinking"));
|
|
1842
|
+
await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
|
|
1843
|
+
await waitFor(
|
|
1844
|
+
() => continuation.spawnBackgroundContinuation.mock.calls.length === 1,
|
|
1845
|
+
);
|
|
1846
|
+
await waitFor(() =>
|
|
1847
|
+
frames.some((frame) => frame.type === "utterance_discarded"),
|
|
1848
|
+
);
|
|
1849
|
+
|
|
1850
|
+
continuation.finish("THE_RESULT");
|
|
1851
|
+
await flushAsyncCallbacks();
|
|
1852
|
+
await session.handleClientFrame({ type: "interrupt" });
|
|
1853
|
+
await new Promise((resolve) => setTimeout(resolve, 600));
|
|
1854
|
+
expect(announcementOf(calls)).toBeUndefined();
|
|
1855
|
+
});
|
|
1856
|
+
|
|
1857
|
+
test("closing the session between finish and fire cancels the announcement", async () => {
|
|
1858
|
+
const continuation = makeControlledContinuation();
|
|
1859
|
+
const { startVoiceTurn, calls } = makeResurfaceTurnStarter();
|
|
1860
|
+
const { frames, session } = createHarness({
|
|
1861
|
+
finals: ["first question", ""],
|
|
1862
|
+
startVoiceTurn,
|
|
1863
|
+
streamTtsAudio: makeImmediateTts(),
|
|
1864
|
+
spawnBackgroundContinuation: continuation.spawnBackgroundContinuation,
|
|
1865
|
+
continuationAnnounceSilenceMs: 300,
|
|
1866
|
+
});
|
|
1867
|
+
|
|
1868
|
+
await session.start();
|
|
1869
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
1870
|
+
await waitFor(() => frames.some((frame) => frame.type === "thinking"));
|
|
1871
|
+
await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
|
|
1872
|
+
await waitFor(
|
|
1873
|
+
() => continuation.spawnBackgroundContinuation.mock.calls.length === 1,
|
|
1874
|
+
);
|
|
1875
|
+
await waitFor(() =>
|
|
1876
|
+
frames.some((frame) => frame.type === "utterance_discarded"),
|
|
1877
|
+
);
|
|
1878
|
+
|
|
1879
|
+
continuation.finish("THE_RESULT");
|
|
1880
|
+
await flushAsyncCallbacks();
|
|
1881
|
+
await session.close("client_end");
|
|
1882
|
+
await new Promise((resolve) => setTimeout(resolve, 600));
|
|
1883
|
+
expect(announcementOf(calls)).toBeUndefined();
|
|
1884
|
+
});
|
|
1885
|
+
|
|
1886
|
+
test("a barge-in over an announcement spawns no continuation", async () => {
|
|
1887
|
+
const continuation = makeControlledContinuation();
|
|
1888
|
+
const { startVoiceTurn, calls } = makeNeverCompletingTurnStarter();
|
|
1889
|
+
const { frames, session } = createHarness({
|
|
1890
|
+
finals: ["first question", "", "third question"],
|
|
1891
|
+
startVoiceTurn,
|
|
1892
|
+
streamTtsAudio: makeImmediateTts(),
|
|
1893
|
+
spawnBackgroundContinuation: continuation.spawnBackgroundContinuation,
|
|
1894
|
+
continuationAnnounceSilenceMs: 20,
|
|
1895
|
+
});
|
|
1896
|
+
|
|
1897
|
+
await session.start();
|
|
1898
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
1899
|
+
await waitFor(() => frames.some((frame) => frame.type === "thinking"));
|
|
1900
|
+
await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
|
|
1901
|
+
await waitFor(
|
|
1902
|
+
() => continuation.spawnBackgroundContinuation.mock.calls.length === 1,
|
|
1903
|
+
);
|
|
1904
|
+
await waitFor(() =>
|
|
1905
|
+
frames.some((frame) => frame.type === "utterance_discarded"),
|
|
1906
|
+
);
|
|
1907
|
+
|
|
1908
|
+
continuation.finish("THE_RESULT");
|
|
1909
|
+
await waitFor(() => announcementOf(calls) !== undefined);
|
|
1910
|
+
|
|
1911
|
+
// Cutting in over an announcement means the user is answering it, not
|
|
1912
|
+
// asking for it to be finished in the background — the skip reason is
|
|
1913
|
+
// `announcement_turn` and nothing new is forked.
|
|
1914
|
+
await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
|
|
1915
|
+
await waitFor(() => countType(frames, "turn_cancelled") === 2);
|
|
1916
|
+
await flushAsyncCallbacks();
|
|
1917
|
+
expect(continuation.spawnBackgroundContinuation.mock.calls.length).toBe(1);
|
|
1918
|
+
});
|
|
1919
|
+
|
|
1920
|
+
test("a barge-in over an announcement returns the answer to the stash", async () => {
|
|
1921
|
+
const continuation = makeControlledContinuation();
|
|
1922
|
+
const { startVoiceTurn, calls } = makeNeverCompletingTurnStarter();
|
|
1923
|
+
const { frames, session } = createHarness({
|
|
1924
|
+
finals: ["first question", "", "third question"],
|
|
1925
|
+
startVoiceTurn,
|
|
1926
|
+
streamTtsAudio: makeImmediateTts(),
|
|
1927
|
+
spawnBackgroundContinuation: continuation.spawnBackgroundContinuation,
|
|
1928
|
+
continuationAnnounceSilenceMs: 20,
|
|
1929
|
+
});
|
|
1930
|
+
|
|
1931
|
+
await session.start();
|
|
1932
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
1933
|
+
await waitFor(() => frames.some((frame) => frame.type === "thinking"));
|
|
1934
|
+
await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
|
|
1935
|
+
await waitFor(
|
|
1936
|
+
() => continuation.spawnBackgroundContinuation.mock.calls.length === 1,
|
|
1937
|
+
);
|
|
1938
|
+
await waitFor(() =>
|
|
1939
|
+
frames.some((frame) => frame.type === "utterance_discarded"),
|
|
1940
|
+
);
|
|
1941
|
+
|
|
1942
|
+
continuation.finish("THE_RESULT");
|
|
1943
|
+
await waitFor(() => announcementOf(calls) !== undefined);
|
|
1944
|
+
|
|
1945
|
+
// The user cuts in while the announcement turn is still in flight, so it
|
|
1946
|
+
// never delivers. Nothing is forked to replace it (the work is finished),
|
|
1947
|
+
// so the stash is the only route left for the answer.
|
|
1948
|
+
await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
|
|
1949
|
+
await waitFor(() => countType(frames, "turn_cancelled") === 2);
|
|
1950
|
+
|
|
1951
|
+
await waitFor(() => calls.some((c) => c.content === "third question"));
|
|
1952
|
+
const later = calls.find((c) => c.content === "third question");
|
|
1953
|
+
expect(later?.voiceControlPrompt).toContain("THE_RESULT");
|
|
1954
|
+
// One announcement, and no second one racing the real turn: the answer is
|
|
1955
|
+
// recoverable, not duplicated.
|
|
1956
|
+
await flushAsyncCallbacks();
|
|
1957
|
+
expect(announcementCount(calls)).toBe(1);
|
|
1958
|
+
});
|
|
1959
|
+
|
|
1960
|
+
test("an announcement turn that cannot start hands the answer back to the stash", async () => {
|
|
1961
|
+
const continuation = makeControlledContinuation();
|
|
1962
|
+
const calls: VoiceTurnOptions[] = [];
|
|
1963
|
+
const startVoiceTurn = mock(async (options: VoiceTurnOptions) => {
|
|
1964
|
+
calls.push(options);
|
|
1965
|
+
// The announcement turn cannot start — the conversation is busy from
|
|
1966
|
+
// another surface. Real user turns still start normally.
|
|
1967
|
+
if (options.content === CONTINUATION_DELIVERY_CONTENT) {
|
|
1968
|
+
throw new Error("conversation is busy");
|
|
1969
|
+
}
|
|
1970
|
+
if (options.content !== "first question") {
|
|
1971
|
+
options.callbacks?.assistant_text_delta?.(makeTextDelta("ok"));
|
|
1972
|
+
options.callbacks?.message_complete?.(makeMessageComplete());
|
|
1973
|
+
}
|
|
1974
|
+
return { turnId: `bridge-turn-${calls.length}`, abort: mock() };
|
|
1975
|
+
});
|
|
1976
|
+
const { frames, session } = createHarness({
|
|
1977
|
+
finals: ["first question", "", "third question"],
|
|
1978
|
+
startVoiceTurn,
|
|
1979
|
+
streamTtsAudio: makeImmediateTts(),
|
|
1980
|
+
spawnBackgroundContinuation: continuation.spawnBackgroundContinuation,
|
|
1981
|
+
continuationAnnounceSilenceMs: 20,
|
|
1982
|
+
});
|
|
1983
|
+
|
|
1984
|
+
await session.start();
|
|
1985
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
1986
|
+
await waitFor(() => frames.some((frame) => frame.type === "thinking"));
|
|
1987
|
+
await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
|
|
1988
|
+
await waitFor(
|
|
1989
|
+
() => continuation.spawnBackgroundContinuation.mock.calls.length === 1,
|
|
1990
|
+
);
|
|
1991
|
+
await waitFor(() =>
|
|
1992
|
+
frames.some((frame) => frame.type === "utterance_discarded"),
|
|
1993
|
+
);
|
|
1994
|
+
|
|
1995
|
+
continuation.finish("THE_RESULT");
|
|
1996
|
+
await waitFor(() => announcementOf(calls) !== undefined);
|
|
1997
|
+
await waitFor(() => frames.some((frame) => frame.type === "error"));
|
|
1998
|
+
await flushAsyncCallbacks();
|
|
1999
|
+
// The failed attempt is not retried; the stash is the fallback route.
|
|
2000
|
+
expect(announcementCount(calls)).toBe(1);
|
|
2001
|
+
|
|
2002
|
+
// Nothing was spoken, so the answer is still there for the user's next turn
|
|
2003
|
+
// rather than being dropped with the announcement.
|
|
2004
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
2005
|
+
await waitFor(() => calls.some((c) => c.content === "third question"));
|
|
2006
|
+
const later = calls.find((c) => c.content === "third question");
|
|
2007
|
+
expect(later?.voiceControlPrompt).toContain("THE_RESULT");
|
|
2008
|
+
});
|
|
2009
|
+
|
|
2010
|
+
test("an announcement whose launch throws hands the answer back to the stash", async () => {
|
|
2011
|
+
const { startVoiceTurn, calls } = makeResurfaceTurnStarter();
|
|
2012
|
+
const { session } = createHarness({
|
|
2013
|
+
finals: ["stashed question"],
|
|
2014
|
+
startVoiceTurn,
|
|
2015
|
+
continuationAnnounceSilenceMs: 20,
|
|
2016
|
+
});
|
|
2017
|
+
|
|
2018
|
+
await session.start();
|
|
2019
|
+
await flushAsyncCallbacks();
|
|
2020
|
+
|
|
2021
|
+
const internals = session as unknown as {
|
|
2022
|
+
pendingContinuationResult: string | null;
|
|
2023
|
+
pendingAnnouncement: { request: string; answer: string } | null;
|
|
2024
|
+
scheduleContinuationAnnouncement: () => void;
|
|
2025
|
+
launchAssistantTurn: (
|
|
2026
|
+
utterance: unknown,
|
|
2027
|
+
content: string,
|
|
2028
|
+
opts?: unknown,
|
|
2029
|
+
) => Promise<boolean>;
|
|
2030
|
+
};
|
|
2031
|
+
// The announcement's launch REJECTS rather than returning false — a frame
|
|
2032
|
+
// write failing ahead of the leg's own error handling. The `started ===
|
|
2033
|
+
// false` re-stash never runs on that path, so only the catch can save the
|
|
2034
|
+
// answer.
|
|
2035
|
+
const launch = internals.launchAssistantTurn.bind(session);
|
|
2036
|
+
internals.launchAssistantTurn = async (utterance, content, opts) => {
|
|
2037
|
+
if (content === CONTINUATION_DELIVERY_CONTENT) {
|
|
2038
|
+
throw new Error("thinking frame write failed");
|
|
2039
|
+
}
|
|
2040
|
+
return launch(utterance, content, opts);
|
|
2041
|
+
};
|
|
2042
|
+
internals.pendingContinuationResult = "THE_RESULT";
|
|
2043
|
+
internals.pendingAnnouncement = {
|
|
2044
|
+
request: "the first question",
|
|
2045
|
+
answer: "THE_RESULT",
|
|
2046
|
+
};
|
|
2047
|
+
internals.scheduleContinuationAnnouncement();
|
|
2048
|
+
|
|
2049
|
+
// The attempt runs (the queue empties) and throws.
|
|
2050
|
+
await waitFor(() => internals.pendingAnnouncement === null);
|
|
2051
|
+
await flushAsyncCallbacks();
|
|
2052
|
+
expect(announcementOf(calls)).toBeUndefined();
|
|
2053
|
+
expect(internals.pendingContinuationResult).toBe("THE_RESULT");
|
|
2054
|
+
|
|
2055
|
+
// ...so the user's next turn still carries it.
|
|
2056
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
2057
|
+
await waitFor(() => calls.some((c) => c.content === "stashed question"));
|
|
2058
|
+
expect(
|
|
2059
|
+
calls.find((c) => c.content === "stashed question")?.voiceControlPrompt,
|
|
2060
|
+
).toContain("THE_RESULT");
|
|
2061
|
+
});
|
|
2062
|
+
|
|
2063
|
+
test("closing the room delivers a queued announcement into the conversation", async () => {
|
|
2064
|
+
injectMessageIntoParentMock.mockClear();
|
|
2065
|
+
const continuation = makeControlledContinuation();
|
|
2066
|
+
const { startVoiceTurn } = makeResurfaceTurnStarter();
|
|
2067
|
+
const { frames, session } = createHarness({
|
|
2068
|
+
finals: ["first question", ""],
|
|
2069
|
+
startVoiceTurn,
|
|
2070
|
+
streamTtsAudio: makeImmediateTts(),
|
|
2071
|
+
spawnBackgroundContinuation: continuation.spawnBackgroundContinuation,
|
|
2072
|
+
// Long enough that the announcement is still queued when the room closes.
|
|
2073
|
+
continuationAnnounceSilenceMs: 5_000,
|
|
2074
|
+
});
|
|
2075
|
+
|
|
2076
|
+
await session.start();
|
|
2077
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
2078
|
+
await waitFor(() => frames.some((frame) => frame.type === "thinking"));
|
|
2079
|
+
await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
|
|
2080
|
+
await waitFor(
|
|
2081
|
+
() => continuation.spawnBackgroundContinuation.mock.calls.length === 1,
|
|
2082
|
+
);
|
|
2083
|
+
await waitFor(() =>
|
|
2084
|
+
frames.some((frame) => frame.type === "utterance_discarded"),
|
|
2085
|
+
);
|
|
2086
|
+
|
|
2087
|
+
continuation.finish("THE_RESULT");
|
|
2088
|
+
await flushAsyncCallbacks();
|
|
2089
|
+
expect(injectMessageIntoParentMock).not.toHaveBeenCalled();
|
|
2090
|
+
|
|
2091
|
+
// The user hangs up before the silence elapses. Neither armed route can
|
|
2092
|
+
// still run, so the finished answer takes the closed-session route into the
|
|
2093
|
+
// thread instead of being dropped.
|
|
2094
|
+
await session.close("client_end");
|
|
2095
|
+
expect(injectMessageIntoParentMock).toHaveBeenCalledTimes(1);
|
|
2096
|
+
const [conversationId, message] =
|
|
2097
|
+
injectMessageIntoParentMock.mock.calls[0] ?? [];
|
|
2098
|
+
expect(conversationId).toBe("conversation-123");
|
|
2099
|
+
expect(message).toContain("THE_RESULT");
|
|
2100
|
+
expect(message).toContain("[Background work finished]");
|
|
2101
|
+
expect(message).toContain("first question");
|
|
2102
|
+
});
|
|
2103
|
+
|
|
2104
|
+
test("closing during an active announcement delivers the result into the conversation", async () => {
|
|
2105
|
+
injectMessageIntoParentMock.mockClear();
|
|
2106
|
+
const continuation = makeControlledContinuation();
|
|
2107
|
+
const { startVoiceTurn, calls } = makeNeverCompletingTurnStarter();
|
|
2108
|
+
const { frames, session } = createHarness({
|
|
2109
|
+
finals: ["first question", ""],
|
|
2110
|
+
startVoiceTurn,
|
|
2111
|
+
streamTtsAudio: makeImmediateTts(),
|
|
2112
|
+
spawnBackgroundContinuation: continuation.spawnBackgroundContinuation,
|
|
2113
|
+
continuationAnnounceSilenceMs: 20,
|
|
2114
|
+
});
|
|
2115
|
+
|
|
2116
|
+
await session.start();
|
|
2117
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
2118
|
+
await waitFor(() => frames.some((frame) => frame.type === "thinking"));
|
|
2119
|
+
await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
|
|
2120
|
+
await waitFor(
|
|
2121
|
+
() => continuation.spawnBackgroundContinuation.mock.calls.length === 1,
|
|
2122
|
+
);
|
|
2123
|
+
await waitFor(() =>
|
|
2124
|
+
frames.some((frame) => frame.type === "utterance_discarded"),
|
|
2125
|
+
);
|
|
2126
|
+
|
|
2127
|
+
continuation.finish("THE_RESULT");
|
|
2128
|
+
await waitFor(() => announcementOf(calls) !== undefined);
|
|
2129
|
+
expect(injectMessageIntoParentMock).not.toHaveBeenCalled();
|
|
2130
|
+
|
|
2131
|
+
await session.close("client_end");
|
|
2132
|
+
expect(injectMessageIntoParentMock).toHaveBeenCalledTimes(1);
|
|
2133
|
+
const [conversationId, message] =
|
|
2134
|
+
injectMessageIntoParentMock.mock.calls[0] ?? [];
|
|
2135
|
+
expect(conversationId).toBe("conversation-123");
|
|
2136
|
+
expect(message).toContain("THE_RESULT");
|
|
2137
|
+
expect(message).toContain("[Background work finished]");
|
|
2138
|
+
expect(message).toContain("first question");
|
|
2139
|
+
});
|
|
2140
|
+
|
|
2141
|
+
test("a speculatively dispatched turn carries the stashed answer and cancels the announcement", async () => {
|
|
2142
|
+
const continuation = makeControlledContinuation();
|
|
2143
|
+
const { startVoiceTurn, calls } = makeResurfaceTurnStarter();
|
|
2144
|
+
const { frames, session, transcribers } = createHarness({
|
|
2145
|
+
finals: ["first question", "", "third question"],
|
|
2146
|
+
startVoiceTurn,
|
|
2147
|
+
streamTtsAudio: makeImmediateTts(),
|
|
2148
|
+
spawnBackgroundContinuation: continuation.spawnBackgroundContinuation,
|
|
2149
|
+
// Long enough for the speculative turn below to launch first, short
|
|
2150
|
+
// enough that an announcement that survived it would fire inside the
|
|
2151
|
+
// test's own wait.
|
|
2152
|
+
continuationAnnounceSilenceMs: 250,
|
|
2153
|
+
});
|
|
2154
|
+
|
|
2155
|
+
await session.start();
|
|
2156
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
2157
|
+
await waitFor(() => frames.some((frame) => frame.type === "thinking"));
|
|
2158
|
+
await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
|
|
2159
|
+
await waitFor(
|
|
2160
|
+
() => continuation.spawnBackgroundContinuation.mock.calls.length === 1,
|
|
2161
|
+
);
|
|
2162
|
+
await waitFor(() =>
|
|
2163
|
+
frames.some((frame) => frame.type === "utterance_discarded"),
|
|
2164
|
+
);
|
|
2165
|
+
|
|
2166
|
+
// The continuation finishes on an idle call: stash and announcement are
|
|
2167
|
+
// both armed for the one answer.
|
|
2168
|
+
continuation.finish("THE_RESULT");
|
|
2169
|
+
await flushAsyncCallbacks();
|
|
2170
|
+
|
|
2171
|
+
// The user speaks again with a partial already in hand, so the silence
|
|
2172
|
+
// boundary dispatches the turn SPECULATIVELY — the default hands-free path
|
|
2173
|
+
// (endpointMaxExtensions defaults to 2) rather than the release path.
|
|
2174
|
+
await waitFor(() => transcribers.length === 3);
|
|
2175
|
+
transcribers[2]?.emit({ type: "partial", text: "third question" });
|
|
2176
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
2177
|
+
|
|
2178
|
+
await waitFor(() => calls.some((c) => c.content === "third question"));
|
|
2179
|
+
const speculative = calls.find((c) => c.content === "third question");
|
|
2180
|
+
// The verdict rule is the proof this leg was dispatched speculatively.
|
|
2181
|
+
expect(speculative?.unifiedVerdict).toBe(true);
|
|
2182
|
+
expect(speculative?.voiceControlPrompt).toContain("THE_RESULT");
|
|
2183
|
+
|
|
2184
|
+
// The user's own turn is the delivery, so the queued announcement is
|
|
2185
|
+
// cancelled rather than speaking the same answer again afterwards.
|
|
2186
|
+
await new Promise((resolve) => setTimeout(resolve, 600));
|
|
2187
|
+
expect(announcementCount(calls)).toBe(0);
|
|
2188
|
+
});
|
|
2189
|
+
|
|
2190
|
+
test("a held speculative turn hands the stashed answer back for the replay", async () => {
|
|
2191
|
+
const continuation = makeControlledContinuation();
|
|
2192
|
+
const calls: VoiceTurnOptions[] = [];
|
|
2193
|
+
const discard = mock(async () => {});
|
|
2194
|
+
// The first turn stays thinking so the barge-in lands on it. Every
|
|
2195
|
+
// speculative leg afterwards answers with the hold token while it is
|
|
2196
|
+
// offered one (`unifiedVerdict`), and answers for real on the replay.
|
|
2197
|
+
const startVoiceTurn: LiveVoiceTurnStarter = async (options) => {
|
|
2198
|
+
calls.push(options);
|
|
2199
|
+
const index = calls.length;
|
|
2200
|
+
if (options.content !== "first question") {
|
|
2201
|
+
const reply = options.unifiedVerdict === true ? "[0]" : "Sure thing.";
|
|
2202
|
+
setTimeout(() => {
|
|
2203
|
+
options.callbacks?.assistant_text_delta?.(makeTextDelta(reply));
|
|
2204
|
+
if (reply !== "[0]") {
|
|
2205
|
+
options.callbacks?.message_complete?.(makeMessageComplete());
|
|
2206
|
+
}
|
|
2207
|
+
}, 0);
|
|
2208
|
+
}
|
|
2209
|
+
return { turnId: `bridge-turn-${index}`, abort: mock(), discard };
|
|
2210
|
+
};
|
|
2211
|
+
const { frames, session, transcribers } = createHarness({
|
|
2212
|
+
finals: ["first question", "", "third question"],
|
|
2213
|
+
startVoiceTurn,
|
|
2214
|
+
streamTtsAudio: makeImmediateTts(),
|
|
2215
|
+
spawnBackgroundContinuation: continuation.spawnBackgroundContinuation,
|
|
2216
|
+
frontModelConfig: { endpointExtensionMs: 30 },
|
|
2217
|
+
// The announcement must not fire during the hold window — the replay
|
|
2218
|
+
// turn is the delivery this test is about.
|
|
2219
|
+
continuationAnnounceSilenceMs: 5_000,
|
|
2220
|
+
});
|
|
2221
|
+
|
|
2222
|
+
await session.start();
|
|
2223
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
2224
|
+
await waitFor(() => frames.some((frame) => frame.type === "thinking"));
|
|
2225
|
+
await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
|
|
2226
|
+
await waitFor(
|
|
2227
|
+
() => continuation.spawnBackgroundContinuation.mock.calls.length === 1,
|
|
2228
|
+
);
|
|
2229
|
+
await waitFor(() =>
|
|
2230
|
+
frames.some((frame) => frame.type === "utterance_discarded"),
|
|
2231
|
+
);
|
|
2232
|
+
|
|
2233
|
+
continuation.finish("THE_RESULT");
|
|
2234
|
+
await flushAsyncCallbacks();
|
|
2235
|
+
|
|
2236
|
+
await waitFor(() => transcribers.length === 3);
|
|
2237
|
+
transcribers[2]?.emit({ type: "partial", text: "third question" });
|
|
2238
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
2239
|
+
|
|
2240
|
+
// The speculative leg holds: it is rolled back, taking nothing with it.
|
|
2241
|
+
await waitFor(() => discard.mock.calls.length === 1);
|
|
2242
|
+
|
|
2243
|
+
// The extension replays the boundary and the replay leg answers. It is a
|
|
2244
|
+
// different dispatch, so it only carries the answer if the rollback gave
|
|
2245
|
+
// it back.
|
|
2246
|
+
await waitFor(
|
|
2247
|
+
() => calls.filter((c) => c.content === "third question").length === 2,
|
|
2248
|
+
);
|
|
2249
|
+
const replay = calls.filter((c) => c.content === "third question")[1];
|
|
2250
|
+
expect(replay?.unifiedVerdict).toBeUndefined();
|
|
2251
|
+
expect(replay?.voiceControlPrompt).toContain("THE_RESULT");
|
|
2252
|
+
});
|
|
2253
|
+
|
|
2254
|
+
test("an announcement waits out the client's queued playback", async () => {
|
|
2255
|
+
const continuation = makeControlledContinuation();
|
|
2256
|
+
const { startVoiceTurn, calls } = makeNeverCompletingTurnStarter();
|
|
2257
|
+
const { frames, session } = createHarness({
|
|
2258
|
+
finals: ["first question", "second question", "third question"],
|
|
2259
|
+
startVoiceTurn,
|
|
2260
|
+
streamTtsAudio: makeLongTailTts(),
|
|
2261
|
+
spawnBackgroundContinuation: continuation.spawnBackgroundContinuation,
|
|
2262
|
+
continuationAnnounceSilenceMs: 200,
|
|
2263
|
+
});
|
|
2264
|
+
|
|
2265
|
+
await session.start();
|
|
2266
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
2267
|
+
await waitFor(() => frames.some((frame) => frame.type === "thinking"));
|
|
2268
|
+
await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
|
|
2269
|
+
await waitFor(
|
|
2270
|
+
() => continuation.spawnBackgroundContinuation.mock.calls.length === 1,
|
|
2271
|
+
);
|
|
2272
|
+
await waitFor(() => calls.some((c) => c.content === "second question"));
|
|
2273
|
+
|
|
2274
|
+
// The continuation finishes while the follow-up turn still holds the floor.
|
|
2275
|
+
continuation.finish("THE_RESULT");
|
|
2276
|
+
await flushAsyncCallbacks();
|
|
2277
|
+
|
|
2278
|
+
// That reply completes: the turn is cleared server-side, but the client
|
|
2279
|
+
// still has ~600 ms of its audio queued.
|
|
2280
|
+
const followUp = calls.find((c) => c.content === "second question");
|
|
2281
|
+
followUp?.callbacks?.assistant_text_delta?.(makeTextDelta("ok"));
|
|
2282
|
+
followUp?.callbacks?.message_complete?.(makeMessageComplete());
|
|
2283
|
+
await waitFor(() => frames.some((frame) => frame.type === "tts_done"));
|
|
2284
|
+
|
|
2285
|
+
// The silence timer elapses mid-tail: the call looks idle but the previous
|
|
2286
|
+
// reply is still audible, so the announcement holds.
|
|
2287
|
+
await new Promise((resolve) => setTimeout(resolve, 400));
|
|
2288
|
+
expect(announcementOf(calls)).toBeUndefined();
|
|
2289
|
+
|
|
2290
|
+
// Once the queued audio has drained it lands, instead of being dropped for
|
|
2291
|
+
// having been blocked.
|
|
2292
|
+
await waitFor(() => announcementOf(calls) !== undefined);
|
|
2293
|
+
expect(announcementOf(calls)?.voiceControlPrompt).toContain("THE_RESULT");
|
|
2294
|
+
});
|
|
2295
|
+
|
|
2296
|
+
test("captured push-to-talk audio with no transcript yet defers the announcement", async () => {
|
|
2297
|
+
const { startVoiceTurn, calls } = makeResurfaceTurnStarter();
|
|
2298
|
+
const { session } = createHarness({
|
|
2299
|
+
startFrame: MANUAL_START_FRAME,
|
|
2300
|
+
finals: ["ptt question"],
|
|
2301
|
+
startVoiceTurn,
|
|
2302
|
+
continuationAnnounceSilenceMs: 20,
|
|
2303
|
+
});
|
|
2304
|
+
|
|
2305
|
+
await session.start();
|
|
2306
|
+
// Settle the arm so the cycle is streaming: from here audio goes straight
|
|
2307
|
+
// to the transcriber and never lands in the pending buffer.
|
|
2308
|
+
await flushAsyncCallbacks();
|
|
2309
|
+
|
|
2310
|
+
// The user is holding the button and talking. The transcriber emits
|
|
2311
|
+
// nothing until the release, so the cycle has no partial and no final —
|
|
2312
|
+
// every transcript-derived idle signal still reads "nobody is speaking".
|
|
2313
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
2314
|
+
|
|
2315
|
+
// A manual session reaches the announcement path only through machinery a
|
|
2316
|
+
// server_vad barge-in owns, so the state a finished continuation leaves
|
|
2317
|
+
// behind is staged directly: the stashed answer plus its queued
|
|
2318
|
+
// announcement.
|
|
2319
|
+
const internals = session as unknown as {
|
|
2320
|
+
pendingContinuationResult: string | null;
|
|
2321
|
+
pendingAnnouncement: { request: string; answer: string } | null;
|
|
2322
|
+
scheduleContinuationAnnouncement: () => void;
|
|
2323
|
+
};
|
|
2324
|
+
internals.pendingContinuationResult = "THE_RESULT";
|
|
2325
|
+
internals.pendingAnnouncement = {
|
|
2326
|
+
request: "the first question",
|
|
2327
|
+
answer: "THE_RESULT",
|
|
2328
|
+
};
|
|
2329
|
+
internals.scheduleContinuationAnnouncement();
|
|
2330
|
+
|
|
2331
|
+
// The silence timer and the single retry both find the utterance in
|
|
2332
|
+
// flight, so the session never speaks over the in-progress utterance.
|
|
2333
|
+
await new Promise((resolve) => setTimeout(resolve, 200));
|
|
2334
|
+
expect(announcementOf(calls)).toBeUndefined();
|
|
2335
|
+
|
|
2336
|
+
// ...and the answer is still stashed for the turn the user does start.
|
|
2337
|
+
await session.handleClientFrame({ type: "ptt_release" });
|
|
2338
|
+
await waitFor(() => calls.some((c) => c.content === "ptt question"));
|
|
2339
|
+
expect(
|
|
2340
|
+
calls.find((c) => c.content === "ptt question")?.voiceControlPrompt,
|
|
2341
|
+
).toContain("THE_RESULT");
|
|
2342
|
+
});
|
|
2343
|
+
|
|
2344
|
+
test("a late assistant_text_delta after a thinking barge-in never reaches the client", async () => {
|
|
2345
|
+
let callbacks: VoiceTurnCallbacks | undefined;
|
|
2346
|
+
const abort = mock();
|
|
2347
|
+
const startVoiceTurn = mock(async (options: VoiceTurnOptions) => {
|
|
2348
|
+
callbacks ??= options.callbacks;
|
|
2349
|
+
return { turnId: "bridge-turn", abort };
|
|
2350
|
+
});
|
|
2351
|
+
let releaseTurnCancelled: (() => void) | undefined;
|
|
2352
|
+
const { frames, session } = createHarness({
|
|
2353
|
+
finals: ["first question", "second question"],
|
|
2354
|
+
startVoiceTurn,
|
|
2355
|
+
// Suspend the async barge-in teardown at the turn_cancelled send so the
|
|
2356
|
+
// aborted turn stays non-finalized — the exact window a late model delta
|
|
2357
|
+
// could race into before cancelAssistantTurn finishes.
|
|
2358
|
+
holdSendFrame: (payload) => {
|
|
2359
|
+
if (payload.type !== "turn_cancelled" || releaseTurnCancelled) {
|
|
2360
|
+
return null;
|
|
2361
|
+
}
|
|
2362
|
+
return new Promise<void>((resolve) => {
|
|
2363
|
+
releaseTurnCancelled = resolve;
|
|
2364
|
+
});
|
|
2365
|
+
},
|
|
2366
|
+
});
|
|
2367
|
+
|
|
2368
|
+
await session.start();
|
|
2369
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
2370
|
+
await waitFor(() => callbacks !== undefined);
|
|
2371
|
+
|
|
2372
|
+
// Barge in while thinking; teardown blocks on the held turn_cancelled, so
|
|
2373
|
+
// the turn is aborted but not yet finalized.
|
|
2374
|
+
await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
|
|
2375
|
+
await waitFor(() => releaseTurnCancelled !== undefined);
|
|
2376
|
+
|
|
2377
|
+
// A first assistant_text_delta lands in that window — fenced on the abort
|
|
2378
|
+
// signal, it must not be forwarded to the client.
|
|
2379
|
+
callbacks?.assistant_text_delta?.(makeTextDelta("stale thinking reply"));
|
|
2380
|
+
await flushAsyncCallbacks();
|
|
2381
|
+
expect(countType(frames, "assistant_text_delta")).toBe(0);
|
|
2382
|
+
|
|
2383
|
+
releaseTurnCancelled?.();
|
|
2384
|
+
await waitFor(() =>
|
|
2385
|
+
frames.some((frame) => frame.type === "turn_cancelled"),
|
|
2386
|
+
);
|
|
2387
|
+
expect(countType(frames, "assistant_text_delta")).toBe(0);
|
|
2388
|
+
});
|
|
2389
|
+
|
|
2390
|
+
test("a queued assistant_text_delta is dropped at send time once a thinking barge-in aborts", async () => {
|
|
2391
|
+
let callbacks: VoiceTurnCallbacks | undefined;
|
|
2392
|
+
const abort = mock();
|
|
2393
|
+
const startVoiceTurn = mock(async (options: VoiceTurnOptions) => {
|
|
2394
|
+
callbacks ??= options.callbacks;
|
|
2395
|
+
return { turnId: "bridge-turn", abort };
|
|
2396
|
+
});
|
|
2397
|
+
let releaseFirstDelta: (() => void) | undefined;
|
|
2398
|
+
// No TTS streamer: the turn emits text but never leaves the pre-TTS phase.
|
|
2399
|
+
const { frames, session } = createHarness({
|
|
2400
|
+
finals: ["first question", "second question"],
|
|
2401
|
+
startVoiceTurn,
|
|
2402
|
+
streamTtsAudio: null,
|
|
2403
|
+
// Hold the first assistant_text_delta's transport write so the second
|
|
2404
|
+
// one sits queued behind it (a backed-up outbound queue).
|
|
2405
|
+
holdSendFrame: (payload) => {
|
|
2406
|
+
if (
|
|
2407
|
+
payload.type !== "assistant_text_delta" ||
|
|
2408
|
+
releaseFirstDelta !== undefined
|
|
2409
|
+
) {
|
|
2410
|
+
return null;
|
|
2411
|
+
}
|
|
2412
|
+
return new Promise<void>((resolve) => {
|
|
2413
|
+
releaseFirstDelta = resolve;
|
|
2414
|
+
});
|
|
2415
|
+
},
|
|
2416
|
+
});
|
|
2417
|
+
|
|
2418
|
+
await session.start();
|
|
2419
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
2420
|
+
await waitFor(() => callbacks !== undefined);
|
|
2421
|
+
|
|
2422
|
+
// First delta passes shouldSend, then blocks in the transport; the second
|
|
2423
|
+
// is enqueued behind it and has not yet been send-time checked.
|
|
2424
|
+
callbacks?.assistant_text_delta?.(makeTextDelta("early reply"));
|
|
2425
|
+
await waitFor(() => releaseFirstDelta !== undefined);
|
|
2426
|
+
callbacks?.assistant_text_delta?.(makeTextDelta("leaked tail"));
|
|
2427
|
+
|
|
2428
|
+
// Barge in while both deltas are queued: the turn aborts before the second
|
|
2429
|
+
// delta drains.
|
|
2430
|
+
await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
|
|
2431
|
+
await flushAsyncCallbacks();
|
|
2432
|
+
|
|
2433
|
+
// Release the queue: the first (already-committed) delta writes, but the
|
|
2434
|
+
// second must be dropped by the send-time guard — no cancelled-reply text
|
|
2435
|
+
// leaks after the abort.
|
|
2436
|
+
releaseFirstDelta?.();
|
|
2437
|
+
await waitFor(() =>
|
|
2438
|
+
frames.some((frame) => frame.type === "turn_cancelled"),
|
|
2439
|
+
);
|
|
2440
|
+
expect(countType(frames, "assistant_text_delta")).toBe(1);
|
|
2441
|
+
expect(
|
|
2442
|
+
frames.some(
|
|
2443
|
+
(frame) =>
|
|
2444
|
+
frame.type === "assistant_text_delta" && frame.text === "leaked tail",
|
|
2445
|
+
),
|
|
2446
|
+
).toBe(false);
|
|
2447
|
+
});
|
|
2448
|
+
|
|
2449
|
+
test("a thinking barge-in that rejects the pending turn start emits no error frame", async () => {
|
|
2450
|
+
// startVoiceTurn hangs like a real turn waiting for the conversation lock
|
|
2451
|
+
// and rejects when the turn's signal aborts (the waitForIdle behavior), so
|
|
2452
|
+
// the turn is still "thinking" with no handle when barge-in hits.
|
|
2453
|
+
const startVoiceTurn: LiveVoiceTurnStarter = async (options) => {
|
|
2454
|
+
await new Promise<void>((_resolve, reject) => {
|
|
2455
|
+
const fail = () =>
|
|
2456
|
+
reject(new Error("turn aborted while waiting for the lock"));
|
|
2457
|
+
if (options.signal?.aborted) {
|
|
2458
|
+
fail();
|
|
2459
|
+
return;
|
|
2460
|
+
}
|
|
2461
|
+
options.signal?.addEventListener("abort", fail, { once: true });
|
|
2462
|
+
});
|
|
2463
|
+
return { turnId: "bridge-turn", abort: mock() };
|
|
2464
|
+
};
|
|
2465
|
+
let releaseTurnCancelled: (() => void) | undefined;
|
|
2466
|
+
const { frames, session } = createHarness({
|
|
2467
|
+
finals: ["first question", "second question"],
|
|
2468
|
+
startVoiceTurn,
|
|
2469
|
+
holdSendFrame: (payload) => {
|
|
2470
|
+
if (payload.type !== "turn_cancelled" || releaseTurnCancelled) {
|
|
2471
|
+
return null;
|
|
2472
|
+
}
|
|
2473
|
+
return new Promise<void>((resolve) => {
|
|
2474
|
+
releaseTurnCancelled = resolve;
|
|
2475
|
+
});
|
|
2476
|
+
},
|
|
2477
|
+
});
|
|
2478
|
+
|
|
2479
|
+
await session.start();
|
|
2480
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
2481
|
+
await waitFor(() => frames.some((frame) => frame.type === "thinking"));
|
|
2482
|
+
|
|
2483
|
+
// Barge in while the turn's start is still pending on the lock: the abort
|
|
2484
|
+
// rejects startVoiceTurn, whose catch must treat the aborted turn as dead
|
|
2485
|
+
// rather than surface a stray error frame while teardown is in flight.
|
|
2486
|
+
await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
|
|
2487
|
+
await waitFor(() => releaseTurnCancelled !== undefined);
|
|
2488
|
+
await flushAsyncCallbacks();
|
|
2489
|
+
expect(countType(frames, "error")).toBe(0);
|
|
2490
|
+
|
|
2491
|
+
releaseTurnCancelled?.();
|
|
2492
|
+
await waitFor(() =>
|
|
2493
|
+
frames.some((frame) => frame.type === "turn_cancelled"),
|
|
2494
|
+
);
|
|
2495
|
+
expect(countType(frames, "error")).toBe(0);
|
|
2496
|
+
});
|
|
2497
|
+
|
|
2498
|
+
test("a brief blip while thinking arms the guard but does not cancel", async () => {
|
|
2499
|
+
let callbacks: VoiceTurnCallbacks | undefined;
|
|
2500
|
+
const abort = mock();
|
|
2501
|
+
const startVoiceTurn = mock(async (options: VoiceTurnOptions) => {
|
|
2502
|
+
callbacks ??= options.callbacks;
|
|
2503
|
+
return { turnId: "bridge-turn", abort };
|
|
2504
|
+
});
|
|
2505
|
+
const streamTtsAudio = mock(async (options: LiveVoiceTtsOptions) => {
|
|
2506
|
+
options.onAudioChunk(makeTtsChunk("assistant audio"));
|
|
2507
|
+
return makeTtsResult("assistant audio");
|
|
2508
|
+
});
|
|
2509
|
+
const { frames, session } = createHarness({
|
|
2510
|
+
finals: ["first question", "second question"],
|
|
2511
|
+
startVoiceTurn,
|
|
2512
|
+
streamTtsAudio,
|
|
2513
|
+
});
|
|
2514
|
+
|
|
2515
|
+
await session.start();
|
|
2516
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
2517
|
+
await waitFor(() => frames.some((frame) => frame.type === "thinking"));
|
|
2518
|
+
|
|
2519
|
+
// A short blip (well under bargeInMinSpeechMs) while the turn is still
|
|
2520
|
+
// "thinking": the sustained-speech guard arms but does not trip, so a
|
|
2521
|
+
// cough or noise cannot kill the in-flight agent loop.
|
|
2522
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
2523
|
+
await flushAsyncCallbacks();
|
|
2524
|
+
|
|
2525
|
+
expect(countType(frames, "turn_cancelled")).toBe(0);
|
|
2526
|
+
expect(abort).not.toHaveBeenCalled();
|
|
2527
|
+
|
|
2528
|
+
// The unspoken reply still completes normally.
|
|
1603
2529
|
callbacks?.assistant_text_delta?.(makeTextDelta("Here is the answer."));
|
|
1604
2530
|
callbacks?.message_complete?.(makeMessageComplete());
|
|
1605
2531
|
await waitFor(() =>
|
|
@@ -2691,395 +3617,7 @@ describe("LiveVoiceSession sustained-speech barge-in guard", () => {
|
|
|
2691
3617
|
});
|
|
2692
3618
|
});
|
|
2693
3619
|
|
|
2694
|
-
// Stub semantic-endpointing decider: scripted hold/release decisions consumed
|
|
2695
|
-
// in order, with every consulted input captured. Falls back to "release" once
|
|
2696
|
-
// the script is exhausted, mirroring the real decider's fail-open bias.
|
|
2697
|
-
function makeFrontDecider(decisions: Array<"hold" | "release">): {
|
|
2698
|
-
decider: VoiceFrontDecider;
|
|
2699
|
-
calls: VoiceEndpointDecisionInput[];
|
|
2700
|
-
} {
|
|
2701
|
-
const calls: VoiceEndpointDecisionInput[] = [];
|
|
2702
|
-
return {
|
|
2703
|
-
calls,
|
|
2704
|
-
decider: {
|
|
2705
|
-
decideEndpoint: async (input) => {
|
|
2706
|
-
calls.push(input);
|
|
2707
|
-
return { action: decisions[calls.length - 1] ?? "release" };
|
|
2708
|
-
},
|
|
2709
|
-
generateAckText: async () => null,
|
|
2710
|
-
generateProgressText: async () => null,
|
|
2711
|
-
},
|
|
2712
|
-
};
|
|
2713
|
-
}
|
|
2714
|
-
|
|
2715
|
-
describe("LiveVoiceSession semantic endpointing", () => {
|
|
2716
|
-
// Arms the harness session, waits for its transcriber, and seeds a partial
|
|
2717
|
-
// so the silence boundary has transcript text for the decider to judge.
|
|
2718
|
-
async function startWithPartial(
|
|
2719
|
-
session: LiveVoiceSession,
|
|
2720
|
-
transcribers: MockStreamingTranscriber[],
|
|
2721
|
-
partialText = "hello wor",
|
|
2722
|
-
): Promise<void> {
|
|
2723
|
-
await session.start();
|
|
2724
|
-
await waitFor(() => transcribers.length === 1);
|
|
2725
|
-
// Let the transcriber's start() wiring settle before emitting events.
|
|
2726
|
-
await flushAsyncCallbacks();
|
|
2727
|
-
transcribers[0]?.emit({ type: "partial", text: partialText });
|
|
2728
|
-
}
|
|
2729
|
-
|
|
2730
|
-
test("a held silence boundary sends no utterance_end and replays after the extension", async () => {
|
|
2731
|
-
const { decider, calls } = makeFrontDecider(["hold", "release"]);
|
|
2732
|
-
const turnStarter = makeAutoCompletingTurnStarter(["Hi there."]);
|
|
2733
|
-
const { frames, session, transcribers } = createHarness({
|
|
2734
|
-
finals: ["hello world"],
|
|
2735
|
-
startVoiceTurn: turnStarter.startVoiceTurn,
|
|
2736
|
-
frontDecider: decider,
|
|
2737
|
-
frontModelConfig: { endpointExtensionMs: 30 },
|
|
2738
|
-
});
|
|
2739
|
-
|
|
2740
|
-
await startWithPartial(session, transcribers);
|
|
2741
|
-
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
2742
|
-
|
|
2743
|
-
// First silence boundary: the decider holds — no utterance_end, no turn.
|
|
2744
|
-
await waitFor(() => calls.length === 1);
|
|
2745
|
-
expect(calls[0]).toEqual({
|
|
2746
|
-
transcriptSoFar: "",
|
|
2747
|
-
latestPartial: "hello wor",
|
|
2748
|
-
silenceThresholdMs: 40,
|
|
2749
|
-
extensionCount: 0,
|
|
2750
|
-
});
|
|
2751
|
-
expect(countType(frames, "utterance_end")).toBe(0);
|
|
2752
|
-
expect(countType(frames, "thinking")).toBe(0);
|
|
2753
|
-
|
|
2754
|
-
// The extension elapses in continued silence: the boundary replays, the
|
|
2755
|
-
// decider releases, and the turn launches normally.
|
|
2756
|
-
await waitFor(() => frames.some((frame) => frame.type === "tts_done"));
|
|
2757
|
-
expect(calls).toHaveLength(2);
|
|
2758
|
-
expect(calls[1]).toMatchObject({ extensionCount: 1 });
|
|
2759
|
-
expect(frameTypes(frames)).toEqual([
|
|
2760
|
-
"ready",
|
|
2761
|
-
"stt_partial",
|
|
2762
|
-
"speech_started",
|
|
2763
|
-
"utterance_end",
|
|
2764
|
-
"stt_final",
|
|
2765
|
-
"thinking",
|
|
2766
|
-
"assistant_text_delta",
|
|
2767
|
-
"tts_done",
|
|
2768
|
-
]);
|
|
2769
|
-
expect(
|
|
2770
|
-
frames.find((frame) => frame.type === "utterance_end"),
|
|
2771
|
-
).toMatchObject({ reason: "silence" });
|
|
2772
|
-
expect(turnStarter.calls[0]).toMatchObject({ content: "hello world" });
|
|
2773
|
-
});
|
|
2774
|
-
|
|
2775
|
-
test("ptt_release during a held pause still emits utterance_end", async () => {
|
|
2776
|
-
const { decider, calls } = makeFrontDecider(["hold"]);
|
|
2777
|
-
const turnStarter = makeAutoCompletingTurnStarter(["Hi."]);
|
|
2778
|
-
const { frames, session, transcribers } = createHarness({
|
|
2779
|
-
finals: ["hello world"],
|
|
2780
|
-
startVoiceTurn: turnStarter.startVoiceTurn,
|
|
2781
|
-
frontDecider: decider,
|
|
2782
|
-
// Long extension so the pending replay cannot race the manual release.
|
|
2783
|
-
frontModelConfig: { endpointExtensionMs: 5_000 },
|
|
2784
|
-
});
|
|
2785
|
-
|
|
2786
|
-
await startWithPartial(session, transcribers);
|
|
2787
|
-
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
2788
|
-
await waitFor(() => calls.length === 1);
|
|
2789
|
-
expect(countType(frames, "utterance_end")).toBe(0);
|
|
2790
|
-
|
|
2791
|
-
// The detector's turn already ended (the hold suppressed its boundary),
|
|
2792
|
-
// so this release takes the no-active-detector path — the client must
|
|
2793
|
-
// still receive the utterance_end frame it uses to leave `listening`.
|
|
2794
|
-
await session.handleClientFrame({ type: "ptt_release" });
|
|
2795
|
-
await waitFor(() => frames.some((frame) => frame.type === "tts_done"));
|
|
2796
|
-
expect(countType(frames, "utterance_end")).toBe(1);
|
|
2797
|
-
// Manual release never re-consults the decider.
|
|
2798
|
-
expect(calls).toHaveLength(1);
|
|
2799
|
-
expect(turnStarter.calls[0]).toMatchObject({ content: "hello world" });
|
|
2800
|
-
});
|
|
2801
|
-
|
|
2802
|
-
test("speech resuming during a hold cancels the replay and the utterance keeps accumulating", async () => {
|
|
2803
|
-
const { decider, calls } = makeFrontDecider(["hold", "release"]);
|
|
2804
|
-
const turnStarter = makeAutoCompletingTurnStarter(["Okay."]);
|
|
2805
|
-
const { frames, session, transcribers } = createHarness({
|
|
2806
|
-
finals: ["hello there world"],
|
|
2807
|
-
startVoiceTurn: turnStarter.startVoiceTurn,
|
|
2808
|
-
frontDecider: decider,
|
|
2809
|
-
// Comfortably longer than the resumed speech's own silence boundary,
|
|
2810
|
-
// so a leaked (uncancelled) replay would surface as a third decider
|
|
2811
|
-
// consult / second utterance_end below.
|
|
2812
|
-
frontModelConfig: { endpointExtensionMs: 200 },
|
|
2813
|
-
});
|
|
2814
|
-
|
|
2815
|
-
await startWithPartial(session, transcribers, "hello");
|
|
2816
|
-
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
2817
|
-
await waitFor(() => calls.length === 1);
|
|
2818
|
-
expect(countType(frames, "utterance_end")).toBe(0);
|
|
2819
|
-
|
|
2820
|
-
// Speech resumes during the hold; the fresh silence boundary after it —
|
|
2821
|
-
// not the cancelled replay timer — drives the next decision.
|
|
2822
|
-
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
2823
|
-
await waitFor(() => calls.length === 2);
|
|
2824
|
-
expect(calls[1]).toMatchObject({ extensionCount: 1 });
|
|
2825
|
-
await waitFor(() => frames.some((frame) => frame.type === "tts_done"));
|
|
2826
|
-
|
|
2827
|
-
// Wait out the original extension window: the cancelled replay never
|
|
2828
|
-
// re-fires the boundary.
|
|
2829
|
-
await new Promise((resolve) => setTimeout(resolve, 250));
|
|
2830
|
-
expect(calls).toHaveLength(2);
|
|
2831
|
-
expect(countType(frames, "utterance_end")).toBe(1);
|
|
2832
|
-
// Both speech bursts fed the same utterance's transcriber (the second
|
|
2833
|
-
// resolves only when the completed turn re-arms), producing one released
|
|
2834
|
-
// turn end-to-end.
|
|
2835
|
-
expect(transcribers[0]?.received.length).toBeGreaterThanOrEqual(2);
|
|
2836
|
-
expect(turnStarter.calls).toHaveLength(1);
|
|
2837
|
-
expect(turnStarter.calls[0]).toMatchObject({
|
|
2838
|
-
content: "hello there world",
|
|
2839
|
-
});
|
|
2840
|
-
});
|
|
2841
|
-
|
|
2842
|
-
test("a release decision produces the same frame sequence as the flag-off path", async () => {
|
|
2843
|
-
const { decider, calls } = makeFrontDecider(["release"]);
|
|
2844
|
-
const turnStarter = makeAutoCompletingTurnStarter(["Hi there."]);
|
|
2845
|
-
const { frames, session, transcribers } = createHarness({
|
|
2846
|
-
finals: ["hello world"],
|
|
2847
|
-
startVoiceTurn: turnStarter.startVoiceTurn,
|
|
2848
|
-
frontDecider: decider,
|
|
2849
|
-
});
|
|
2850
|
-
|
|
2851
|
-
await startWithPartial(session, transcribers);
|
|
2852
|
-
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
2853
|
-
await waitFor(() => frames.some((frame) => frame.type === "tts_done"));
|
|
2854
|
-
|
|
2855
|
-
expect(calls).toHaveLength(1);
|
|
2856
|
-
expect(frameTypes(frames)).toEqual([
|
|
2857
|
-
"ready",
|
|
2858
|
-
"stt_partial",
|
|
2859
|
-
"speech_started",
|
|
2860
|
-
"utterance_end",
|
|
2861
|
-
"stt_final",
|
|
2862
|
-
"thinking",
|
|
2863
|
-
"assistant_text_delta",
|
|
2864
|
-
"tts_done",
|
|
2865
|
-
]);
|
|
2866
|
-
expect(
|
|
2867
|
-
frames.find((frame) => frame.type === "utterance_end"),
|
|
2868
|
-
).toMatchObject({ reason: "silence" });
|
|
2869
|
-
});
|
|
2870
|
-
|
|
2871
|
-
test("a rejecting decider still releases the boundary with the normal frame sequence", async () => {
|
|
2872
|
-
// The decider contract never rejects, but a buggy injected stub (or a
|
|
2873
|
-
// future regression) must not silently drop the silence boundary: the
|
|
2874
|
-
// session's belt catches the rejection and releases.
|
|
2875
|
-
const calls: VoiceEndpointDecisionInput[] = [];
|
|
2876
|
-
const decider: VoiceFrontDecider = {
|
|
2877
|
-
decideEndpoint: async (input) => {
|
|
2878
|
-
calls.push(input);
|
|
2879
|
-
throw new Error("stub decider boom");
|
|
2880
|
-
},
|
|
2881
|
-
generateAckText: async () => null,
|
|
2882
|
-
generateProgressText: async () => null,
|
|
2883
|
-
};
|
|
2884
|
-
const turnStarter = makeAutoCompletingTurnStarter(["Hi there."]);
|
|
2885
|
-
const { frames, session, transcribers } = createHarness({
|
|
2886
|
-
finals: ["hello world"],
|
|
2887
|
-
startVoiceTurn: turnStarter.startVoiceTurn,
|
|
2888
|
-
frontDecider: decider,
|
|
2889
|
-
});
|
|
2890
|
-
|
|
2891
|
-
await startWithPartial(session, transcribers);
|
|
2892
|
-
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
2893
|
-
await waitFor(() => frames.some((frame) => frame.type === "tts_done"));
|
|
2894
|
-
|
|
2895
|
-
expect(calls).toHaveLength(1);
|
|
2896
|
-
expect(frameTypes(frames)).toEqual([
|
|
2897
|
-
"ready",
|
|
2898
|
-
"stt_partial",
|
|
2899
|
-
"speech_started",
|
|
2900
|
-
"utterance_end",
|
|
2901
|
-
"stt_final",
|
|
2902
|
-
"thinking",
|
|
2903
|
-
"assistant_text_delta",
|
|
2904
|
-
"tts_done",
|
|
2905
|
-
]);
|
|
2906
|
-
expect(
|
|
2907
|
-
frames.find((frame) => frame.type === "utterance_end"),
|
|
2908
|
-
).toMatchObject({ reason: "silence" });
|
|
2909
|
-
expect(turnStarter.calls[0]).toMatchObject({ content: "hello world" });
|
|
2910
|
-
});
|
|
2911
|
-
|
|
2912
|
-
test("a slow decider adds only its own bounded delay and then releases normally", async () => {
|
|
2913
|
-
// Resolves "release" only after a delay comfortably longer than every
|
|
2914
|
-
// other timer in the flow — the boundary must wait it out and then run
|
|
2915
|
-
// the unchanged release sequence.
|
|
2916
|
-
const calls: VoiceEndpointDecisionInput[] = [];
|
|
2917
|
-
const decider: VoiceFrontDecider = {
|
|
2918
|
-
decideEndpoint: async (input) => {
|
|
2919
|
-
calls.push(input);
|
|
2920
|
-
await new Promise((resolve) => setTimeout(resolve, 150));
|
|
2921
|
-
return { action: "release" };
|
|
2922
|
-
},
|
|
2923
|
-
generateAckText: async () => null,
|
|
2924
|
-
generateProgressText: async () => null,
|
|
2925
|
-
};
|
|
2926
|
-
const turnStarter = makeAutoCompletingTurnStarter(["Hi there."]);
|
|
2927
|
-
const { frames, session, transcribers } = createHarness({
|
|
2928
|
-
finals: ["hello world"],
|
|
2929
|
-
startVoiceTurn: turnStarter.startVoiceTurn,
|
|
2930
|
-
frontDecider: decider,
|
|
2931
|
-
});
|
|
2932
|
-
|
|
2933
|
-
await startWithPartial(session, transcribers);
|
|
2934
|
-
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
2935
|
-
|
|
2936
|
-
// While the decision is pending, the boundary stays unreleased.
|
|
2937
|
-
await waitFor(() => calls.length === 1);
|
|
2938
|
-
expect(countType(frames, "utterance_end")).toBe(0);
|
|
2939
|
-
expect(countType(frames, "thinking")).toBe(0);
|
|
2940
|
-
|
|
2941
|
-
await waitFor(() => frames.some((frame) => frame.type === "tts_done"));
|
|
2942
|
-
expect(calls).toHaveLength(1);
|
|
2943
|
-
expect(frameTypes(frames)).toEqual([
|
|
2944
|
-
"ready",
|
|
2945
|
-
"stt_partial",
|
|
2946
|
-
"speech_started",
|
|
2947
|
-
"utterance_end",
|
|
2948
|
-
"stt_final",
|
|
2949
|
-
"thinking",
|
|
2950
|
-
"assistant_text_delta",
|
|
2951
|
-
"tts_done",
|
|
2952
|
-
]);
|
|
2953
|
-
expect(
|
|
2954
|
-
frames.find((frame) => frame.type === "utterance_end"),
|
|
2955
|
-
).toMatchObject({ reason: "silence" });
|
|
2956
|
-
expect(turnStarter.calls[0]).toMatchObject({ content: "hello world" });
|
|
2957
|
-
});
|
|
2958
|
-
|
|
2959
|
-
test("endpointMaxExtensions caps consecutive holds and forces the release", async () => {
|
|
2960
|
-
const { decider, calls } = makeFrontDecider(["hold", "hold", "hold"]);
|
|
2961
|
-
const turnStarter = makeAutoCompletingTurnStarter(["Hi there."]);
|
|
2962
|
-
const { frames, session, transcribers } = createHarness({
|
|
2963
|
-
finals: ["hello world"],
|
|
2964
|
-
startVoiceTurn: turnStarter.startVoiceTurn,
|
|
2965
|
-
frontDecider: decider,
|
|
2966
|
-
frontModelConfig: { endpointExtensionMs: 20, endpointMaxExtensions: 1 },
|
|
2967
|
-
});
|
|
2968
|
-
|
|
2969
|
-
await startWithPartial(session, transcribers);
|
|
2970
|
-
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
2971
|
-
await waitFor(() => frames.some((frame) => frame.type === "tts_done"));
|
|
2972
|
-
|
|
2973
|
-
// The single allowed hold consumed the decider's one consult; the replay
|
|
2974
|
-
// hit the cap and released without asking again.
|
|
2975
|
-
expect(calls).toHaveLength(1);
|
|
2976
|
-
expect(countType(frames, "utterance_end")).toBe(1);
|
|
2977
|
-
expect(turnStarter.calls).toHaveLength(1);
|
|
2978
|
-
expect(turnStarter.calls[0]).toMatchObject({ content: "hello world" });
|
|
2979
|
-
});
|
|
2980
|
-
|
|
2981
|
-
test("a max-duration boundary bypasses the decider entirely", async () => {
|
|
2982
|
-
const { decider, calls } = makeFrontDecider(["hold"]);
|
|
2983
|
-
const turnStarter = makeAutoCompletingTurnStarter(["Hi there."]);
|
|
2984
|
-
const { frames, session, transcribers } = createHarness({
|
|
2985
|
-
finals: ["hello world"],
|
|
2986
|
-
startVoiceTurn: turnStarter.startVoiceTurn,
|
|
2987
|
-
frontDecider: decider,
|
|
2988
|
-
// Silence can never fire; the max-duration cap ends the turn instead.
|
|
2989
|
-
turnDetectorConfig: { silenceThresholdMs: 10_000, maxTurnDurationMs: 40 },
|
|
2990
|
-
});
|
|
2991
|
-
|
|
2992
|
-
await startWithPartial(session, transcribers);
|
|
2993
|
-
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
2994
|
-
await waitFor(() => frames.some((frame) => frame.type === "tts_done"));
|
|
2995
|
-
|
|
2996
|
-
expect(calls).toHaveLength(0);
|
|
2997
|
-
expect(
|
|
2998
|
-
frames.find((frame) => frame.type === "utterance_end"),
|
|
2999
|
-
).toMatchObject({ reason: "max-duration" });
|
|
3000
|
-
expect(turnStarter.calls).toHaveLength(1);
|
|
3001
|
-
});
|
|
3002
|
-
|
|
3003
|
-
test("a ptt_release forced boundary bypasses the decider and releases immediately", async () => {
|
|
3004
|
-
// A hold-happy decider: if the manual release were routed through it,
|
|
3005
|
-
// the boundary would be deferred and the frame sequence below would gain
|
|
3006
|
-
// an extension delay (or stall entirely).
|
|
3007
|
-
const { decider, calls } = makeFrontDecider(["hold", "hold"]);
|
|
3008
|
-
const turnStarter = makeAutoCompletingTurnStarter(["Hi there."]);
|
|
3009
|
-
const { frames, session, transcribers } = createHarness({
|
|
3010
|
-
finals: ["hello world"],
|
|
3011
|
-
startVoiceTurn: turnStarter.startVoiceTurn,
|
|
3012
|
-
frontDecider: decider,
|
|
3013
|
-
// A genuine silence boundary can never fire during the test; only the
|
|
3014
|
-
// client's explicit release ends the turn.
|
|
3015
|
-
turnDetectorConfig: { silenceThresholdMs: 10_000 },
|
|
3016
|
-
});
|
|
3017
|
-
|
|
3018
|
-
await startWithPartial(session, transcribers);
|
|
3019
|
-
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
3020
|
-
await waitFor(() =>
|
|
3021
|
-
frames.some((frame) => frame.type === "speech_started"),
|
|
3022
|
-
);
|
|
3023
|
-
|
|
3024
|
-
await session.handleClientFrame({ type: "ptt_release" });
|
|
3025
|
-
await waitFor(() => frames.some((frame) => frame.type === "tts_done"));
|
|
3026
|
-
|
|
3027
|
-
// The explicit release was never second-guessed: no decider consult, and
|
|
3028
|
-
// the boundary produced the normal utterance_end → turn-launch sequence.
|
|
3029
|
-
expect(calls).toHaveLength(0);
|
|
3030
|
-
expect(frameTypes(frames)).toEqual([
|
|
3031
|
-
"ready",
|
|
3032
|
-
"stt_partial",
|
|
3033
|
-
"speech_started",
|
|
3034
|
-
"utterance_end",
|
|
3035
|
-
"stt_final",
|
|
3036
|
-
"thinking",
|
|
3037
|
-
"assistant_text_delta",
|
|
3038
|
-
"tts_done",
|
|
3039
|
-
]);
|
|
3040
|
-
expect(
|
|
3041
|
-
frames.find((frame) => frame.type === "utterance_end"),
|
|
3042
|
-
).toMatchObject({ reason: "silence" });
|
|
3043
|
-
expect(turnStarter.calls).toHaveLength(1);
|
|
3044
|
-
expect(turnStarter.calls[0]).toMatchObject({ content: "hello world" });
|
|
3045
|
-
});
|
|
3046
|
-
|
|
3047
|
-
test("session close during a hold clears the extension timer", async () => {
|
|
3048
|
-
const { decider, calls } = makeFrontDecider(["hold", "hold"]);
|
|
3049
|
-
const { frames, session, transcribers } = createHarness({
|
|
3050
|
-
finals: ["hello world"],
|
|
3051
|
-
frontDecider: decider,
|
|
3052
|
-
frontModelConfig: { endpointExtensionMs: 30 },
|
|
3053
|
-
});
|
|
3054
|
-
|
|
3055
|
-
await startWithPartial(session, transcribers);
|
|
3056
|
-
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
3057
|
-
await waitFor(() => calls.length === 1);
|
|
3058
|
-
expect(countType(frames, "utterance_end")).toBe(0);
|
|
3059
|
-
|
|
3060
|
-
await session.close("client_end");
|
|
3061
|
-
const framesAtClose = frames.length;
|
|
3062
|
-
|
|
3063
|
-
// Well past the extension window: the cleared timer never replays the
|
|
3064
|
-
// boundary — no utterance_end, no second decider consult, no new frames.
|
|
3065
|
-
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
3066
|
-
expect(calls).toHaveLength(1);
|
|
3067
|
-
expect(countType(frames, "utterance_end")).toBe(0);
|
|
3068
|
-
expect(frames.length).toBe(framesAtClose);
|
|
3069
|
-
});
|
|
3070
|
-
});
|
|
3071
|
-
|
|
3072
3620
|
describe("LiveVoiceSession unified front-door endpointing", () => {
|
|
3073
|
-
const enableUnifiedFlags = () =>
|
|
3074
|
-
setCachedOverrides(
|
|
3075
|
-
{
|
|
3076
|
-
"voice-mode": true,
|
|
3077
|
-
},
|
|
3078
|
-
{ fromGateway: true },
|
|
3079
|
-
);
|
|
3080
|
-
|
|
3081
|
-
afterEach(() => clearCachedOverrides());
|
|
3082
|
-
|
|
3083
3621
|
function spokenDeltaText(frames: LiveVoiceServerFrame[]): string {
|
|
3084
3622
|
return frames
|
|
3085
3623
|
.filter((frame) => frame.type === "assistant_text_delta")
|
|
@@ -3114,23 +3652,12 @@ describe("LiveVoiceSession unified front-door endpointing", () => {
|
|
|
3114
3652
|
return { startVoiceTurn, calls, discard };
|
|
3115
3653
|
}
|
|
3116
3654
|
|
|
3117
|
-
// A decider that
|
|
3118
|
-
//
|
|
3119
|
-
function
|
|
3120
|
-
decider: VoiceFrontDecider;
|
|
3121
|
-
endpointCalls: () => number;
|
|
3122
|
-
} {
|
|
3123
|
-
let count = 0;
|
|
3655
|
+
// A decider that phrases nothing: the front-door leg IS the endpoint
|
|
3656
|
+
// decision, so these tests only need the decider seam to exist.
|
|
3657
|
+
function makeSilentDecider(): VoiceFrontDecider {
|
|
3124
3658
|
return {
|
|
3125
|
-
|
|
3126
|
-
|
|
3127
|
-
count += 1;
|
|
3128
|
-
return { action: "release" as const };
|
|
3129
|
-
},
|
|
3130
|
-
generateAckText: async () => null,
|
|
3131
|
-
generateProgressText: async () => null,
|
|
3132
|
-
},
|
|
3133
|
-
endpointCalls: () => count,
|
|
3659
|
+
generateAckText: async () => null,
|
|
3660
|
+
generateProgressText: async () => null,
|
|
3134
3661
|
};
|
|
3135
3662
|
}
|
|
3136
3663
|
|
|
@@ -3146,8 +3673,7 @@ describe("LiveVoiceSession unified front-door endpointing", () => {
|
|
|
3146
3673
|
}
|
|
3147
3674
|
|
|
3148
3675
|
test("a chatty answer commits: verdict leg replaces the decider, frames follow commit order", async () => {
|
|
3149
|
-
|
|
3150
|
-
const { decider, endpointCalls } = makeUntouchableDecider();
|
|
3676
|
+
const decider = makeSilentDecider();
|
|
3151
3677
|
const starter = makeVerdictTurnStarter([["Hey! Not much."]]);
|
|
3152
3678
|
const { frames, session, transcribers } = createHarness({
|
|
3153
3679
|
finals: ["hello world"],
|
|
@@ -3159,8 +3685,7 @@ describe("LiveVoiceSession unified front-door endpointing", () => {
|
|
|
3159
3685
|
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
3160
3686
|
await waitFor(() => frames.some((frame) => frame.type === "tts_done"));
|
|
3161
3687
|
|
|
3162
|
-
// The leg IS the endpoint decision
|
|
3163
|
-
expect(endpointCalls()).toBe(0);
|
|
3688
|
+
// The leg IS the endpoint decision.
|
|
3164
3689
|
expect(starter.calls).toHaveLength(1);
|
|
3165
3690
|
// Dispatched speculatively on the pre-finalize transcript, as the
|
|
3166
3691
|
// front-door leg with the verdict rule requested.
|
|
@@ -3182,7 +3707,6 @@ describe("LiveVoiceSession unified front-door endpointing", () => {
|
|
|
3182
3707
|
});
|
|
3183
3708
|
|
|
3184
3709
|
test("a hold verdict discards the leg silently and the extension replays the boundary", async () => {
|
|
3185
|
-
enableUnifiedFlags();
|
|
3186
3710
|
const starter = makeVerdictTurnStarter([["[0]"], ["Sure thing."]]);
|
|
3187
3711
|
const { frames, session, transcribers } = createHarness({
|
|
3188
3712
|
finals: ["hello world"],
|
|
@@ -3215,7 +3739,6 @@ describe("LiveVoiceSession unified front-door endpointing", () => {
|
|
|
3215
3739
|
});
|
|
3216
3740
|
|
|
3217
3741
|
test("a verdict that misses the deadline commits the turn (fail-open)", async () => {
|
|
3218
|
-
enableUnifiedFlags();
|
|
3219
3742
|
// The leg never produces a verdict — a provider TTFT tail. The deadline
|
|
3220
3743
|
// must commit the turn so the caller gets the thinking frame (and the
|
|
3221
3744
|
// ack timer arms) instead of unbounded structural silence.
|
|
@@ -3240,7 +3763,6 @@ describe("LiveVoiceSession unified front-door endpointing", () => {
|
|
|
3240
3763
|
});
|
|
3241
3764
|
|
|
3242
3765
|
test("a final that extends the held transcript replays the boundary immediately", async () => {
|
|
3243
|
-
enableUnifiedFlags();
|
|
3244
3766
|
const starter = makeVerdictTurnStarter([["[0]"], ["Got it."]]);
|
|
3245
3767
|
const { frames, session, transcribers } = createHarness({
|
|
3246
3768
|
finals: ["hello world how are you"],
|
|
@@ -3265,7 +3787,6 @@ describe("LiveVoiceSession unified front-door endpointing", () => {
|
|
|
3265
3787
|
});
|
|
3266
3788
|
|
|
3267
3789
|
test("speech resuming mid-verdict discards the leg and the utterance keeps accumulating", async () => {
|
|
3268
|
-
enableUnifiedFlags();
|
|
3269
3790
|
// First leg never produces a verdict (in flight); second answers.
|
|
3270
3791
|
const starter = makeVerdictTurnStarter([[], ["Got it all."]]);
|
|
3271
3792
|
const { frames, session, transcribers } = createHarness({
|
|
@@ -3294,7 +3815,6 @@ describe("LiveVoiceSession unified front-door endpointing", () => {
|
|
|
3294
3815
|
});
|
|
3295
3816
|
|
|
3296
3817
|
test("a discard that beats the bridge handle still rolls back the user row", async () => {
|
|
3297
|
-
enableUnifiedFlags();
|
|
3298
3818
|
const discard = mock(async () => {});
|
|
3299
3819
|
const abort = mock();
|
|
3300
3820
|
let resolveHandle: (() => void) | null = null;
|
|
@@ -3336,7 +3856,6 @@ describe("LiveVoiceSession unified front-door endpointing", () => {
|
|
|
3336
3856
|
});
|
|
3337
3857
|
|
|
3338
3858
|
test("a manual release during the verdict window commits the leg instead of discarding it", async () => {
|
|
3339
|
-
enableUnifiedFlags();
|
|
3340
3859
|
const discard = mock(async () => {});
|
|
3341
3860
|
let callbacks: VoiceTurnCallbacks | undefined;
|
|
3342
3861
|
const calls: VoiceTurnOptions[] = [];
|
|
@@ -3376,7 +3895,6 @@ describe("LiveVoiceSession unified front-door endpointing", () => {
|
|
|
3376
3895
|
});
|
|
3377
3896
|
|
|
3378
3897
|
test("a hold verdict after a manual release relaunches a fresh leg on the released utterance", async () => {
|
|
3379
|
-
enableUnifiedFlags();
|
|
3380
3898
|
const discard = mock(async () => {});
|
|
3381
3899
|
let firstCallbacks: VoiceTurnCallbacks | undefined;
|
|
3382
3900
|
const calls: VoiceTurnOptions[] = [];
|