@vellumai/assistant 0.12.2 → 0.12.3-staging.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +2 -0
- package/AGENTS.md +1 -1
- package/ARCHITECTURE.md +43 -29
- package/Dockerfile +63 -0
- package/docs/architecture/integrations.md +7 -0
- package/docs/architecture/memory.md +17 -2
- package/docs/architecture/security.md +10 -5
- package/docs/architecture/turn-actor.md +2 -0
- package/docs/credential-execution-service.md +1 -1
- package/docs/desktop-browser-cli.md +50 -5
- package/docs/live-voice-task-outcomes.md +100 -0
- package/docs/mode-sessions.md +183 -0
- package/docs/voice-input-diagnostics.md +58 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/package.json +2 -1
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +1 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/secret-detection.ts +4 -1
- package/node_modules/@vellumai/environments/src/shell.test.ts +21 -0
- package/node_modules/@vellumai/environments/src/shell.ts +24 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/package.json +2 -1
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +1 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/secret-detection.ts +4 -1
- package/node_modules/@vellumai/gateway-client/src/gateway-ipc-contracts.ts +20 -0
- package/node_modules/@vellumai/gateway-client/src/inbound-contract.ts +8 -2
- package/node_modules/@vellumai/gateway-client/src/index.ts +4 -1
- package/node_modules/@vellumai/gateway-client/src/outbound-contract.ts +38 -7
- package/node_modules/@vellumai/gateway-client/src/trust-verdict-contract.ts +5 -6
- package/node_modules/@vellumai/service-contracts/package.json +2 -1
- package/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +1 -0
- package/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
- package/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
- package/node_modules/@vellumai/service-contracts/src/secret-detection.ts +4 -1
- package/node_modules/@vellumai/slack-text/src/index.ts +4 -8
- package/openapi.yaml +716 -16
- package/package.json +1 -1
- package/scripts/smoke-desktop-browser-cli.ts +11 -26
- package/scripts/smoke-desktop-window-theme.ts +2 -3
- package/scripts/sync-llm-catalog.ts +3 -0
- package/scripts/voice-ttft-spike.ts +2 -2
- package/src/__tests__/agent-loop-output-hooks.test.ts +150 -1
- package/src/__tests__/agent-loop-provider-error-recording.test.ts +87 -19
- package/src/__tests__/agent-loop.test.ts +259 -0
- package/src/__tests__/agent-wake-delegation-prompt.test.ts +64 -2
- package/src/__tests__/annotate-activity-metadata.test.ts +3 -1
- package/src/__tests__/annotate-risk-options.test.ts +2 -0
- package/src/__tests__/answered-question-persistence.test.ts +2 -0
- package/src/__tests__/anthropic-gateway-shared.test.ts +22 -0
- package/src/__tests__/anthropic-provider.test.ts +55 -0
- package/src/__tests__/approval-interception-trust-gates.test.ts +40 -0
- package/src/__tests__/approval-message-composer.test.ts +1 -118
- package/src/__tests__/approval-routes-http.test.ts +3 -0
- package/src/__tests__/assistant-event-hub-connection-history.test.ts +111 -0
- package/src/__tests__/assistant-event-hub-targeted.test.ts +50 -0
- package/src/__tests__/assistant-event-targeting.test.ts +186 -0
- package/src/__tests__/assistant-stream-state.test.ts +29 -9
- package/src/__tests__/attachments-store.test.ts +22 -3
- package/src/__tests__/browser-fill-credential.test.ts +11 -22
- package/src/__tests__/btw-routes.test.ts +4 -1
- package/src/__tests__/build-persisted-content.test.ts +1 -1
- package/src/__tests__/call-conversation-messages.test.ts +5 -0
- package/src/__tests__/call-recovery.test.ts +7 -2
- package/src/__tests__/call-routes-http.test.ts +12 -0
- package/src/__tests__/call-store.test.ts +7 -2
- package/src/__tests__/channel-approval.test.ts +9 -14
- package/src/__tests__/channel-reply-delivery.test.ts +56 -0
- package/src/__tests__/channel-retry-sweep.test.ts +9 -0
- package/src/__tests__/chat-credential-redaction.test.ts +23 -0
- package/src/__tests__/chat-reveal-guard-priming.test.ts +2 -0
- package/src/__tests__/checker.test.ts +11 -3
- package/src/__tests__/client-os-metadata-persistence.test.ts +35 -1
- package/src/__tests__/computer-use-screenshot-attachments.test.ts +411 -0
- package/src/__tests__/computer-use-screenshot-selection.test.ts +50 -0
- package/src/__tests__/computer-use-skill-manifest-regression.test.ts +0 -25
- package/src/__tests__/computer-use-tools.test.ts +1 -233
- package/src/__tests__/config-model-image-provider.test.ts +39 -0
- package/src/__tests__/config-schema.test.ts +16 -0
- package/src/__tests__/config-watcher.test.ts +24 -0
- package/src/__tests__/context-search-agent-runner.test.ts +260 -1
- package/src/__tests__/context-search-fanout.test.ts +1 -1
- package/src/__tests__/context-search-types.test.ts +7 -7
- package/src/__tests__/conversation-agent-loop-disk-pressure.test.ts +8 -2
- package/src/__tests__/conversation-agent-loop-fatal-cleanup.test.ts +2 -0
- package/src/__tests__/conversation-agent-loop-inference-profile.test.ts +2 -0
- package/src/__tests__/conversation-agent-loop-overflow.test.ts +2 -0
- package/src/__tests__/conversation-agent-loop.test.ts +797 -16
- package/src/__tests__/conversation-app-control-lifecycle.test.ts +53 -1
- package/src/__tests__/conversation-attachments.test.ts +106 -0
- package/src/__tests__/conversation-evictor.test.ts +62 -37
- package/src/__tests__/conversation-fork-crud.test.ts +191 -0
- package/src/__tests__/conversation-history-boundary-repair-failure.test.ts +174 -0
- package/src/__tests__/conversation-loop-db-lock-resilience.test.ts +2 -0
- package/src/__tests__/conversation-process-app-control-preactivation.test.ts +2 -0
- package/src/__tests__/conversation-rendered-delegation-state.test.ts +276 -0
- package/src/__tests__/conversation-retry-route.test.ts +22 -0
- package/src/__tests__/conversation-routes-disk-view.test.ts +2 -0
- package/src/__tests__/conversation-routes-enabled-plugins.test.ts +2 -0
- package/src/__tests__/conversation-routes-guardian-reply.test.ts +10 -0
- package/src/__tests__/conversation-routes-hidden-queue.test.ts +2 -0
- package/src/__tests__/conversation-routes-slash-commands.test.ts +2 -0
- package/src/__tests__/conversation-runtime-assembly.test.ts +29 -0
- package/src/__tests__/conversation-surface-snapshots.test.ts +543 -0
- package/src/__tests__/conversation-surfaces-action-delivery.test.ts +151 -0
- package/src/__tests__/conversation-surfaces-point-at-budget.test.ts +63 -1
- package/src/__tests__/conversation-tool-setup-attribution.test.ts +23 -0
- package/src/__tests__/conversation-undo.test.ts +61 -0
- package/src/__tests__/credential-execution-client.test.ts +72 -55
- package/src/__tests__/credential-prompt-route.test.ts +35 -3
- package/src/__tests__/cu-unified-flow.test.ts +82 -1
- package/src/__tests__/db-conversation-tool-surface.test.ts +221 -0
- package/src/__tests__/db-mode-sessions-migration.test.ts +175 -0
- package/src/__tests__/deterministic-verification-control-plane.test.ts +2 -0
- package/src/__tests__/dm-backfill.test.ts +91 -1
- package/src/__tests__/document-create-route.test.ts +88 -0
- package/src/__tests__/document-sync-tags.test.ts +27 -0
- package/src/__tests__/emit-signal-routing-intent.test.ts +142 -2
- package/src/__tests__/events-tail-route.test.ts +58 -0
- package/src/__tests__/evict-conversations-for-reload.test.ts +57 -8
- package/src/__tests__/guardian-action-sweep.test.ts +16 -0
- package/src/__tests__/guardian-dispatch.test.ts +13 -0
- package/src/__tests__/headless-browser-interactions.test.ts +64 -20
- package/src/__tests__/heartbeat-service.test.ts +2 -59
- package/src/__tests__/helpers/mock-conversation.ts +21 -0
- package/src/__tests__/helpers/native-web-search-harness.ts +2 -0
- package/src/__tests__/helpers/verification-sessions-ipc-sim.ts +5 -0
- package/src/__tests__/history-repair.test.ts +53 -0
- package/src/__tests__/host-cu-proxy.test.ts +46 -2
- package/src/__tests__/http-user-message-parity.test.ts +2 -0
- package/src/__tests__/image-credentials.test.ts +49 -0
- package/src/__tests__/image-service-dispatcher.test.ts +45 -0
- package/src/__tests__/inference-profile-session-handler.test.ts +26 -0
- package/src/__tests__/interrupt-on-send.test.ts +57 -43
- package/src/__tests__/interrupt-turn-note.test.ts +5 -6
- package/src/__tests__/list-messages-tool-merge.test.ts +85 -1
- package/src/__tests__/llm-catalog-parity.test.ts +27 -4
- package/src/__tests__/media-generate-image.test.ts +49 -1
- package/src/__tests__/message-provenance-contact-id.test.ts +69 -0
- package/src/__tests__/migration-export-to-gcs.test.ts +142 -0
- package/src/__tests__/mode-session-history.test.ts +245 -0
- package/src/__tests__/mode-session-recovery.test.ts +222 -0
- package/src/__tests__/mode-session-wire.test.ts +141 -0
- package/src/__tests__/notification-guardian-path.test.ts +7 -0
- package/src/__tests__/notification-telegram-adapter.test.ts +2 -2
- package/src/__tests__/notification-vellum-adapter.test.ts +34 -0
- package/src/__tests__/oauth-apps-routes.test.ts +1 -0
- package/src/__tests__/oauth-commands-routes.test.ts +114 -101
- package/src/__tests__/oauth-connect-orchestrator.test.ts +2 -0
- package/src/__tests__/oauth-provider-serializer.test.ts +13 -0
- package/src/__tests__/oauth-providers-routes.test.ts +3 -0
- package/src/__tests__/outbound-slack-persistence.test.ts +2 -0
- package/src/__tests__/persist-media-references.test.ts +50 -0
- package/src/__tests__/persistence-secret-redaction.test.ts +2 -0
- package/src/__tests__/plugin-api-provider.test.ts +5 -0
- package/src/__tests__/plugin-api-resolve-credential.test.ts +97 -0
- package/src/__tests__/plugin-import-boundary-guard.test.ts +9 -1
- package/src/__tests__/process-message-display-content.test.ts +56 -0
- package/src/__tests__/reaction-persistence.test.ts +5 -0
- package/src/__tests__/request-secret-standalone-channel-gate.test.ts +25 -9
- package/src/__tests__/run-conversation-turn-persistence.test.ts +138 -1
- package/src/__tests__/runtime-events-sse-reconnect.test.ts +1 -1
- package/src/__tests__/schedule-routes.test.ts +20 -0
- package/src/__tests__/scheduler-result-notification.test.ts +23 -4
- package/src/__tests__/script-proxy-certs.test.ts +1 -1
- package/src/__tests__/scripted-turn-metadata-persistence.test.ts +73 -0
- package/src/__tests__/secret-ingress-http.test.ts +2 -0
- package/src/__tests__/secret-routes-platform-proxy.test.ts +86 -31
- package/src/__tests__/secret-routes-scrub.test.ts +22 -27
- package/src/__tests__/secret-scanner.test.ts +20 -0
- package/src/__tests__/secure-keys-managed-failover.test.ts +8 -0
- package/src/__tests__/secure-keys.test.ts +50 -4
- package/src/__tests__/send-endpoint-busy.test.ts +163 -1
- package/src/__tests__/server-history-render.test.ts +91 -1
- package/src/__tests__/server-tool-pairing.test.ts +107 -0
- package/src/__tests__/settings-routes.test.ts +50 -1
- package/src/__tests__/skill-script-runner-sandbox.test.ts +32 -0
- package/src/__tests__/skills.test.ts +5 -4
- package/src/__tests__/subagent-fork-notifications.test.ts +1 -2
- package/src/__tests__/subagent-manager-notify.test.ts +1 -2
- package/src/__tests__/subagent-notify-parent.test.ts +146 -39
- package/src/__tests__/subagent-notify-stale-rebuild.test.ts +2 -4
- package/src/__tests__/subagent-tool-gate-mode.test.ts +226 -0
- package/src/__tests__/surface-completion-in-flight-snapshot.test.ts +2 -0
- package/src/__tests__/terminal-tools.test.ts +8 -0
- package/src/__tests__/thread-backfill.test.ts +53 -0
- package/src/__tests__/tool-preview-lifecycle.test.ts +2 -0
- package/src/__tests__/tool-result-follow-up.test.ts +99 -0
- package/src/__tests__/tool-result-metadata-plumbing.test.ts +66 -1
- package/src/__tests__/tool-start-timestamp.test.ts +2 -0
- package/src/__tests__/twilio-routes.test.ts +1 -0
- package/src/__tests__/unicode.test.ts +36 -0
- package/src/__tests__/web-search-backend-failure.test.ts +1 -1
- package/src/activation/progress-store.test.ts +12 -7
- package/src/agent/loop.ts +136 -53
- package/src/agent/tool-result-follow-up.ts +70 -0
- package/src/api/attachment-provenance.test.ts +68 -0
- package/src/api/computer-use-tool.test.ts +55 -0
- package/src/api/computer-use-tool.ts +27 -0
- package/src/api/events/assistant-outbound-attachment.ts +3 -0
- package/src/api/events/assistant-turn-start.ts +3 -0
- package/src/api/events/desktop-activity-changed.ts +10 -0
- package/src/api/events/generation-handoff.ts +2 -0
- package/src/api/events/message-complete.ts +2 -0
- package/src/api/events/notification-conversation-created.ts +14 -16
- package/src/api/events/notification-intent.ts +4 -2
- package/src/api/events/question-request.ts +1 -0
- package/src/api/events/tool-result.ts +85 -0
- package/src/api/events/tool-use-start.ts +3 -0
- package/src/api/events/user-message-echo.ts +3 -0
- package/src/api/index.ts +42 -6
- package/src/api/mode-session.test.ts +131 -0
- package/src/api/mode-session.ts +148 -0
- package/src/api/remember-tool.ts +15 -0
- package/src/api/responses/conversation-message.ts +9 -0
- package/src/approvals/AGENTS.md +4 -2
- package/src/approvals/approval-primitive.ts +5 -2
- package/src/approvals/scoped-approval-grants.ts +6 -2
- package/src/browser/__tests__/operations.test.ts +29 -0
- package/src/browser/operation-meta.ts +2 -1
- package/src/browser/operations.ts +29 -0
- package/src/browser/virtual-desktop-target.ts +2 -9
- package/src/bundler/bundle-signer.ts +1 -3
- package/src/calls/__tests__/barge-in-guard.test.ts +3 -0
- package/src/calls/__tests__/call-controller.test.ts +94 -0
- package/src/calls/__tests__/call-funnel-telemetry.test.ts +204 -0
- package/src/calls/__tests__/voice-control-protocol.test.ts +15 -0
- package/src/calls/__tests__/voice-escalation-judge.test.ts +209 -0
- package/src/calls/__tests__/voice-leg-coordinator.test.ts +168 -0
- package/src/{live-voice/__tests__/live-voice-metrics.test.ts → calls/__tests__/voice-metrics.test.ts} +32 -38
- package/src/calls/__tests__/voice-session-bridge.test.ts +576 -122
- package/src/calls/barge-in-guard.ts +5 -0
- package/src/calls/call-controller.ts +196 -1
- package/src/calls/call-domain.ts +4 -0
- package/src/calls/call-store.ts +87 -2
- package/src/calls/guardian-action-sweep.ts +22 -8
- package/src/calls/media-stream-audio-transcode.ts +1 -3
- package/src/calls/typesafe-noul.ts +156 -0
- package/src/calls/voice-control-protocol.ts +6 -0
- package/src/calls/voice-escalation-judge.ts +145 -0
- package/src/calls/voice-escalation-target.ts +10 -0
- package/src/calls/voice-leg-coordinator.ts +121 -3
- package/src/{live-voice/live-voice-metrics.ts → calls/voice-metrics.ts} +94 -87
- package/src/calls/voice-session-bridge.ts +256 -112
- package/src/calls/voice-triage-escalate.ts +7 -0
- package/src/cli/__tests__/roadmap-cli.test.ts +4 -41
- package/src/cli/commands/__tests__/cli-test-harness.ts +21 -3
- package/src/cli/commands/__tests__/notifications.test.ts +35 -10
- package/src/cli/commands/__tests__/plugins.test.ts +20 -1
- package/src/cli/commands/__tests__/schedules.test.ts +14 -0
- package/src/cli/commands/bash.help.ts +4 -3
- package/src/cli/commands/browser.help.ts +9 -6
- package/src/cli/commands/browser.ts +0 -1
- package/src/cli/commands/clients.help.ts +46 -0
- package/src/cli/commands/clients.ts +116 -0
- package/src/cli/commands/credentials.help.ts +9 -4
- package/src/cli/commands/notifications.help.ts +16 -2
- package/src/cli/commands/notifications.ts +30 -8
- package/src/cli/commands/oauth/index.help.ts +10 -0
- package/src/cli/commands/oauth/providers.ts +12 -0
- package/src/cli/commands/oauth/request.test.ts +179 -2
- package/src/cli/commands/oauth/request.ts +41 -10
- package/src/cli/commands/plugins.ts +12 -5
- package/src/cli/commands/roadmap.help.ts +18 -21
- package/src/cli/commands/roadmap.ts +4 -21
- package/src/cli/commands/schedules.ts +2 -0
- package/src/cli/lib/__tests__/inspect-plugin.test.ts +54 -0
- package/src/cli/lib/__tests__/install-from-github.test.ts +41 -0
- package/src/cli/lib/__tests__/list-installed-plugins.test.ts +5 -2
- package/src/cli/lib/__tests__/local-plugin-upgrade.test.ts +133 -51
- package/src/cli/lib/__tests__/plugin-catalog-cache.test.ts +8 -2
- package/src/cli/lib/bundled-marketplace.json +906 -1
- package/src/cli/lib/bundled-plugin-packages.json +783 -1
- package/src/cli/lib/inspect-plugin.ts +11 -4
- package/src/cli/lib/plugin-marketplace.ts +8 -0
- package/src/cli/lib/publish-plugin.ts +1 -2
- package/src/cli/lib/upgrade-plugin.ts +19 -11
- package/src/cli/output.ts +18 -4
- package/src/config/__tests__/default-provider.test.ts +6 -0
- package/src/config/__tests__/memory-retrospective-schema.test.ts +14 -1
- package/src/config/bundled-skills/computer-use/SKILL.md +1 -1
- package/src/config/bundled-skills/computer-use/TOOLS.json +89 -1
- package/src/config/bundled-skills/image-studio/SKILL.md +17 -2
- package/src/config/bundled-skills/image-studio/tools/media-generate-image.ts +7 -11
- package/src/config/bundled-skills/schedule/SKILL.md +6 -6
- package/src/config/bundled-skills/sequences/TOOLS.json +1 -5
- package/src/config/call-site-defaults.ts +9 -0
- package/src/config/feature-flag-registry.json +16 -0
- package/src/config/loader.ts +9 -17
- package/src/config/platform-identity.ts +8 -8
- package/src/config/profile-text-generation.test.ts +51 -0
- package/src/config/profile-text-generation.ts +51 -0
- package/src/config/schemas/__tests__/memory-v3.test.ts +12 -0
- package/src/config/schemas/__tests__/tools.test.ts +31 -0
- package/src/config/schemas/call-site-catalog.ts +14 -0
- package/src/config/schemas/channels.ts +0 -18
- package/src/config/schemas/llm.ts +9 -2
- package/src/config/schemas/mcp.ts +6 -1
- package/src/config/schemas/memory-retrospective.ts +11 -2
- package/src/config/schemas/memory-v3.ts +12 -0
- package/src/config/schemas/services.ts +15 -6
- package/src/config/schemas/tools.ts +10 -0
- package/src/config/session-groups-gate.ts +21 -0
- package/src/config/skills.ts +2 -6
- package/src/conversations/__tests__/message-consolidation.test.ts +149 -0
- package/src/conversations/message-consolidation.ts +102 -5
- package/src/credential-execution/ces-connect.test.ts +69 -0
- package/src/credential-execution/ces-connect.ts +115 -0
- package/src/credential-execution/ces-runtime.ts +55 -93
- package/src/credential-execution/executable-discovery.ts +34 -142
- package/src/credential-execution/process-manager.test.ts +20 -24
- package/src/credential-execution/process-manager.ts +24 -19
- package/src/daemon/__tests__/conversation-surfaces-launch.test.ts +10 -1
- package/src/daemon/__tests__/conversation-tool-setup.test.ts +43 -0
- package/src/daemon/__tests__/inference-profile-notification.test.ts +2 -0
- package/src/daemon/__tests__/turn-tail-deleted-conversation.test.ts +36 -0
- package/src/daemon/assistant-attachments.ts +20 -12
- package/src/daemon/browser-mode-session.test.ts +402 -0
- package/src/daemon/browser-mode-session.ts +186 -0
- package/src/daemon/chat-credential-redaction.ts +16 -1
- package/src/daemon/computer-use-mode-session.test.ts +210 -0
- package/src/daemon/computer-use-mode-session.ts +105 -0
- package/src/daemon/config-watcher.ts +10 -0
- package/src/daemon/conversation-agent-loop-handlers.ts +132 -13
- package/src/daemon/conversation-agent-loop.ts +231 -25
- package/src/daemon/conversation-attachments.ts +93 -12
- package/src/daemon/conversation-evictor.ts +5 -24
- package/src/daemon/conversation-history.ts +10 -0
- package/src/daemon/conversation-interrupt-repair.ts +4 -43
- package/src/daemon/conversation-interrupt.ts +7 -11
- package/src/daemon/conversation-messaging.ts +58 -5
- package/src/daemon/conversation-mode-session.test.ts +1667 -0
- package/src/daemon/conversation-mode-session.ts +1008 -0
- package/src/daemon/conversation-process.ts +5 -0
- package/src/daemon/conversation-runtime-assembly.ts +10 -20
- package/src/daemon/conversation-skill-tools.ts +1 -0
- package/src/daemon/conversation-store.ts +4 -4
- package/src/daemon/conversation-surface-snapshots.ts +135 -0
- package/src/daemon/conversation-surfaces.ts +113 -5
- package/src/daemon/conversation-tool-setup.ts +79 -6
- package/src/daemon/conversation-turn-finalize.ts +26 -13
- package/src/daemon/conversation.ts +215 -58
- package/src/daemon/daemon-control.ts +3 -7
- package/src/daemon/handlers/config-model.ts +7 -5
- package/src/daemon/handlers/shared.ts +55 -16
- package/src/daemon/host-cu-proxy.ts +51 -22
- package/src/daemon/lifecycle.ts +30 -13
- package/src/daemon/mcp-reload-service.ts +1 -1
- package/src/daemon/message-protocol.ts +0 -1
- package/src/daemon/message-provenance.ts +28 -4
- package/src/daemon/message-types/shared.ts +1 -0
- package/src/daemon/mode-session-startup-recovery.ts +23 -0
- package/src/daemon/mode-session-tracking.ts +54 -0
- package/src/daemon/orphan-reaper.ts +4 -3
- package/src/daemon/persist-media-references.ts +20 -3
- package/src/daemon/process-message.ts +13 -4
- package/src/daemon/tool-setup-types.ts +6 -0
- package/src/daemon/virtual-desktop-context.ts +21 -0
- package/src/daemon/wake-conversation-ops.ts +50 -15
- package/src/desktop/__tests__/fake-desktop.ts +1 -0
- package/src/desktop/desktop-accessibility-script.ts +88 -0
- package/src/desktop/desktop-accessibility.test.ts +160 -0
- package/src/desktop/desktop-accessibility.ts +169 -0
- package/src/desktop/desktop-automation-lease.test.ts +302 -64
- package/src/desktop/desktop-automation-lease.ts +129 -54
- package/src/desktop/desktop-browser-endpoint.ts +4 -0
- package/src/desktop/desktop-computer-use-routing.test.ts +330 -0
- package/src/desktop/desktop-computer-use.test.ts +449 -0
- package/src/desktop/desktop-computer-use.ts +502 -0
- package/src/desktop/desktop-dependencies.ts +47 -293
- package/src/desktop/desktop-display.ts +2 -2
- package/src/desktop/desktop-help.ts +15 -0
- package/src/desktop/desktop-panel-config.test.ts +13 -3
- package/src/desktop/desktop-panel-config.ts +109 -15
- package/src/desktop/desktop-session-manager.test.ts +69 -21
- package/src/desktop/desktop-session-manager.ts +76 -22
- package/src/desktop/desktop-stream-bridge.test.ts +76 -132
- package/src/desktop/desktop-stream-bridge.ts +9 -11
- package/src/desktop/desktop-wallpaper-renderer.ts +132 -0
- package/src/desktop/desktop-wallpaper-worker.ts +17 -0
- package/src/desktop/desktop-wallpaper.test.ts +48 -7
- package/src/desktop/desktop-wallpaper.ts +34 -102
- package/src/desktop/desktop-window-theme.ts +5 -25
- package/src/desktop/virtual-desktop-feature.ts +53 -3
- package/src/desktop/virtual-desktop-platform.test.ts +5 -9
- package/src/documents/document-store.ts +23 -0
- package/src/heartbeat/__tests__/heartbeat-service.test.ts +22 -26
- package/src/heartbeat/heartbeat-service.ts +5 -33
- package/src/ipc/__tests__/browser-ipc.test.ts +215 -6
- package/src/ipc/assistant-server.ts +2 -0
- package/src/ipc/routes/__tests__/channel-reply-ipc-routes.test.ts +111 -0
- package/src/ipc/routes/channel-reply-ipc-routes.ts +73 -0
- package/src/live-voice/__tests__/continuation-judge.test.ts +151 -0
- package/src/live-voice/__tests__/live-activity-reporter.test.ts +154 -15
- package/src/live-voice/__tests__/live-voice-flux-turn-end.test.ts +743 -31
- package/src/live-voice/__tests__/live-voice-look-follow-up.test.ts +191 -10
- package/src/live-voice/__tests__/live-voice-session-groups-gate.test.ts +87 -0
- package/src/live-voice/__tests__/live-voice-session-manager.test.ts +5 -1
- package/src/live-voice/__tests__/live-voice-sight-frame-echo.test.ts +41 -0
- package/src/live-voice/__tests__/live-voice-sight-frame.test.ts +1017 -9
- package/src/live-voice/__tests__/live-voice-triage-escalate.test.ts +224 -1
- package/src/live-voice/__tests__/live-voice-tts-session.test.ts +101 -0
- package/src/live-voice/__tests__/live-voice-tts.test.ts +44 -0
- package/src/live-voice/__tests__/live-voice-vad.test.ts +939 -75
- package/src/live-voice/__tests__/microphone-idle-gate.test.ts +87 -0
- package/src/live-voice/__tests__/session-controls.test.ts +23 -0
- package/src/live-voice/__tests__/task-outcomes.test.ts +117 -0
- package/src/live-voice/__tests__/voice-input-diagnostics.test.ts +117 -0
- package/src/live-voice/camera-mode-session.test.ts +265 -0
- package/src/live-voice/camera-mode-session.ts +277 -0
- package/src/live-voice/continuation-judge.ts +123 -0
- package/src/live-voice/live-activity-reporter.ts +115 -6
- package/src/live-voice/live-voice-manager.ts +36 -1
- package/src/live-voice/live-voice-photo.ts +88 -7
- package/src/live-voice/live-voice-session-manager.ts +15 -0
- package/src/live-voice/live-voice-session.ts +1443 -747
- package/src/live-voice/live-voice-tts.ts +22 -4
- package/src/live-voice/microphone-idle-gate.ts +56 -0
- package/src/live-voice/protocol.ts +171 -4
- package/src/live-voice/session-controls.ts +11 -3
- package/src/live-voice/task-outcomes.ts +131 -0
- package/src/live-voice/voice-input-diagnostics.ts +117 -0
- package/src/mcp/__tests__/manager-tool-caps.test.ts +156 -0
- package/src/mcp/__tests__/startup.test.ts +30 -7
- package/src/mcp/__tests__/tool-caps.test.ts +134 -0
- package/src/mcp/manager.ts +171 -101
- package/src/mcp/startup.ts +33 -13
- package/src/mcp/tool-caps.ts +184 -0
- package/src/media/__tests__/image-models.test.ts +58 -0
- package/src/media/image-credentials.ts +5 -0
- package/src/media/image-models.ts +52 -0
- package/src/media/image-service.ts +8 -0
- package/src/media/openrouter-image-service.test.ts +133 -0
- package/src/media/openrouter-image-service.ts +153 -0
- package/src/media/types.ts +1 -1
- package/src/messaging/provider-message-metadata.ts +3 -3
- package/src/messaging/providers/__tests__/transport-dispatch.test.ts +65 -11
- package/src/messaging/providers/discord/send.ts +4 -17
- package/src/messaging/providers/discord/transport.ts +2 -2
- package/src/messaging/providers/outlook/client.ts +16 -22
- package/src/messaging/providers/telegram-bot/send.test.ts +236 -1
- package/src/messaging/providers/telegram-bot/send.ts +10 -7
- package/src/messaging/providers/telegram-bot/transport.ts +3 -10
- package/src/messaging/providers/whatsapp/send.test.ts +169 -4
- package/src/monitoring/__tests__/control.test.ts +17 -5
- package/src/monitoring/control.ts +15 -5
- package/src/notifications/AGENTS.md +1 -1
- package/src/notifications/README.md +83 -99
- package/src/notifications/__tests__/broadcaster.test.ts +1 -1
- package/src/notifications/__tests__/channel-allowlist.test.ts +39 -0
- package/src/notifications/__tests__/copy-composer.test.ts +71 -1
- package/src/notifications/__tests__/decision-engine.test.ts +79 -2
- package/src/notifications/__tests__/deterministic-checks.test.ts +10 -10
- package/src/notifications/__tests__/home-feed-side-effect.test.ts +93 -4
- package/src/notifications/__tests__/notification-utils.test.ts +18 -0
- package/src/notifications/adapters/macos.ts +37 -46
- package/src/notifications/broadcaster.ts +5 -6
- package/src/notifications/channel-allowlist.ts +44 -0
- package/src/notifications/copy-composer.ts +11 -26
- package/src/notifications/decision-engine.ts +24 -14
- package/src/notifications/deterministic-checks.ts +1 -1
- package/src/notifications/emit-signal.ts +123 -37
- package/src/notifications/home-feed-side-effect.ts +9 -7
- package/src/notifications/notification-utils.ts +19 -0
- package/src/notifications/schedule-result-producer.ts +4 -6
- package/src/notifications/signal.ts +4 -19
- package/src/notifications/types.ts +15 -0
- package/src/oauth/AGENTS.md +2 -0
- package/src/oauth/__tests__/identity-verifier.test.ts +36 -1
- package/src/oauth/connection-resolver.ts +1 -2
- package/src/oauth/identity-verifier.ts +24 -0
- package/src/oauth/oauth-store.ts +12 -4
- package/src/oauth/provider-serializer.ts +8 -0
- package/src/oauth/seed-providers.ts +42 -0
- package/src/onboarding/onboarding-events-store.ts +99 -12
- package/src/permissions/confirmation-guardian-request.test.ts +16 -0
- package/src/permissions/confirmation-guardian-request.ts +2 -3
- package/src/permissions/question-prompter.test.ts +31 -0
- package/src/permissions/question-prompter.ts +2 -0
- package/src/persistence/__tests__/client-connection-events-store.test.ts +322 -0
- package/src/persistence/attachments-store.ts +84 -44
- package/src/persistence/client-connection-events-store.ts +377 -0
- package/src/persistence/conversation-crud.ts +166 -26
- package/src/persistence/conversation-mode-sessions.test.ts +424 -0
- package/src/persistence/conversation-mode-sessions.ts +460 -0
- package/src/persistence/conversation-plugin-facade.ts +30 -0
- package/src/persistence/conversation-queries.ts +6 -19
- package/src/persistence/conversation-tool-surface.ts +135 -0
- package/src/persistence/conversation-types.test.ts +32 -0
- package/src/persistence/conversation-types.ts +26 -4
- package/src/persistence/llm-usage-store.ts +3 -9
- package/src/persistence/message-metadata.ts +25 -0
- package/src/persistence/migrations/378-create-conversation-tool-surfaces.test.ts +78 -0
- package/src/persistence/migrations/378-create-conversation-tool-surfaces.ts +29 -0
- package/src/persistence/migrations/379-oauth-providers-response-ok-field.test.ts +92 -0
- package/src/persistence/migrations/379-oauth-providers-response-ok-field.ts +32 -0
- package/src/persistence/migrations/380-conversation-tool-surfaces-delegate-independent-tasks.test.ts +90 -0
- package/src/persistence/migrations/380-conversation-tool-surfaces-delegate-independent-tasks.ts +34 -0
- package/src/persistence/migrations/381-create-conversation-mode-sessions.ts +41 -0
- package/src/persistence/migrations/382-create-client-connection-events.ts +40 -0
- package/src/persistence/migrations/__tests__/382-create-client-connection-events.test.ts +56 -0
- package/src/persistence/schema/conversation-tool-surfaces.ts +32 -0
- package/src/persistence/schema/conversations.ts +67 -0
- package/src/persistence/schema/index.ts +1 -0
- package/src/persistence/schema/infrastructure.ts +29 -0
- package/src/persistence/schema/oauth.ts +1 -0
- package/src/persistence/steps.ts +41 -3
- package/src/plugin-api/__tests__/plugin-name-env.test.ts +249 -0
- package/src/plugin-api/conversation-turn.ts +31 -7
- package/src/plugin-api/effective-context-window.ts +33 -0
- package/src/plugin-api/index.ts +35 -6
- package/src/plugin-api/plugin-channel-turn-trust.test.ts +133 -0
- package/src/plugin-api/plugin-channel-turn-trust.ts +71 -0
- package/src/plugin-api/plugin-name-env.ts +125 -0
- package/src/plugin-api/resolve-credential.ts +79 -20
- package/src/plugin-api/vision-support.test.ts +22 -0
- package/src/plugin-api/vision-support.ts +17 -5
- package/src/plugins/__tests__/active-plugin-env.test.ts +136 -0
- package/src/plugins/active-plugin-env.ts +51 -0
- package/src/plugins/defaults/index.ts +24 -0
- package/src/plugins/defaults/memory/AGENTS.md +39 -5
- package/src/plugins/defaults/memory/__tests__/archive-removal-framing.test.ts +64 -0
- package/src/plugins/defaults/memory/__tests__/buffer-file.test.ts +365 -0
- package/src/plugins/defaults/memory/__tests__/buffer-format.test.ts +43 -0
- package/src/plugins/defaults/memory/__tests__/conversation-memory-purge.test.ts +1 -0
- package/src/plugins/defaults/memory/__tests__/db-memory-attach.test.ts +2 -0
- package/src/plugins/defaults/memory/__tests__/fixtures/buffer-appender.ts +17 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-job.test.ts +74 -0
- package/src/plugins/defaults/memory/__tests__/memory-run-evidence.test.ts +161 -0
- package/src/plugins/defaults/memory/__tests__/relocated-memory-test-rows.ts +10 -0
- package/src/plugins/defaults/memory/__tests__/worker-control.test.ts +19 -6
- package/src/plugins/defaults/memory/buffer-file.ts +354 -0
- package/src/plugins/defaults/memory/buffer-format.ts +40 -0
- package/src/plugins/defaults/memory/context-search/agent-protocol.ts +9 -12
- package/src/plugins/defaults/memory/context-search/agent-runner.ts +77 -22
- package/src/plugins/defaults/memory/context-search/format.ts +53 -8
- package/src/plugins/defaults/memory/context-search/limits.ts +10 -12
- package/src/plugins/defaults/memory/context-search/recall-input.ts +38 -0
- package/src/plugins/defaults/memory/context-search/search.ts +8 -14
- package/src/plugins/defaults/memory/context-search/sources/conversations.ts +1 -0
- package/src/plugins/defaults/memory/context-search/sources/memory-v2.ts +2 -1
- package/src/plugins/defaults/memory/context-search/sources/workspace.ts +2 -1
- package/src/plugins/defaults/memory/context-search/types.ts +1 -11
- package/src/plugins/defaults/memory/conversation-memory-purge.ts +4 -0
- package/src/plugins/defaults/memory/graph/__tests__/handle-remember-v2.test.ts +11 -4
- package/src/plugins/defaults/memory/graph/capability-seed.ts +1 -2
- package/src/plugins/defaults/memory/graph/tool-handlers.ts +13 -63
- package/src/plugins/defaults/memory/graph/tools.ts +35 -64
- package/src/plugins/defaults/memory/host-utils.ts +0 -10
- package/src/plugins/defaults/memory/injectors.ts +4 -3
- package/src/plugins/defaults/memory/memory-retrospective-job.ts +72 -183
- package/src/plugins/defaults/memory/memory-run-evidence.ts +213 -0
- package/src/plugins/defaults/memory/src/memory-eval-routes.ts +2 -3
- package/src/plugins/defaults/memory/src/memory-item-routes.test.ts +6 -1
- package/src/plugins/defaults/memory/src/memory-item-routes.ts +2 -1
- package/src/plugins/defaults/memory/substrate/__tests__/consolidation-job.test.ts +407 -99
- package/src/plugins/defaults/memory/substrate/__tests__/consolidation-prompt-flag-gating-guard.test.ts +10 -0
- package/src/plugins/defaults/memory/substrate/__tests__/prompts-consolidation.test.ts +107 -7
- package/src/plugins/defaults/memory/substrate/consolidation-job.ts +307 -86
- package/src/plugins/defaults/memory/substrate/consolidation-tool-surface.ts +34 -0
- package/src/plugins/defaults/memory/substrate/page-index.ts +2 -1
- package/src/plugins/defaults/memory/substrate/prompts/consolidation.ts +91 -51
- package/src/plugins/defaults/memory/substrate/sweep-job.ts +1 -1
- package/src/plugins/defaults/memory/tools.test.ts +124 -9
- package/src/plugins/defaults/memory/tools.ts +38 -11
- package/src/plugins/defaults/memory/v1/graph/consolidation.ts +2 -2
- package/src/plugins/defaults/memory/v1/graph/extraction.ts +2 -1
- package/src/plugins/defaults/memory/v1/graph/retriever.ts +1 -1
- package/src/plugins/defaults/memory/v2/__tests__/migration.test.ts +5 -0
- package/src/plugins/defaults/memory/v2/__tests__/reranker.test.ts +5 -2
- package/src/plugins/defaults/memory/v2/reranker.ts +2 -1
- package/src/plugins/defaults/memory/v3/__tests__/injection.test.ts +81 -1
- package/src/plugins/defaults/memory/v3/__tests__/orchestrate.test.ts +87 -0
- package/src/plugins/defaults/memory/v3/__tests__/plugin-schema.test.ts +12 -0
- package/src/plugins/defaults/memory/v3/__tests__/pool-log-store.test.ts +332 -15
- package/src/plugins/defaults/memory/v3/__tests__/pool-select.test.ts +626 -27
- package/src/plugins/defaults/memory/v3/__tests__/shadow-plugin.test.ts +148 -28
- package/src/plugins/defaults/memory/v3/card.ts +2 -1
- package/src/plugins/defaults/memory/v3/injector.ts +212 -178
- package/src/plugins/defaults/memory/v3/orchestrate.ts +159 -32
- package/src/plugins/defaults/memory/v3/plugin-schema.ts +54 -4
- package/src/plugins/defaults/memory/v3/pool-log-store.ts +275 -37
- package/src/plugins/defaults/memory/v3/pool-select.test.ts +47 -5
- package/src/plugins/defaults/memory/v3/pool-select.ts +710 -59
- package/src/plugins/defaults/memory/v3/sections.ts +2 -1
- package/src/plugins/defaults/memory/v3/shadow-plugin.ts +41 -14
- package/src/plugins/defaults/memory/worker-control.ts +15 -5
- package/src/plugins/defaults/task-progress/__tests__/registration.test.ts +70 -0
- package/src/plugins/defaults/task-progress/__tests__/task-progress-context.test.ts +380 -0
- package/src/plugins/defaults/task-progress/hooks/post-compact.ts +19 -0
- package/src/plugins/defaults/task-progress/hooks/user-prompt-submit.ts +24 -0
- package/src/plugins/defaults/task-progress/package.json +14 -0
- package/src/plugins/defaults/task-progress/src/apply-task-progress-context.ts +93 -0
- package/src/plugins/defaults/task-progress/src/format-task-progress-context.ts +67 -0
- package/src/plugins/defaults/tool-error/hooks/post-tool-use.ts +4 -1
- package/src/plugins/defaults/tool-result-truncate/terminal.ts +1 -46
- package/src/prompts/__tests__/parallel-tasks-section.test.ts +25 -0
- package/src/prompts/delegation-gate.ts +57 -0
- package/src/prompts/system-prompt.ts +14 -32
- package/src/providers/__tests__/context-overflow-error.test.ts +10 -0
- package/src/providers/__tests__/search-provider-catalog.test.ts +18 -0
- package/src/providers/anthropic/client.ts +10 -1
- package/src/providers/anthropic-gateway-shared.ts +6 -0
- package/src/providers/fetch-provider-catalog.ts +11 -0
- package/src/providers/gemini/client.ts +25 -8
- package/src/providers/inference/adapter-factory.ts +6 -0
- package/src/providers/jev/client.test.ts +270 -0
- package/src/providers/jev/client.ts +534 -0
- package/src/providers/model-catalog.ts +59 -3
- package/src/providers/openai/__tests__/chat-completions-extra-body.test.ts +94 -1
- package/src/providers/openai/__tests__/chat-completions-provider-reasoning.test.ts +106 -1
- package/src/providers/openai/__tests__/chat-template-flatten.test.ts +24 -0
- package/src/providers/openai/__tests__/google-thought-signature.test.ts +1 -2
- package/src/providers/openai/chat-completions-provider.ts +100 -29
- package/src/providers/openai/google-thought-signature.ts +0 -6
- package/src/providers/openai/responses-provider.ts +23 -1
- package/src/providers/ratelimit.ts +1 -3
- package/src/providers/search-provider-catalog.ts +24 -0
- package/src/providers/server-tool-pairing.ts +16 -7
- package/src/providers/speech-to-text/__tests__/deepgram-flux-frames.test.ts +24 -0
- package/src/providers/speech-to-text/__tests__/deepgram-flux-realtime.test.ts +7 -1
- package/src/providers/speech-to-text/__tests__/vellum-managed-realtime.test.ts +50 -1
- package/src/providers/speech-to-text/deepgram-flux-frames.ts +11 -0
- package/src/providers/speech-to-text/vellum-managed-flux-realtime.ts +13 -0
- package/src/providers/types.ts +5 -0
- package/src/routes/control.ts +7 -5
- package/src/runtime/AGENTS.md +6 -4
- package/src/runtime/__tests__/agent-wake.test.ts +94 -1
- package/src/runtime/actor-trust-resolver.ts +3 -3
- package/src/runtime/agent-wake.ts +32 -5
- package/src/runtime/approval-message-composer.ts +3 -69
- package/src/runtime/assistant-event-hub.ts +73 -75
- package/src/runtime/assistant-event-publish-options.ts +7 -0
- package/src/runtime/assistant-event-targeting.ts +79 -0
- package/src/runtime/assistant-stream-state.ts +13 -100
- package/src/runtime/background-job-runner.ts +5 -5
- package/src/runtime/channel-retry-sweep.ts +1 -0
- package/src/runtime/guardian-action-message-composer.ts +8 -161
- package/src/runtime/guardian-action-service.ts +2 -17
- package/src/runtime/guardian-reply-router.ts +29 -40
- package/src/runtime/http-server.ts +2 -2
- package/src/runtime/http-types.ts +6 -0
- package/src/runtime/message-composer-types.ts +3 -61
- package/src/runtime/migrations/__tests__/vbundle-import-parity.test.ts +67 -1
- package/src/runtime/migrations/__tests__/vbundle-import-policy.test.ts +89 -0
- package/src/runtime/migrations/vbundle-builder.ts +24 -1
- package/src/runtime/migrations/vbundle-import-analyzer.ts +13 -4
- package/src/runtime/migrations/vbundle-import-policy.ts +43 -11
- package/src/runtime/migrations/vbundle-importer.ts +4 -1
- package/src/runtime/migrations/vbundle-streaming-importer.ts +4 -1
- package/src/runtime/migrations/vbundle-validator.ts +1 -0
- package/src/runtime/pending-interactions-mode-session.test.ts +62 -0
- package/src/runtime/pending-interactions.ts +29 -0
- package/src/runtime/routes/__tests__/client-history-routes.test.ts +104 -0
- package/src/runtime/routes/__tests__/image-generation-routes.test.ts +97 -0
- package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +24 -2
- package/src/runtime/routes/__tests__/roadmap-routes.test.ts +29 -6
- package/src/runtime/routes/browser-context.ts +31 -11
- package/src/runtime/routes/browser-routes.ts +43 -4
- package/src/runtime/routes/btw-routes.ts +9 -4
- package/src/runtime/routes/channel-route-shared.ts +1 -9
- package/src/runtime/routes/client-routes.ts +108 -0
- package/src/runtime/routes/conversation-management-routes.ts +4 -1
- package/src/runtime/routes/conversation-routes.ts +125 -14
- package/src/runtime/routes/credential-prompt-routes.ts +4 -3
- package/src/runtime/routes/credential-routes.ts +1 -2
- package/src/runtime/routes/desktop-setup-routes.test.ts +9 -18
- package/src/runtime/routes/desktop-setup-routes.ts +8 -7
- package/src/runtime/routes/documents-routes.ts +51 -0
- package/src/runtime/routes/email-routes.ts +3 -7
- package/src/runtime/routes/events-routes.ts +75 -54
- package/src/runtime/routes/guardian-approval-interception.ts +24 -0
- package/src/runtime/routes/identity-routes.ts +1 -5
- package/src/runtime/routes/image-generation-routes.ts +8 -14
- package/src/runtime/routes/inbound-message-handler.ts +37 -41
- package/src/runtime/routes/inbound-stages/backfill-sender-provenance.test.ts +168 -0
- package/src/runtime/routes/inbound-stages/backfill-sender-provenance.ts +96 -0
- package/src/runtime/routes/inbound-stages/background-dispatch.test.ts +39 -10
- package/src/runtime/routes/inbound-stages/background-dispatch.ts +9 -9
- package/src/runtime/routes/inbound-stages/reaction-intercept.ts +2 -0
- package/src/runtime/routes/inference-profile-session-handler.ts +11 -1
- package/src/runtime/routes/inference-profiles-routes.ts +9 -0
- package/src/runtime/routes/llm-context-normalization.ts +1 -7
- package/src/runtime/routes/migration-routes.ts +73 -2
- package/src/runtime/routes/notification-routes.ts +10 -0
- package/src/runtime/routes/oauth-commands-routes.ts +57 -54
- package/src/runtime/routes/oauth-providers.ts +11 -0
- package/src/runtime/routes/oauth-request-hints.test.ts +246 -0
- package/src/runtime/routes/oauth-request-hints.ts +149 -0
- package/src/runtime/routes/plugins-routes.ts +6 -0
- package/src/runtime/routes/roadmap-routes.ts +47 -8
- package/src/runtime/routes/schedule-routes.ts +2 -0
- package/src/runtime/routes/secret-routes.ts +99 -119
- package/src/runtime/routes/settings-routes.ts +10 -4
- package/src/runtime/routes/watcher-routes.ts +1 -3
- package/src/runtime/trust-class.test.ts +7 -6
- package/src/runtime/trust-class.ts +9 -17
- package/src/schedule/__tests__/worker-control.test.ts +36 -5
- package/src/schedule/__tests__/worker-mcp-bootstrap.test.ts +107 -0
- package/src/schedule/__tests__/worker-mcp-readiness.test.ts +173 -0
- package/src/schedule/__tests__/worker-mcp-tools.test.ts +13 -47
- package/src/schedule/run-script.ts +2 -2
- package/src/schedule/scheduler.ts +47 -11
- package/src/schedule/tool-surface-readiness.ts +65 -0
- package/src/schedule/worker-control.ts +15 -5
- package/src/schedule/worker-mcp.ts +85 -0
- package/src/schedule/worker.ts +9 -31
- package/src/security/secure-keys.ts +74 -112
- package/src/skills/clawhub.ts +3 -8
- package/src/skills/managed-store.ts +1 -5
- package/src/stt/types.ts +4 -0
- package/src/subagent/notify.ts +54 -34
- package/src/subagent/parent-notification.ts +6 -0
- package/src/telemetry/__tests__/phone-call-funnel.test.ts +72 -0
- package/src/telemetry/phone-call-funnel.ts +144 -0
- package/src/tools/__tests__/tool-schema-root-combinator-guard.test.ts +81 -0
- package/src/tools/ask-question/ask-question-tool.test.ts +167 -5
- package/src/tools/ask-question/ask-question-tool.ts +120 -22
- package/src/tools/browser/browser-execution.ts +29 -60
- package/src/tools/browser/cdp-client/__tests__/cdp-dom-helpers.test.ts +561 -39
- package/src/tools/browser/cdp-client/cdp-dom-helpers.ts +435 -39
- package/src/tools/browser/cdp-client/cdp-inspect-client.ts +1 -8
- package/src/tools/client-os.ts +15 -0
- package/src/tools/computer-use/skill-proxy-bridge.ts +1 -1
- package/src/tools/computer-use/target.ts +42 -0
- package/src/tools/document/document-tool.ts +13 -18
- package/src/tools/execution-target.ts +14 -16
- package/src/tools/host-terminal/host-shell.ts +16 -7
- package/src/tools/network/__tests__/web-fetch-tinyfish.test.ts +250 -0
- package/src/tools/network/__tests__/web-search.test.ts +203 -0
- package/src/tools/network/web-fetch.ts +274 -7
- package/src/tools/network/web-search.ts +468 -6
- package/src/tools/permission-checker.ts +3 -3
- package/src/tools/policy-context.ts +3 -1
- package/src/tools/shared/filesystem/file-ops-service.ts +1 -31
- package/src/tools/shared/shell-output.test.ts +10 -0
- package/src/tools/shared/shell-output.ts +14 -2
- package/src/tools/skills/load.ts +2 -1
- package/src/tools/skills/sandbox-runner.ts +24 -4
- package/src/tools/skills/scaffold-managed.ts +2 -1
- package/src/tools/skills/skill-script-runner.ts +7 -0
- package/src/tools/skills/skill-tool-factory.ts +15 -1
- package/src/tools/terminal/__tests__/safe-env.test.ts +44 -4
- package/src/tools/terminal/__tests__/sanitized-bash.test.ts +0 -14
- package/src/tools/terminal/safe-env.ts +39 -24
- package/src/tools/terminal/sanitized-bash.ts +15 -2
- package/src/tools/terminal/shell-launch.test.ts +162 -0
- package/src/tools/terminal/shell.test.ts +29 -0
- package/src/tools/terminal/shell.ts +15 -7
- package/src/tools/tool-approval-handler.ts +5 -1
- package/src/tools/tool-types.ts +3 -3
- package/src/tools/types.ts +8 -2
- package/src/util/__tests__/worker-process-command.test.ts +5 -4
- package/src/util/abort-reasons.ts +18 -26
- package/src/util/browser-human-verification.ts +14 -0
- package/src/util/canonicalize-identity.test.ts +21 -0
- package/src/util/canonicalize-identity.ts +16 -0
- package/src/util/errors.ts +13 -3
- package/src/util/host-process.test.ts +17 -1
- package/src/util/host-process.ts +24 -0
- package/src/util/provider-error-patterns.ts +3 -0
- package/src/util/unicode.ts +29 -0
- package/src/util/worker-process.ts +77 -35
- package/src/workspace/migrations/157-add-desktop-files-launcher.ts +94 -0
- package/src/workspace/migrations/__tests__/156-extract-workspace-mcp-json.test.ts +4 -10
- package/src/workspace/migrations/__tests__/157-add-desktop-files-launcher.test.ts +113 -0
- package/src/workspace/migrations/registry.ts +2 -0
- package/src/__tests__/computer-use-window-schema.test.ts +0 -24
- package/src/api/events/heartbeat-alert.ts +0 -20
- package/src/daemon/message-types/web-activity.ts +0 -65
- package/src/desktop/desktop-dependencies.test.ts +0 -131
- package/src/tools/computer-use/definitions.ts +0 -607
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { Buffer } from "node:buffer";
|
|
2
2
|
import { randomUUID } from "node:crypto";
|
|
3
|
+
import { setTimeout as delay } from "node:timers/promises";
|
|
3
4
|
|
|
4
5
|
import {
|
|
5
6
|
type BargeInGuard,
|
|
@@ -26,17 +27,29 @@ import { sanitizeForTts } from "../calls/tts-text-sanitizer.js";
|
|
|
26
27
|
import {
|
|
27
28
|
createControlMarkerHoldback,
|
|
28
29
|
TASK_STOP_MARKER,
|
|
30
|
+
TASK_UPDATE_SILENT_MARKER,
|
|
29
31
|
} from "../calls/voice-control-protocol.js";
|
|
32
|
+
import type { VoiceEscalationTarget } from "../calls/voice-escalation-target.js";
|
|
30
33
|
import {
|
|
31
34
|
createFrontDoorLegCoordinator,
|
|
32
35
|
type FrontDoorLegCoordinator,
|
|
33
36
|
type SpokenEscalationBridge,
|
|
34
37
|
} from "../calls/voice-leg-coordinator.js";
|
|
38
|
+
import {
|
|
39
|
+
getVoiceMetricsAggregateFields,
|
|
40
|
+
type VoiceEndpointAction,
|
|
41
|
+
type VoiceEndpointSource,
|
|
42
|
+
type VoiceMetricsClock,
|
|
43
|
+
VoiceMetricsCollector,
|
|
44
|
+
type VoiceMetricsEvent,
|
|
45
|
+
type VoiceTurnSeedMarks,
|
|
46
|
+
} from "../calls/voice-metrics.js";
|
|
35
47
|
import {
|
|
36
48
|
createProgressCadence,
|
|
37
49
|
type ProgressCadence,
|
|
38
50
|
} from "../calls/voice-progress-cadence.js";
|
|
39
51
|
import type {
|
|
52
|
+
VoiceTurnCallbacks,
|
|
40
53
|
VoiceTurnHandle,
|
|
41
54
|
VoiceTurnOptions,
|
|
42
55
|
} from "../calls/voice-session-bridge.js";
|
|
@@ -56,6 +69,10 @@ import {
|
|
|
56
69
|
VoiceFrontModelConfigSchema,
|
|
57
70
|
} from "../config/schemas/voice.js";
|
|
58
71
|
import { ABORT_WATCHDOG_MS } from "../daemon/abort-watchdog.js";
|
|
72
|
+
import type {
|
|
73
|
+
ConversationModeSessionCoordinator,
|
|
74
|
+
ModeSessionSourceHandle,
|
|
75
|
+
} from "../daemon/conversation-mode-session.js";
|
|
59
76
|
import { isRefusedInReadOnlyPass } from "../daemon/conversation-tool-setup.js";
|
|
60
77
|
import type { TrustContext } from "../daemon/trust-context-types.js";
|
|
61
78
|
import {
|
|
@@ -95,6 +112,7 @@ import type {
|
|
|
95
112
|
SttStreamServerEvent,
|
|
96
113
|
} from "../stt/types.js";
|
|
97
114
|
import { getSubagentManager } from "../subagent/index.js";
|
|
115
|
+
import type { SubagentParentNotification } from "../subagent/parent-notification.js";
|
|
98
116
|
import {
|
|
99
117
|
liveVoiceEndScreen,
|
|
100
118
|
liveVoiceSilenceReason,
|
|
@@ -116,6 +134,12 @@ import {
|
|
|
116
134
|
dismissesUiSurface,
|
|
117
135
|
revealsUiSurface,
|
|
118
136
|
} from "./activity-label.js";
|
|
137
|
+
import { CameraModeSessionProducer } from "./camera-mode-session.js";
|
|
138
|
+
import {
|
|
139
|
+
type ContinuationJudgement,
|
|
140
|
+
createContinuationJudge,
|
|
141
|
+
type LiveVoiceContinuationJudge,
|
|
142
|
+
} from "./continuation-judge.js";
|
|
119
143
|
import {
|
|
120
144
|
buildDuplexContinuationLabel,
|
|
121
145
|
createContinuationLabeler,
|
|
@@ -128,15 +152,7 @@ import type {
|
|
|
128
152
|
} from "./live-voice-archive.js";
|
|
129
153
|
import type { LiveVoiceCredentialReadiness } from "./live-voice-credential-preflight.js";
|
|
130
154
|
import {
|
|
131
|
-
|
|
132
|
-
type LiveVoiceMetricsClock,
|
|
133
|
-
LiveVoiceMetricsCollector,
|
|
134
|
-
type LiveVoiceMetricsEvent,
|
|
135
|
-
type LiveVoiceTurnSeedMarks,
|
|
136
|
-
type VoiceEndpointAction,
|
|
137
|
-
type VoiceEndpointSource,
|
|
138
|
-
} from "./live-voice-metrics.js";
|
|
139
|
-
import {
|
|
155
|
+
type LiveVoicePhotoResult,
|
|
140
156
|
persistAmbientSightFrame,
|
|
141
157
|
persistLiveVoicePhoto,
|
|
142
158
|
} from "./live-voice-photo.js";
|
|
@@ -151,6 +167,7 @@ import type {
|
|
|
151
167
|
LiveVoiceTtsOptions,
|
|
152
168
|
LiveVoiceTtsResult,
|
|
153
169
|
} from "./live-voice-tts.js";
|
|
170
|
+
import { MicrophoneIdleGate } from "./microphone-idle-gate.js";
|
|
154
171
|
import {
|
|
155
172
|
type LiveVoiceClientAttachFrameFrame,
|
|
156
173
|
type LiveVoiceClientAttachImageFrame,
|
|
@@ -167,12 +184,15 @@ import {
|
|
|
167
184
|
isLookSessionControl,
|
|
168
185
|
LOOK_FOLLOW_UP_CONTENT,
|
|
169
186
|
LOOK_FRAME_REASON,
|
|
187
|
+
type LookFollowUp,
|
|
170
188
|
lookFollowUpNote,
|
|
171
189
|
type LookSessionControl,
|
|
172
190
|
progressConfigForCadence,
|
|
173
191
|
requestedSessionControl,
|
|
174
192
|
sessionControlTeaching,
|
|
175
193
|
} from "./session-controls.js";
|
|
194
|
+
import { type VoiceTaskOutcome, VoiceTaskOutcomes } from "./task-outcomes.js";
|
|
195
|
+
import { VoiceInputDiagnostics } from "./voice-input-diagnostics.js";
|
|
176
196
|
|
|
177
197
|
const log = getLogger("live-voice-session");
|
|
178
198
|
|
|
@@ -263,25 +283,22 @@ const PROVIDER_TURN_END_FALLBACK_MARGIN_MS = 1_000;
|
|
|
263
283
|
// a segment is at most ~180 chars of speech (~10 s of 24 kHz mono PCM
|
|
264
284
|
// ≈ 480 KB), so one buffered segment is an acceptable bound.
|
|
265
285
|
const TTS_MAX_OPEN_SYNTHESIS_JOBS = 2;
|
|
286
|
+
const TTS_MAX_PLAYBACK_LEAD_MS = 500;
|
|
266
287
|
// Audible silence required before a finished background continuation's result
|
|
267
288
|
// is spoken into a live call. Long enough that the announcement lands in a real
|
|
268
289
|
// lull rather than on the heels of the turn that just ended; short enough that
|
|
269
290
|
// the user is not left wondering whether the work survived.
|
|
270
291
|
const CONTINUATION_ANNOUNCE_SILENCE_MS = 1_500;
|
|
271
|
-
// How
|
|
272
|
-
//
|
|
273
|
-
//
|
|
274
|
-
|
|
275
|
-
// falls back to the stash rather than chasing a call that keeps talking.
|
|
276
|
-
const CONTINUATION_ANNOUNCE_MAX_DRAIN_REARMS = 3;
|
|
292
|
+
// How long a barge-in's continuation waits for the interrupting words before
|
|
293
|
+
// the continuation judge gives up on them and the continuation proceeds.
|
|
294
|
+
// Covers the caller finishing the sentence and the utterance releasing.
|
|
295
|
+
const BARGE_IN_INTERRUPTION_WAIT_MS = 4_000;
|
|
277
296
|
const FOREGROUND_TASK_RESUME_SILENCE_MS = 1_500;
|
|
278
297
|
const FOREGROUND_TASK_MAX_SUSPENDED_MS = 120_000;
|
|
279
298
|
const FOREGROUND_TASK_MAX_INTERVENING_TURNS = 3;
|
|
280
|
-
//
|
|
281
|
-
// the model-facing control prompt (buildLiveDeliveryNote), so the only thing
|
|
282
|
-
// persisted on the user side is this marker, and it persists hidden.
|
|
299
|
+
// Hidden content for continuation announcements; findings travel in the control prompt.
|
|
283
300
|
export const CONTINUATION_DELIVERY_CONTENT =
|
|
284
|
-
"(background work finished
|
|
301
|
+
"(background work finished: deliver it now)";
|
|
285
302
|
// How long a look control waits for the fresh frame the client takes for it.
|
|
286
303
|
// Long enough for a camera that has to open and warm up before its first keep;
|
|
287
304
|
// past it the look is dropped rather than answered into a later, unrelated
|
|
@@ -364,10 +381,14 @@ export interface LiveVoiceSessionOptions {
|
|
|
364
381
|
*/
|
|
365
382
|
resolveCredentialReadiness?: LiveVoiceCredentialReadinessResolver | null;
|
|
366
383
|
startVoiceTurn?: LiveVoiceTurnStarter;
|
|
384
|
+
acquireModeSessionResidency?: (conversationId: string) => Promise<{
|
|
385
|
+
coordinator: ConversationModeSessionCoordinator;
|
|
386
|
+
release: () => void;
|
|
387
|
+
}>;
|
|
367
388
|
streamTtsAudio?: LiveVoiceTtsStreamer | null;
|
|
368
389
|
archiveAudio?: LiveVoiceSessionAudioArchiver | null;
|
|
369
390
|
emitMetrics?: boolean;
|
|
370
|
-
metricsClock?:
|
|
391
|
+
metricsClock?: VoiceMetricsClock;
|
|
371
392
|
/**
|
|
372
393
|
* Mirrors phase changes to the iOS Live Activity. Injectable so tests can
|
|
373
394
|
* assert what a session reports without reaching the platform.
|
|
@@ -448,6 +469,13 @@ export interface LiveVoiceSessionOptions {
|
|
|
448
469
|
* Absent, the continuation carries the deterministic transcript label.
|
|
449
470
|
*/
|
|
450
471
|
labelBackgroundContinuation?: LiveVoiceContinuationLabeler;
|
|
472
|
+
/**
|
|
473
|
+
* Decides from the caller's interruption whether the interrupted request
|
|
474
|
+
* should still continue in the background. The factory wires the
|
|
475
|
+
* TypeSafe-backed judge; tests inject a stub. Absent, every eligible
|
|
476
|
+
* barge-in continues, as it does without a TypeSafe profile.
|
|
477
|
+
*/
|
|
478
|
+
judgeBackgroundContinuation?: LiveVoiceContinuationJudge;
|
|
451
479
|
/**
|
|
452
480
|
* Returns the pending teardown promise for a conversation's most recent
|
|
453
481
|
* turn. The barge-in path awaits it before forking the background
|
|
@@ -493,6 +521,7 @@ type LiveVoiceUtterancePhase =
|
|
|
493
521
|
// manual mode and a session-shared instance in persistent server-VAD mode.
|
|
494
522
|
interface UtteranceCycle {
|
|
495
523
|
phase: LiveVoiceUtterancePhase;
|
|
524
|
+
preDialBargeInGuard: BargeInGuard | null;
|
|
496
525
|
released: boolean;
|
|
497
526
|
assistantTurnStarted: boolean;
|
|
498
527
|
// The whole cycle (turn included) finalized; the record can no longer
|
|
@@ -510,12 +539,8 @@ interface UtteranceCycle {
|
|
|
510
539
|
// server_vad has the turn detector for the same question, and never sets
|
|
511
540
|
// this — its ingress is handleServerVadAudio.
|
|
512
541
|
manualAudioCaptured: boolean;
|
|
513
|
-
//
|
|
514
|
-
//
|
|
515
|
-
// silence from one already carrying the user's utterance: the
|
|
516
|
-
// stale-language interception in handleServerVadAudio may retire the
|
|
517
|
-
// former, never the latter. turnId cannot answer this, because a
|
|
518
|
-
// silence-only pre-roll flush assigns it too.
|
|
542
|
+
// Local VAD or the provider detected speech. A language change may retire
|
|
543
|
+
// only a silence-only cycle; pre-roll can assign turnId before speech.
|
|
519
544
|
speechRouted: boolean;
|
|
520
545
|
pendingAudioChunks: Buffer[];
|
|
521
546
|
pendingAudioBytes: number;
|
|
@@ -601,6 +626,8 @@ type UtteranceStartResult =
|
|
|
601
626
|
// client in job-list order.
|
|
602
627
|
interface TtsSegmentJob {
|
|
603
628
|
readonly text: string;
|
|
629
|
+
readonly isReply: boolean;
|
|
630
|
+
audioSent: boolean;
|
|
604
631
|
// Per-segment language-hint override, preferred over the turn's language.
|
|
605
632
|
// Set on fixed phrases whose localized table lacks the turn's language:
|
|
606
633
|
// the English fallback text carries "en" so an enforcing provider never
|
|
@@ -635,6 +662,9 @@ interface ActiveAssistantTurn {
|
|
|
635
662
|
language: string | undefined;
|
|
636
663
|
abortController: AbortController;
|
|
637
664
|
handle: VoiceTurnHandle | null;
|
|
665
|
+
modeSessionSource: ModeSessionSourceHandle | undefined;
|
|
666
|
+
modeSessionRequestIds: string[];
|
|
667
|
+
modeSessionDeliveryTurnId: string | null;
|
|
638
668
|
// When the turn launched, for narration's turnElapsedMs.
|
|
639
669
|
launchedAtMs: number;
|
|
640
670
|
// Tool-activity log and spoken progress narration for the turn
|
|
@@ -659,6 +689,10 @@ interface ActiveAssistantTurn {
|
|
|
659
689
|
// whole frame rather than its wording. What the CLIENT believes, as against
|
|
660
690
|
// `pendingApproval`, which is what is true.
|
|
661
691
|
publishedApprovalRequestId: string | null;
|
|
692
|
+
// Structured activity the client most recently received. TTS can
|
|
693
|
+
// temporarily move system surfaces to speaking without changing this
|
|
694
|
+
// logical activity, while a tool or approval frame clears it.
|
|
695
|
+
publishedActivityKind: "escalation" | null;
|
|
662
696
|
// Set while the turn is blocked on a decision the user has to make, and null
|
|
663
697
|
// when it is not. Suppresses progress narration, whose entire vocabulary
|
|
664
698
|
// ("still on it", "almost there") describes work in flight and would be
|
|
@@ -681,6 +715,7 @@ interface ActiveAssistantTurn {
|
|
|
681
715
|
// A tts_audio frame actually went out to the client — latches on the first
|
|
682
716
|
// forwarded chunk so the firstTtsAudio metric is marked exactly once per turn.
|
|
683
717
|
ttsAudioStarted: boolean;
|
|
718
|
+
ttsFailed: boolean;
|
|
684
719
|
finalized: boolean;
|
|
685
720
|
// Unified front-door speculative dispatch: the leg is in flight but its
|
|
686
721
|
// leading verdict (hold vs commit) has not arrived. The thinking
|
|
@@ -718,19 +753,12 @@ interface ActiveAssistantTurn {
|
|
|
718
753
|
// merges it with this turn's utterance instead of treating that utterance as
|
|
719
754
|
// a fresh follow-up. Null for an ordinary (non-barge-in) turn.
|
|
720
755
|
interruptedRequest: string | null;
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
//
|
|
726
|
-
|
|
727
|
-
// no user utterance behind it — `content` is CONTINUATION_DELIVERY_CONTENT and
|
|
728
|
-
// the answer rides the control prompt (buildLiveDeliveryNote).
|
|
729
|
-
continuationDelivery: ContinuationDelivery | null;
|
|
730
|
-
// Set only on the turn that answers a look: which look it answers. The turn
|
|
731
|
-
// has no user utterance behind it; the instruction rides the control prompt
|
|
732
|
-
// (lookFollowUpNote).
|
|
733
|
-
lookFollowUp: LookSessionControl | null;
|
|
756
|
+
contextOutcomes: VoiceTaskOutcome[];
|
|
757
|
+
taskOutcome: VoiceTaskOutcome | null;
|
|
758
|
+
taskDeliveryContext: string | null;
|
|
759
|
+
notificationHandledSilently: boolean;
|
|
760
|
+
// The fresh view and original caller request this hidden turn resumes.
|
|
761
|
+
lookFollowUp: LookFollowUp | null;
|
|
734
762
|
// The turn's content is an internal instruction rather than user speech (the
|
|
735
763
|
// greeting that opens a session, say). The row still persists and the model
|
|
736
764
|
// still sees it; `hiddenSyntheticPrompt` keeps it out of the transcript.
|
|
@@ -754,15 +782,8 @@ interface ActiveAssistantTurn {
|
|
|
754
782
|
// slot while this turn was in flight. Nothing will ever send it, so it is
|
|
755
783
|
// collected once the discard has removed the row still linking it.
|
|
756
784
|
staleFrameId: string | null;
|
|
757
|
-
// The queued announcement this turn consumed alongside `continuationResult`
|
|
758
|
-
// — the two are routes for one answer, so launching consumes both. Held here
|
|
759
|
-
// so a rolled-back speculative dispatch can re-arm it (see
|
|
760
|
-
// restorePendingTurnContext). Null on the announcement turn itself, which
|
|
761
|
-
// owns its delivery outright.
|
|
762
|
-
consumedAnnouncement: ContinuationDelivery | null;
|
|
763
785
|
// `detachStopGeneration` at the moment this turn consumed the pending
|
|
764
|
-
// context. A
|
|
765
|
-
// dropped the continuation, so a rollback must not resurrect it.
|
|
786
|
+
// context. A stop invalidates that context so a rollback cannot restore it.
|
|
766
787
|
pendingContextStopGeneration: number;
|
|
767
788
|
// Counts assistant text deltas seen this turn. A narration generation
|
|
768
789
|
// captures it at launch and discards its result if it moved: text the model
|
|
@@ -777,6 +798,9 @@ interface ActiveAssistantTurn {
|
|
|
777
798
|
// The front-door leg's coordinator: whether it handed the turn off to the
|
|
778
799
|
// escalated leg. Null until the front-door leg starts.
|
|
779
800
|
frontDoor: FrontDoorLegCoordinator | null;
|
|
801
|
+
// Final post-hook inference target for an escalated leg. Retained so the
|
|
802
|
+
// neutral waiting phase can be restored after bridge audio drains.
|
|
803
|
+
escalationTarget: VoiceEscalationTarget | null;
|
|
780
804
|
ttsBuffer: string;
|
|
781
805
|
// What the caller actually hears this turn, summed over the model's own
|
|
782
806
|
// segments (acks and progress narration do not count). Logged at tts_done
|
|
@@ -802,6 +826,19 @@ interface ActiveAssistantTurn {
|
|
|
802
826
|
assistantAudioSampleRate?: number;
|
|
803
827
|
}
|
|
804
828
|
|
|
829
|
+
function sameModeSessionSource(
|
|
830
|
+
a: ModeSessionSourceHandle | undefined,
|
|
831
|
+
b: ModeSessionSourceHandle,
|
|
832
|
+
): boolean {
|
|
833
|
+
return (
|
|
834
|
+
a?.id === b.id &&
|
|
835
|
+
a.mode === b.mode &&
|
|
836
|
+
a.sourceId === b.sourceId &&
|
|
837
|
+
a.generation === b.generation &&
|
|
838
|
+
a.activation === b.activation
|
|
839
|
+
);
|
|
840
|
+
}
|
|
841
|
+
|
|
805
842
|
// Base control prompt for every live-voice turn. Opens with the spoken-reply
|
|
806
843
|
// rules shared with the phone path (spoken-reply-rules.ts): this is the only
|
|
807
844
|
// place the model learns that its text is spoken, since the system prompt has
|
|
@@ -849,14 +886,10 @@ function buildInterruptionMergeNote(interruptedRequest: string): string {
|
|
|
849
886
|
return `The user interrupted your previous, unfinished reply. Their earlier request was: "${interruptedRequest}". Treat their current message as a continuation of that request and address both together. If they corrected or redirected the unfinished work, continue executing it with that correction instead of merely acknowledging it. If they clearly want the unfinished task abandoned, use the task-stop control.`;
|
|
850
887
|
}
|
|
851
888
|
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
// result is offered only if it fits. Reaches the model only; it is not a user
|
|
857
|
-
// message and never renders as a transcript bubble.
|
|
858
|
-
function buildResurfaceContextNote(continuationResult: string): string {
|
|
859
|
-
return `Earlier the user interrupted you, and in the background you finished the reply they cut off. What you worked out was: "${continuationResult}". If their current message relates to it, use it to answer; otherwise you may briefly offer it or leave it aside, and do not repeat it verbatim if it no longer fits.`;
|
|
889
|
+
function buildResurfaceContextNote(
|
|
890
|
+
outcomes: readonly VoiceTaskOutcome[],
|
|
891
|
+
): string {
|
|
892
|
+
return `Answer the user's current message. These pending background reports are context, not new requests. Use their findings if relevant; otherwise leave them for a separate announcement. Reading them does not acknowledge delivery. Pending results:\n${outcomes.map((outcome) => outcome.message).join("\n\n")}`;
|
|
860
893
|
}
|
|
861
894
|
|
|
862
895
|
// Appended to the turn that follows a barge-in whose interrupted work was
|
|
@@ -879,19 +912,11 @@ function describeInterruptedRequest(request: string): string {
|
|
|
879
912
|
: "their earlier request";
|
|
880
913
|
}
|
|
881
914
|
|
|
882
|
-
// A
|
|
883
|
-
|
|
884
|
-
interface PendingLook {
|
|
885
|
-
action: LookSessionControl;
|
|
915
|
+
// A bounded wait to resume the caller request from a fresh view.
|
|
916
|
+
interface PendingLook extends LookFollowUp {
|
|
886
917
|
armedAtMs: number;
|
|
887
918
|
timer: ReturnType<typeof setTimeout>;
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
// A finished background continuation waiting to be delivered: the request it
|
|
891
|
-
// took over and the answer it produced.
|
|
892
|
-
interface ContinuationDelivery {
|
|
893
|
-
request: string;
|
|
894
|
-
answer: string;
|
|
919
|
+
frameLanded: boolean;
|
|
895
920
|
}
|
|
896
921
|
|
|
897
922
|
interface OwnedForegroundTask {
|
|
@@ -914,17 +939,6 @@ interface SuspendedForegroundTask {
|
|
|
914
939
|
|
|
915
940
|
type ForegroundTaskState = OwnedForegroundTask | SuspendedForegroundTask;
|
|
916
941
|
|
|
917
|
-
// Appended to an announcement turn's control prompt: the turn the session
|
|
918
|
-
// starts on its own when a background continuation finishes while the call is
|
|
919
|
-
// live and nobody is speaking. There is no user utterance behind it, so this
|
|
920
|
-
// note is the whole instruction and the answer rides here rather than in the
|
|
921
|
-
// turn's `content`. Reaches the model only; it is not a user message and never
|
|
922
|
-
// renders as a transcript bubble.
|
|
923
|
-
function buildLiveDeliveryNote(request: string, answer: string): string {
|
|
924
|
-
const what = describeInterruptedRequest(request);
|
|
925
|
-
return `The user interrupted you earlier, and in the background you finished ${what}. The call is still live and nobody is speaking, so tell them briefly that it is done and give them the result. Do not re-run any tool calls; the work is already complete, and do not repeat it verbatim if it no longer fits. What you produced was:\n\n${answer}`;
|
|
926
|
-
}
|
|
927
|
-
|
|
928
942
|
function buildForegroundTaskResumePrompt(request: string): string {
|
|
929
943
|
const task =
|
|
930
944
|
request.length > 0 ? JSON.stringify(request) : "the unfinished task";
|
|
@@ -934,9 +948,8 @@ function buildForegroundTaskResumePrompt(request: string): string {
|
|
|
934
948
|
// Assemble a leg's model-facing control prompt: the base live-voice rules, the
|
|
935
949
|
// screen-reveal and setup-flow teaching (both withheld from the front-door leg,
|
|
936
950
|
// see LIVE_VOICE_SCREEN_REVEAL_TEACHING), plus any pending barge-in merge
|
|
937
|
-
// context,
|
|
938
|
-
//
|
|
939
|
-
// result waiting); the notes are model-only and never render as user bubbles.
|
|
951
|
+
// context, pending task outcomes, and the announcement instruction. These
|
|
952
|
+
// notes are model-only and never render as user bubbles.
|
|
940
953
|
function buildVoiceControlPrompt(
|
|
941
954
|
turn: ActiveAssistantTurn,
|
|
942
955
|
leg: { frontDoor?: boolean },
|
|
@@ -961,8 +974,8 @@ function buildVoiceControlPrompt(
|
|
|
961
974
|
if (turn.interruptedRequest) {
|
|
962
975
|
prompt = `${prompt}\n\n${buildInterruptionMergeNote(turn.interruptedRequest)}`;
|
|
963
976
|
}
|
|
964
|
-
if (turn.
|
|
965
|
-
prompt = `${prompt}\n\n${buildResurfaceContextNote(turn.
|
|
977
|
+
if (turn.contextOutcomes.length > 0) {
|
|
978
|
+
prompt = `${prompt}\n\n${buildResurfaceContextNote(turn.contextOutcomes)}`;
|
|
966
979
|
}
|
|
967
980
|
if (turn.handedOffRequest) {
|
|
968
981
|
prompt = `${prompt}\n\n${buildHandoffAnnouncementNote(turn.handedOffRequest)}`;
|
|
@@ -970,32 +983,24 @@ function buildVoiceControlPrompt(
|
|
|
970
983
|
if (turn.lookFollowUp !== null) {
|
|
971
984
|
prompt = `${prompt}\n\n${lookFollowUpNote(turn.lookFollowUp)}`;
|
|
972
985
|
}
|
|
973
|
-
if (turn.
|
|
974
|
-
prompt
|
|
975
|
-
|
|
976
|
-
turn.
|
|
977
|
-
|
|
986
|
+
if (turn.taskOutcome !== null) {
|
|
987
|
+
prompt += `\n\nThis turn is an internal background task update, not new words from the user. ${turn.taskDeliveryContext} Use the latest conversation context to decide what is useful now. For routine progress, findings covered by completed playback, cancelled work, or findings meant only for internal use, output only ${TASK_UPDATE_SILENT_MARKER}. An interrupted announcement or a displayed card does not establish delivery. Otherwise briefly speak the useful outcome or blocker first, before optional tool calls or a visual summary. Do not just announce completion, read raw worker status, or repeat completed actions.`;
|
|
988
|
+
if (turn.taskOutcome.source === "continuation") {
|
|
989
|
+
prompt += `\n\n${turn.taskOutcome.message}`;
|
|
990
|
+
}
|
|
978
991
|
}
|
|
979
992
|
return prompt;
|
|
980
993
|
}
|
|
981
994
|
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
// normal turn in the thread — where the user actually goes looking for the
|
|
985
|
-
// work. Framed as a system-style report rather than a user request so the
|
|
986
|
-
// reply reads as "here is what I finished", not a fresh instruction.
|
|
987
|
-
function buildClosedSessionDeliveryPrompt(
|
|
988
|
-
interruptedRequest: string,
|
|
989
|
-
answer: string,
|
|
990
|
-
): string {
|
|
991
|
-
const what = describeInterruptedRequest(interruptedRequest);
|
|
992
|
-
return `[Background work finished] The voice call ended before the completed result for ${what} could be delivered. Tell the user briefly that it is done and give them the result. Do not re-run any tool calls; the work is already complete. What you produced was:\n\n${answer}`;
|
|
995
|
+
function buildContinuationResult(request: string, answer: string): string {
|
|
996
|
+
return `[Background work finished] The completed result for ${describeInterruptedRequest(request)} is ready. Give the user the useful findings without re-running the work. Result:\n\n${answer}`;
|
|
993
997
|
}
|
|
994
998
|
|
|
995
999
|
// A fresh cycle: nothing captured, nothing released, no metrics turn open.
|
|
996
1000
|
function createUtteranceCycle(): UtteranceCycle {
|
|
997
1001
|
return {
|
|
998
1002
|
phase: "pending",
|
|
1003
|
+
preDialBargeInGuard: null,
|
|
999
1004
|
released: false,
|
|
1000
1005
|
assistantTurnStarted: false,
|
|
1001
1006
|
completed: false,
|
|
@@ -1160,6 +1165,22 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1160
1165
|
private readonly archiveAudio: LiveVoiceSessionAudioArchiver | null;
|
|
1161
1166
|
private readonly spawnBackgroundContinuation: LiveVoiceBackgroundContinuationSpawner | null;
|
|
1162
1167
|
private readonly labelBackgroundContinuation: LiveVoiceContinuationLabeler | null;
|
|
1168
|
+
private readonly judgeBackgroundContinuation: LiveVoiceContinuationJudge | null;
|
|
1169
|
+
// The latest barge-in's interrupting words, resolved when the turn that
|
|
1170
|
+
// carries them dispatches (or null past BARGE_IN_INTERRUPTION_WAIT_MS). The
|
|
1171
|
+
// continuation judge reads it; a newer barge-in settles the older one null.
|
|
1172
|
+
private bargeInInterruption: {
|
|
1173
|
+
detachSeq: number;
|
|
1174
|
+
promise: Promise<string | null>;
|
|
1175
|
+
settle: (words: string | null) => void;
|
|
1176
|
+
} | null = null;
|
|
1177
|
+
// The continuation judge's pending verdict on a foreground task a barge-in
|
|
1178
|
+
// suspended. The resume waits on it, so a task the caller called off is
|
|
1179
|
+
// cleared instead of resumed.
|
|
1180
|
+
private foregroundTaskJudgement: {
|
|
1181
|
+
epoch: number;
|
|
1182
|
+
settled: Promise<void>;
|
|
1183
|
+
} | null = null;
|
|
1163
1184
|
// Reads the interrupted turn's teardown promise so the barge-in path can wait
|
|
1164
1185
|
// for it to settle before forking the continuation.
|
|
1165
1186
|
private readonly getTurnTeardown:
|
|
@@ -1184,9 +1205,18 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1184
1205
|
// continuation completing (before OR after it) can't surface a stale answer.
|
|
1185
1206
|
private detachSequence = 0;
|
|
1186
1207
|
private readonly emitMetrics: boolean;
|
|
1187
|
-
private readonly metrics:
|
|
1208
|
+
private readonly metrics: VoiceMetricsCollector;
|
|
1188
1209
|
private readonly createTurnId: () => string;
|
|
1189
1210
|
private readonly conversationId: string;
|
|
1211
|
+
private readonly acquireModeSessionResidency?: (
|
|
1212
|
+
conversationId: string,
|
|
1213
|
+
) => Promise<{
|
|
1214
|
+
coordinator: ConversationModeSessionCoordinator;
|
|
1215
|
+
release: () => void;
|
|
1216
|
+
}>;
|
|
1217
|
+
private releaseModeSessionResidency?: () => void;
|
|
1218
|
+
private cameraModeSessions?: CameraModeSessionProducer;
|
|
1219
|
+
private sightFrameSequence: Promise<void> = Promise.resolve();
|
|
1190
1220
|
/**
|
|
1191
1221
|
* Mirrors phase changes to the iOS Live Activity through the platform, for
|
|
1192
1222
|
* the case the client cannot cover: an app backgrounded long enough for iOS
|
|
@@ -1197,6 +1227,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1197
1227
|
private currentUtterance: UtteranceCycle | null = null;
|
|
1198
1228
|
private outboundFrames: Promise<void> = Promise.resolve();
|
|
1199
1229
|
private activeAssistantTurn: ActiveAssistantTurn | null = null;
|
|
1230
|
+
private readonly pendingAssistantStarts = new Set<Promise<void>>();
|
|
1200
1231
|
private sessionEndMetricsEmitted = false;
|
|
1201
1232
|
/**
|
|
1202
1233
|
* Protocol error code of the failure that killed the session, latched by
|
|
@@ -1256,6 +1287,10 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1256
1287
|
// what echoEnergyEma is for). Raises the base gate in a noisy room so the
|
|
1257
1288
|
// room itself stops reading as speech.
|
|
1258
1289
|
private readonly roomNoiseFloor = new RoomNoiseFloor();
|
|
1290
|
+
private readonly inputDiagnostics = new VoiceInputDiagnostics();
|
|
1291
|
+
private readonly microphoneIdleGate: MicrophoneIdleGate;
|
|
1292
|
+
private diagnosticEchoCorrelation: number | null = null;
|
|
1293
|
+
private diagnosticTtsSampleRate: number | null = null;
|
|
1259
1294
|
private readonly echoBargeInMargin: number;
|
|
1260
1295
|
private readonly echoEmaHalfLifeMs: number;
|
|
1261
1296
|
private readonly echoDrainSlackMs: number;
|
|
@@ -1294,6 +1329,8 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1294
1329
|
// The shared sustained-speech accounting (gap tolerance, duty-cycle
|
|
1295
1330
|
// ceiling, threshold).
|
|
1296
1331
|
guard: BargeInGuard;
|
|
1332
|
+
armedAtMs: number;
|
|
1333
|
+
resets: number;
|
|
1297
1334
|
} | null = null;
|
|
1298
1335
|
// Estimated wall-clock ms until the client finishes draining the
|
|
1299
1336
|
// assistant audio sent so far. The server clears the turn right after
|
|
@@ -1308,23 +1345,10 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1308
1345
|
// Consumed (and cleared) when that turn launches; cleared if the barge-in
|
|
1309
1346
|
// utterance is discarded, so it can never attach to a later, unrelated turn.
|
|
1310
1347
|
private pendingInterruptedRequest: string | null = null;
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
// an announcement that delivers the same answer itself — an announcement the
|
|
1316
|
-
// user cuts short hands it straight back.
|
|
1317
|
-
private pendingContinuationResult: string | null = null;
|
|
1318
|
-
// The same finished continuation, queued for an announcement turn: the
|
|
1319
|
-
// session speaks the result on its own once the call has been quiet for
|
|
1320
|
-
// `continuationAnnounceSilenceMs`. This and `pendingContinuationResult` are
|
|
1321
|
-
// armed together and are two routes for ONE answer — whichever fires first
|
|
1322
|
-
// clears the other, so the user hears it exactly once. An announcement whose
|
|
1323
|
-
// turn fails to start, or whose turn a barge-in cuts short, hands the answer
|
|
1324
|
-
// back to the stash, so a lost delivery costs the announcement, not the
|
|
1325
|
-
// answer.
|
|
1326
|
-
private pendingAnnouncement: ContinuationDelivery | null = null;
|
|
1327
|
-
private announcementTimer: ReturnType<typeof setTimeout> | null = null;
|
|
1348
|
+
private readonly taskOutcomes = new VoiceTaskOutcomes();
|
|
1349
|
+
private taskAnnouncementTimer: ReturnType<typeof setTimeout> | null = null;
|
|
1350
|
+
private taskAnnouncementsDeferred = false;
|
|
1351
|
+
private taskOutcomesClosing = false;
|
|
1328
1352
|
// Host-backed work stays on the parent conversation across barge-ins. An
|
|
1329
1353
|
// owned task belongs to the tool-capable turn doing the work; a suspended
|
|
1330
1354
|
// task is waiting for either a merged escalation or a synthetic resume.
|
|
@@ -1338,8 +1362,8 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1338
1362
|
// A look control sent to a client that declared `lookFrames`, waiting for
|
|
1339
1363
|
// the fresh frame the client takes for it. The session answers the look on a
|
|
1340
1364
|
// turn of its own once that frame is in the conversation (see
|
|
1341
|
-
// answerLookWhenFloorIsFree).
|
|
1342
|
-
//
|
|
1365
|
+
// answerLookWhenFloorIsFree). Retained through the floor wait so committed
|
|
1366
|
+
// caller clarifications can join the request until the follow-up launches.
|
|
1343
1367
|
private pendingLook: PendingLook | null = null;
|
|
1344
1368
|
private lookFollowUpTimer: ReturnType<typeof setTimeout> | null = null;
|
|
1345
1369
|
// Set when a continuation actually spawns: the request it took over, so the
|
|
@@ -1370,10 +1394,11 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1370
1394
|
// The ring holds speech parked during the release→turn-start window;
|
|
1371
1395
|
// protected from silent-chunk eviction until it flushes.
|
|
1372
1396
|
private vadPreRollHasSpeech = false;
|
|
1397
|
+
private vadPreRollBargeInGuard: BargeInGuard | null = null;
|
|
1373
1398
|
// Detector turn-end that fired while its speech sat parked in the ring;
|
|
1374
1399
|
// replayed once the parked speech flushes into the next armed utterance.
|
|
1375
1400
|
private vadPendingTurnEnd: "silence" | "max-duration" | null = null;
|
|
1376
|
-
private readonly metricsClock:
|
|
1401
|
+
private readonly metricsClock: VoiceMetricsClock;
|
|
1377
1402
|
// Persistent mode: a server-VAD session keeps one streaming transcriber for
|
|
1378
1403
|
// the whole session when the stream can be sealed per utterance without
|
|
1379
1404
|
// closing it. Either the provider is finalize-capable (release flushes via
|
|
@@ -1406,6 +1431,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1406
1431
|
private readonly finalizeGraceMs: number;
|
|
1407
1432
|
// Rotates through the progress fallback phrases across the session's turns.
|
|
1408
1433
|
private progressPhraseCounter = 0;
|
|
1434
|
+
private modeSessionDeliveryCounter = 0;
|
|
1409
1435
|
// Progress-only phrasing service. It never makes routing decisions and
|
|
1410
1436
|
// never emits an answer.
|
|
1411
1437
|
private readonly progressNarrator: VoiceProgressNarrator | null;
|
|
@@ -1416,17 +1442,14 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1416
1442
|
// Complete Flux tunables (the constructor schema-parses the partial option
|
|
1417
1443
|
// once, so every field carries its `liveVoice.flux` schema default).
|
|
1418
1444
|
private readonly fluxConfig: LiveVoiceFluxConfig;
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
* it is, with no provider turn detection in the picture.
|
|
1428
|
-
*/
|
|
1429
|
-
private providerTurnEndActive = false;
|
|
1445
|
+
// Provider turn starts own hands-free interruption independently of the
|
|
1446
|
+
// end-of-turn setting. Manual sessions keep their client-owned boundaries.
|
|
1447
|
+
private providerTurnStartActive = false;
|
|
1448
|
+
private latestProviderTurnStartIndex: number | null = null;
|
|
1449
|
+
|
|
1450
|
+
private get providerTurnEndActive(): boolean {
|
|
1451
|
+
return this.providerTurnStartActive && this.fluxConfig.turnEnd.enabled;
|
|
1452
|
+
}
|
|
1430
1453
|
// Wall-clock of the newest above-gate audio chunk, tracked in every
|
|
1431
1454
|
// server-VAD session. It is the local VAD's speech-stop mark: the one anchor
|
|
1432
1455
|
// the reported end-of-turn latency is measured from whichever decider
|
|
@@ -1464,6 +1487,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1464
1487
|
options: LiveVoiceSessionOptions = {},
|
|
1465
1488
|
) {
|
|
1466
1489
|
this.context = context;
|
|
1490
|
+
this.microphoneIdleGate = new MicrophoneIdleGate(
|
|
1491
|
+
context.startFrame.audio.sampleRate,
|
|
1492
|
+
);
|
|
1467
1493
|
this.resolveTranscriber =
|
|
1468
1494
|
options.resolveTranscriber ?? defaultResolveStreamingTranscriber;
|
|
1469
1495
|
this.resolveCredentialReadiness =
|
|
@@ -1475,6 +1501,8 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1475
1501
|
options.spawnBackgroundContinuation ?? null;
|
|
1476
1502
|
this.labelBackgroundContinuation =
|
|
1477
1503
|
options.labelBackgroundContinuation ?? null;
|
|
1504
|
+
this.judgeBackgroundContinuation =
|
|
1505
|
+
options.judgeBackgroundContinuation ?? null;
|
|
1478
1506
|
this.getTurnTeardown = options.getTurnTeardown ?? null;
|
|
1479
1507
|
this.detachTeardownSettleTimeoutMs =
|
|
1480
1508
|
options.detachTeardownSettleTimeoutMs ??
|
|
@@ -1493,11 +1521,12 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1493
1521
|
this.createTurnId = options.createTurnId ?? randomUUID;
|
|
1494
1522
|
this.conversationId =
|
|
1495
1523
|
context.startFrame.conversationId ?? context.sessionId;
|
|
1524
|
+
this.acquireModeSessionResidency = options.acquireModeSessionResidency;
|
|
1496
1525
|
this.liveActivityReporter =
|
|
1497
1526
|
options.liveActivityReporter ??
|
|
1498
1527
|
new LiveActivityReporter(this.conversationId);
|
|
1499
1528
|
this.metricsClock = options.metricsClock ?? Date.now;
|
|
1500
|
-
this.metrics = new
|
|
1529
|
+
this.metrics = new VoiceMetricsCollector({
|
|
1501
1530
|
sessionId: context.sessionId,
|
|
1502
1531
|
conversationId: this.conversationId,
|
|
1503
1532
|
...(options.metricsClock ? { clock: options.metricsClock } : {}),
|
|
@@ -1611,6 +1640,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1611
1640
|
// surfaces as a non-recoverable error frame instead of a start rejection.
|
|
1612
1641
|
this.state = "active";
|
|
1613
1642
|
this.reachedActive = true;
|
|
1643
|
+
if (this.turnDetector) {
|
|
1644
|
+
this.logInputDiagnostic("voice_input_started");
|
|
1645
|
+
}
|
|
1614
1646
|
void this.armUtterance().catch(() => {});
|
|
1615
1647
|
this.metrics.markReady();
|
|
1616
1648
|
await this.sendFrame({
|
|
@@ -1622,8 +1654,10 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1622
1654
|
// is a property of the daemon and not of what this client asked for.
|
|
1623
1655
|
// A client reading it back absent is talking to one that predates it.
|
|
1624
1656
|
textInput: true,
|
|
1657
|
+
...(this.acquireModeSessionResidency ? { sightSessions: true } : {}),
|
|
1625
1658
|
...(this.audioInput ? {} : { audioInput: false }),
|
|
1626
1659
|
});
|
|
1660
|
+
this.scheduleTaskAnnouncement();
|
|
1627
1661
|
}
|
|
1628
1662
|
|
|
1629
1663
|
async handleClientFrame(frame: LiveVoiceClientFrame): Promise<void> {
|
|
@@ -1654,8 +1688,14 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1654
1688
|
case "attach_frame":
|
|
1655
1689
|
this.parkTurnFrame(frame);
|
|
1656
1690
|
return;
|
|
1691
|
+
case "sight_start":
|
|
1692
|
+
await this.sequenceSightFrame(frame);
|
|
1693
|
+
return;
|
|
1694
|
+
case "sight_end":
|
|
1695
|
+
await this.sequenceSightFrame(frame);
|
|
1696
|
+
return;
|
|
1657
1697
|
case "sight_frame":
|
|
1658
|
-
this.
|
|
1698
|
+
await this.sequenceSightFrame(frame);
|
|
1659
1699
|
return;
|
|
1660
1700
|
case "text":
|
|
1661
1701
|
await this.handleTextTurn(frame);
|
|
@@ -1663,6 +1703,35 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1663
1703
|
}
|
|
1664
1704
|
}
|
|
1665
1705
|
|
|
1706
|
+
private sequenceSightFrame(
|
|
1707
|
+
frame: Extract<
|
|
1708
|
+
LiveVoiceClientFrame,
|
|
1709
|
+
{ type: "sight_start" | "sight_end" | "sight_frame" }
|
|
1710
|
+
>,
|
|
1711
|
+
): Promise<void> {
|
|
1712
|
+
const operation = this.sightFrameSequence.then(async () => {
|
|
1713
|
+
if (this.isClosed || this.state === "failed") {
|
|
1714
|
+
if (frame.type === "sight_frame") {
|
|
1715
|
+
this.reclaimParkedFrame(frame.attachmentId);
|
|
1716
|
+
}
|
|
1717
|
+
return;
|
|
1718
|
+
}
|
|
1719
|
+
switch (frame.type) {
|
|
1720
|
+
case "sight_start":
|
|
1721
|
+
await this.startSightSession(frame.cameraEpoch, frame.source);
|
|
1722
|
+
return;
|
|
1723
|
+
case "sight_end":
|
|
1724
|
+
this.endSightSession(frame.cameraEpoch);
|
|
1725
|
+
return;
|
|
1726
|
+
case "sight_frame":
|
|
1727
|
+
this.persistSightFrame(frame);
|
|
1728
|
+
return;
|
|
1729
|
+
}
|
|
1730
|
+
});
|
|
1731
|
+
this.sightFrameSequence = operation.catch(() => {});
|
|
1732
|
+
return operation;
|
|
1733
|
+
}
|
|
1734
|
+
|
|
1666
1735
|
/**
|
|
1667
1736
|
* Run a turn the user typed instead of spoke.
|
|
1668
1737
|
*
|
|
@@ -1769,6 +1838,24 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1769
1838
|
*/
|
|
1770
1839
|
private persistSightFrame(frame: LiveVoiceClientSightFrameFrame): void {
|
|
1771
1840
|
const receivedAtMs = Date.now();
|
|
1841
|
+
const cameraToken =
|
|
1842
|
+
frame.cameraEpoch === undefined
|
|
1843
|
+
? undefined
|
|
1844
|
+
: this.cameraModeSessions?.beginFrame(frame.cameraEpoch, frame.source);
|
|
1845
|
+
if (frame.cameraEpoch !== undefined && !cameraToken) {
|
|
1846
|
+
deleteOrphanAttachments([frame.attachmentId]);
|
|
1847
|
+
if (!this.isClosed) {
|
|
1848
|
+
void this.sendFrame({
|
|
1849
|
+
type: "error",
|
|
1850
|
+
code: LiveVoiceProtocolErrorCode.InvalidFrame,
|
|
1851
|
+
message: "That camera run is no longer active.",
|
|
1852
|
+
frameType: "sight_frame",
|
|
1853
|
+
attachmentId: frame.attachmentId,
|
|
1854
|
+
recoverable: true,
|
|
1855
|
+
});
|
|
1856
|
+
}
|
|
1857
|
+
return;
|
|
1858
|
+
}
|
|
1772
1859
|
// One line per keep, written when the row lands, carrying the whole
|
|
1773
1860
|
// timeline: the client leg the frame reported, the daemon leg the persist
|
|
1774
1861
|
// measured, and the distance from the speech onset the frame may have
|
|
@@ -1778,11 +1865,29 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1778
1865
|
this.lastSpeechStartedAtMs === null
|
|
1779
1866
|
? null
|
|
1780
1867
|
: receivedAtMs - this.lastSpeechStartedAtMs;
|
|
1781
|
-
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
|
|
1785
|
-
|
|
1868
|
+
const settleFrame = (
|
|
1869
|
+
result: LiveVoicePhotoResult,
|
|
1870
|
+
rejection?: unknown,
|
|
1871
|
+
): void => {
|
|
1872
|
+
if (rejection !== undefined) {
|
|
1873
|
+
log.warn(
|
|
1874
|
+
{ err: rejection, attachmentId: frame.attachmentId },
|
|
1875
|
+
"Sight frame persistence rejected unexpectedly",
|
|
1876
|
+
);
|
|
1877
|
+
}
|
|
1878
|
+
try {
|
|
1879
|
+
this.cameraModeSessions?.finishFrame(
|
|
1880
|
+
cameraToken,
|
|
1881
|
+
result.ok && result.messageId
|
|
1882
|
+
? { messageId: result.messageId, at: receivedAtMs }
|
|
1883
|
+
: undefined,
|
|
1884
|
+
);
|
|
1885
|
+
} catch (err) {
|
|
1886
|
+
log.warn(
|
|
1887
|
+
{ err, attachmentId: frame.attachmentId },
|
|
1888
|
+
"Could not record camera session ownership",
|
|
1889
|
+
);
|
|
1890
|
+
}
|
|
1786
1891
|
log.info(
|
|
1787
1892
|
{
|
|
1788
1893
|
attachmentId: frame.attachmentId,
|
|
@@ -1815,9 +1920,76 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1815
1920
|
recoverable: true,
|
|
1816
1921
|
});
|
|
1817
1922
|
}
|
|
1923
|
+
};
|
|
1924
|
+
void persistAmbientSightFrame(
|
|
1925
|
+
this.conversationId,
|
|
1926
|
+
frame.attachmentId,
|
|
1927
|
+
"voice",
|
|
1928
|
+
undefined,
|
|
1929
|
+
undefined,
|
|
1930
|
+
cameraToken?.owner,
|
|
1931
|
+
).then(
|
|
1932
|
+
(result) => settleFrame(result),
|
|
1933
|
+
(err) => settleFrame({ ok: false }, err),
|
|
1934
|
+
);
|
|
1935
|
+
}
|
|
1936
|
+
|
|
1937
|
+
private async startSightSession(
|
|
1938
|
+
cameraEpoch: number,
|
|
1939
|
+
source: "live" | "ambient" | undefined,
|
|
1940
|
+
): Promise<void> {
|
|
1941
|
+
let acquiredProducer = false;
|
|
1942
|
+
if (!this.cameraModeSessions && this.acquireModeSessionResidency) {
|
|
1943
|
+
try {
|
|
1944
|
+
const residency = await this.acquireModeSessionResidency(
|
|
1945
|
+
this.conversationId,
|
|
1946
|
+
);
|
|
1947
|
+
if (this.isClosed || this.state === "failed") {
|
|
1948
|
+
residency.release();
|
|
1949
|
+
return;
|
|
1950
|
+
}
|
|
1951
|
+
this.cameraModeSessions = new CameraModeSessionProducer(
|
|
1952
|
+
residency.coordinator,
|
|
1953
|
+
this.context.sessionId,
|
|
1954
|
+
);
|
|
1955
|
+
this.releaseModeSessionResidency = residency.release;
|
|
1956
|
+
acquiredProducer = true;
|
|
1957
|
+
} catch (err) {
|
|
1958
|
+
log.warn({ err, cameraEpoch }, "Could not prepare camera session");
|
|
1959
|
+
}
|
|
1960
|
+
}
|
|
1961
|
+
let handle: ModeSessionSourceHandle | undefined;
|
|
1962
|
+
try {
|
|
1963
|
+
handle = this.cameraModeSessions?.start(cameraEpoch, source ?? "live");
|
|
1964
|
+
} catch (err) {
|
|
1965
|
+
log.warn({ err, cameraEpoch }, "Could not start camera session");
|
|
1966
|
+
}
|
|
1967
|
+
if (!handle && acquiredProducer) {
|
|
1968
|
+
this.cameraModeSessions = undefined;
|
|
1969
|
+
const release = this.releaseModeSessionResidency;
|
|
1970
|
+
this.releaseModeSessionResidency = undefined;
|
|
1971
|
+
release?.();
|
|
1972
|
+
}
|
|
1973
|
+
if (handle || this.isClosed) {
|
|
1974
|
+
return;
|
|
1975
|
+
}
|
|
1976
|
+
void this.sendFrame({
|
|
1977
|
+
type: "error",
|
|
1978
|
+
code: LiveVoiceProtocolErrorCode.InvalidFrame,
|
|
1979
|
+
message: "Could not start that camera run.",
|
|
1980
|
+
frameType: "sight_start",
|
|
1981
|
+
recoverable: true,
|
|
1818
1982
|
});
|
|
1819
1983
|
}
|
|
1820
1984
|
|
|
1985
|
+
private endSightSession(cameraEpoch: number): void {
|
|
1986
|
+
try {
|
|
1987
|
+
this.cameraModeSessions?.end(cameraEpoch);
|
|
1988
|
+
} catch (err) {
|
|
1989
|
+
log.warn({ err, cameraEpoch }, "Could not end camera session");
|
|
1990
|
+
}
|
|
1991
|
+
}
|
|
1992
|
+
|
|
1821
1993
|
/**
|
|
1822
1994
|
* Park a camera frame for the next turn to carry, or give up the parked one.
|
|
1823
1995
|
*
|
|
@@ -1926,6 +2098,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1926
2098
|
if (frame.bargeInMinSpeechMs !== undefined) {
|
|
1927
2099
|
this.bargeInMinSpeechMs = frame.bargeInMinSpeechMs;
|
|
1928
2100
|
}
|
|
2101
|
+
this.logInputDiagnostic("voice_input_config_updated");
|
|
1929
2102
|
}
|
|
1930
2103
|
|
|
1931
2104
|
async handleBinaryAudio(chunk: Uint8Array): Promise<void> {
|
|
@@ -1937,12 +2110,31 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1937
2110
|
return;
|
|
1938
2111
|
}
|
|
1939
2112
|
|
|
2113
|
+
try {
|
|
2114
|
+
this.cameraModeSessions?.close(
|
|
2115
|
+
reason === "client_end"
|
|
2116
|
+
? { status: "completed", endReason: "camera_session_ended" }
|
|
2117
|
+
: { status: "interrupted", endReason: `camera_${reason}` },
|
|
2118
|
+
);
|
|
2119
|
+
} catch (err) {
|
|
2120
|
+
log.warn({ err, reason }, "Could not close camera session");
|
|
2121
|
+
}
|
|
2122
|
+
this.releaseModeSessionResidency?.();
|
|
2123
|
+
this.releaseModeSessionResidency = undefined;
|
|
2124
|
+
|
|
2125
|
+
if (this.turnDetector) {
|
|
2126
|
+
this.logInputDiagnostic("voice_input_closed", {
|
|
2127
|
+
reason,
|
|
2128
|
+
summary: this.inputDiagnostics.flush(),
|
|
2129
|
+
});
|
|
2130
|
+
}
|
|
2131
|
+
|
|
1940
2132
|
// Recorded first, and independently of `shouldEmitSessionEndMetrics`
|
|
1941
2133
|
// below: that flag governs the client-facing `metrics` frame, and a
|
|
1942
2134
|
// failed session (which suppresses the frame) is the one whose end
|
|
1943
2135
|
// telemetry matters most. Session duration downstream is this row's
|
|
1944
2136
|
// `recorded_at` minus the started row's, so it must be written before the
|
|
1945
|
-
// teardown below, which
|
|
2137
|
+
// teardown below, which waits for the active turn and can run long.
|
|
1946
2138
|
const failed = this.state === "failed";
|
|
1947
2139
|
// Only a session that never dispatched a turn gets a silence reason; for
|
|
1948
2140
|
// every other session the question is meaningless.
|
|
@@ -1985,9 +2177,12 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1985
2177
|
|
|
1986
2178
|
const shouldEmitSessionEndMetrics = this.state !== "failed";
|
|
1987
2179
|
this.state = "closed";
|
|
2180
|
+
this.taskOutcomesClosing = true;
|
|
2181
|
+
this.taskOutcomes.interruptPlayback(Date.now());
|
|
2182
|
+
const turnTeardown = this.getTurnTeardown?.(this.conversationId);
|
|
1988
2183
|
this.clearForegroundTask("session_closed");
|
|
1989
2184
|
// Retire the island before the teardown below starts awaiting things. A
|
|
1990
|
-
// close can take a while
|
|
2185
|
+
// close can take a while waiting for turn teardown, and
|
|
1991
2186
|
// an activity left asserting "Speaking…" through it is exactly the stale
|
|
1992
2187
|
// claim this reporter exists to prevent.
|
|
1993
2188
|
this.liveActivityReporter.end();
|
|
@@ -1998,13 +2193,6 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1998
2193
|
// again on its next session, so the staged row and bytes go with the call.
|
|
1999
2194
|
this.reclaimParkedFrame(this.pendingTurnAttachmentId);
|
|
2000
2195
|
this.pendingTurnAttachmentId = null;
|
|
2001
|
-
// There is no longer anyone on the call to speak to, so a queued
|
|
2002
|
-
// announcement cannot be spoken — but the answer behind it is finished
|
|
2003
|
-
// work the user asked for, so it takes the same conversation route a
|
|
2004
|
-
// continuation finishing after this point takes. Then the queue is
|
|
2005
|
-
// cleared: the announcement is dead either way.
|
|
2006
|
-
await this.deliverPendingContinuationToConversation();
|
|
2007
|
-
this.clearContinuationAnnouncement();
|
|
2008
2196
|
this.clearPendingLook();
|
|
2009
2197
|
this.stopSessionTranscriber();
|
|
2010
2198
|
// Detached continuations outlive the call. A deliberate `interrupt()`
|
|
@@ -2012,6 +2200,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2012
2200
|
// turn to fold into, a continuation that finishes after this point delivers
|
|
2013
2201
|
// into the conversation instead (see detachInterruptedTurn).
|
|
2014
2202
|
await this.cancelAssistantTurn("session_closed");
|
|
2203
|
+
await this.deliverTaskOutcomesToConversation(turnTeardown);
|
|
2015
2204
|
if (shouldEmitSessionEndMetrics) {
|
|
2016
2205
|
await this.emitSessionEndMetrics();
|
|
2017
2206
|
}
|
|
@@ -2073,11 +2262,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2073
2262
|
// in exactly the configuration roles exist for (live voice on flux,
|
|
2074
2263
|
// global on base deepgram), which reinstates the race this seed
|
|
2075
2264
|
// prevents.
|
|
2076
|
-
this.
|
|
2077
|
-
|
|
2078
|
-
|
|
2079
|
-
sttCatalogKeyForRole(stt, "liveVoice"),
|
|
2080
|
-
) &&
|
|
2265
|
+
this.latestProviderTurnStartIndex = null;
|
|
2266
|
+
this.setProviderTurnDetectionActive(
|
|
2267
|
+
supportsProviderTurnDetection(sttCatalogKeyForRole(stt, "liveVoice")) &&
|
|
2081
2268
|
this.turnDetector !== null,
|
|
2082
2269
|
);
|
|
2083
2270
|
const transcriber = await this.resolveTranscriber({
|
|
@@ -2093,7 +2280,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2093
2280
|
if (!transcriber) {
|
|
2094
2281
|
// No stream answered, so no end-of-turn ever will: the guess above
|
|
2095
2282
|
// must not outlive the dial that disproved it.
|
|
2096
|
-
this.
|
|
2283
|
+
this.setProviderTurnDetectionActive(false);
|
|
2097
2284
|
return {
|
|
2098
2285
|
status: "unavailable",
|
|
2099
2286
|
message: unavailableTranscriberMessage(),
|
|
@@ -2107,11 +2294,23 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2107
2294
|
// change under a live session, so this normally confirms the guess; it
|
|
2108
2295
|
// clears it when the dial fell back to another provider or resolved
|
|
2109
2296
|
// one the config did not name.
|
|
2110
|
-
this.
|
|
2111
|
-
|
|
2112
|
-
|
|
2297
|
+
const expectedProviderSpeechStart = this.providerTurnStartActive;
|
|
2298
|
+
this.setProviderTurnDetectionActive(
|
|
2299
|
+
supportsProviderTurnDetection(transcriber.providerId) &&
|
|
2113
2300
|
this.turnDetector !== null,
|
|
2114
2301
|
);
|
|
2302
|
+
// A non-Flux fallback cannot confirm the onset suppressed during the
|
|
2303
|
+
// dial. Preserve its guard result even if local silence already released.
|
|
2304
|
+
const preDialBargeInGuard = utterance.preDialBargeInGuard;
|
|
2305
|
+
utterance.preDialBargeInGuard = null;
|
|
2306
|
+
if (
|
|
2307
|
+
expectedProviderSpeechStart &&
|
|
2308
|
+
!this.providerTurnStartActive &&
|
|
2309
|
+
utterance.speechRouted &&
|
|
2310
|
+
!utterance.completed
|
|
2311
|
+
) {
|
|
2312
|
+
this.handleSpeechStart("local", preDialBargeInGuard);
|
|
2313
|
+
}
|
|
2115
2314
|
if (
|
|
2116
2315
|
this.turnDetector &&
|
|
2117
2316
|
(typeof transcriber.finalizeUtterance === "function" ||
|
|
@@ -2155,7 +2354,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2155
2354
|
}
|
|
2156
2355
|
// The dial threw, so the pre-dial guess is disproved the same way the
|
|
2157
2356
|
// unavailable case disproves it: nothing will send an end-of-turn.
|
|
2158
|
-
this.
|
|
2357
|
+
this.setProviderTurnDetectionActive(false);
|
|
2159
2358
|
return {
|
|
2160
2359
|
status: "error",
|
|
2161
2360
|
message: `Live voice transcription could not be started: ${errorMessage(
|
|
@@ -2194,6 +2393,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2194
2393
|
if (transcriber && this.sharedTranscriber === transcriber) {
|
|
2195
2394
|
this.sharedTranscriber = null;
|
|
2196
2395
|
this.sharedTranscriberLanguage = undefined;
|
|
2396
|
+
this.microphoneIdleGate.reset();
|
|
2197
2397
|
}
|
|
2198
2398
|
}
|
|
2199
2399
|
|
|
@@ -2254,6 +2454,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2254
2454
|
// server_vad capability; manual sessions keep single-utterance semantics
|
|
2255
2455
|
// (no speculative post-turn transcriber).
|
|
2256
2456
|
private scheduleRearmAfterTurn(): void {
|
|
2457
|
+
this.scheduleTaskAnnouncement();
|
|
2257
2458
|
if (!this.turnDetector) {
|
|
2258
2459
|
return;
|
|
2259
2460
|
}
|
|
@@ -2372,16 +2573,112 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2372
2573
|
return;
|
|
2373
2574
|
}
|
|
2374
2575
|
|
|
2375
|
-
|
|
2576
|
+
const receivedAtMs = Date.now();
|
|
2577
|
+
const sampleRate = this.context.startFrame.audio.sampleRate;
|
|
2578
|
+
const meanAmplitude = pcm16MeanAmplitude(chunk);
|
|
2579
|
+
const before = {
|
|
2580
|
+
baseThreshold: this.effectiveBaseThreshold(),
|
|
2581
|
+
roomNoiseFloor: this.roomNoiseFloor.floor,
|
|
2582
|
+
echoEnergy: this.echoEnergyEma,
|
|
2583
|
+
echoOnsetLapsed: this.echoOnsetLapsed,
|
|
2584
|
+
echoWindowMs: this.echoWindowTotalAudioMs,
|
|
2585
|
+
echoGuardCarryover: this.echoWindowGuardCarryover,
|
|
2586
|
+
echoReferenceMs: pcm16DurationMs(
|
|
2587
|
+
this.echoReferenceAudio.byteLength,
|
|
2588
|
+
sampleRate,
|
|
2589
|
+
),
|
|
2590
|
+
playbackRemainingMs: Math.max(
|
|
2591
|
+
0,
|
|
2592
|
+
this.assistantPlaybackTailUntilMs - receivedAtMs,
|
|
2593
|
+
),
|
|
2594
|
+
playbackEchoPossible: this.isAssistantPlaybackEchoPossible(),
|
|
2595
|
+
};
|
|
2596
|
+
this.diagnosticEchoCorrelation = null;
|
|
2597
|
+
const classifiedChunks = this.classifyVadEnergy(chunk, meanAmplitude);
|
|
2598
|
+
const durations = { speechMs: 0, silenceMs: 0, echoMs: 0 };
|
|
2599
|
+
for (const classified of classifiedChunks) {
|
|
2600
|
+
durations[`${classified.classification}Ms`] += pcm16DurationMs(
|
|
2601
|
+
classified.chunk.byteLength,
|
|
2602
|
+
sampleRate,
|
|
2603
|
+
);
|
|
2604
|
+
}
|
|
2605
|
+
const summary = this.inputDiagnostics.observe({
|
|
2606
|
+
receivedAtMs,
|
|
2607
|
+
chunkMs: pcm16DurationMs(chunk.byteLength, sampleRate),
|
|
2608
|
+
meanAmplitude,
|
|
2609
|
+
...before,
|
|
2610
|
+
...durations,
|
|
2611
|
+
echoProbeMs: pcm16DurationMs(
|
|
2612
|
+
this.echoProbeChunks.reduce(
|
|
2613
|
+
(bytes, entry) => bytes + entry.byteLength,
|
|
2614
|
+
0,
|
|
2615
|
+
),
|
|
2616
|
+
sampleRate,
|
|
2617
|
+
),
|
|
2618
|
+
echoCorrelation: this.diagnosticEchoCorrelation,
|
|
2619
|
+
});
|
|
2620
|
+
if (summary) {
|
|
2621
|
+
this.logInputDiagnostic("voice_input_window", { summary });
|
|
2622
|
+
}
|
|
2623
|
+
|
|
2624
|
+
for (const classified of classifiedChunks) {
|
|
2376
2625
|
await this.handleClassifiedVadAudio(detector, classified);
|
|
2377
2626
|
}
|
|
2378
2627
|
}
|
|
2379
2628
|
|
|
2629
|
+
private logInputDiagnostic(
|
|
2630
|
+
event: string,
|
|
2631
|
+
details: Record<string, unknown> = {},
|
|
2632
|
+
): void {
|
|
2633
|
+
if (!this.turnDetector) {
|
|
2634
|
+
return;
|
|
2635
|
+
}
|
|
2636
|
+
const turn = this.activeAssistantTurn;
|
|
2637
|
+
const task = this.foregroundTaskState;
|
|
2638
|
+
log.info(
|
|
2639
|
+
{
|
|
2640
|
+
event,
|
|
2641
|
+
sessionId: this.context.sessionId,
|
|
2642
|
+
conversationId: this.conversationId,
|
|
2643
|
+
speechGeneration: this.vadSpeechGeneration,
|
|
2644
|
+
inputTurnId: this.currentUtterance?.turnId ?? null,
|
|
2645
|
+
assistantTurnId: turn?.turnId ?? null,
|
|
2646
|
+
assistantCompleted: turn?.assistantCompleted ?? null,
|
|
2647
|
+
ttsAudioStarted: turn?.ttsAudioStarted ?? false,
|
|
2648
|
+
foregroundTaskPhase: task?.phase ?? null,
|
|
2649
|
+
hostToolStarted: task?.hostToolStarted ?? null,
|
|
2650
|
+
inputSampleRate: this.context.startFrame.audio.sampleRate,
|
|
2651
|
+
ttsSampleRate: this.diagnosticTtsSampleRate,
|
|
2652
|
+
configuredThreshold:
|
|
2653
|
+
this.speechEnergyThreshold ?? DEFAULT_SPEECH_ENERGY_THRESHOLD,
|
|
2654
|
+
effectiveThreshold: this.effectiveBaseThreshold(),
|
|
2655
|
+
roomNoiseFloor: this.roomNoiseFloor.floor,
|
|
2656
|
+
noiseFloorMargin: this.noiseFloorMargin,
|
|
2657
|
+
echoMargin: this.echoBargeInMargin,
|
|
2658
|
+
echoEnergy: this.echoEnergyEma,
|
|
2659
|
+
echoOnsetLapsed: this.echoOnsetLapsed,
|
|
2660
|
+
playbackRemainingMs: Math.max(
|
|
2661
|
+
0,
|
|
2662
|
+
this.assistantPlaybackTailUntilMs - Date.now(),
|
|
2663
|
+
),
|
|
2664
|
+
echoDrainSlackMs: this.echoDrainSlackMs,
|
|
2665
|
+
bargeInMinSpeechMs: this.bargeInMinSpeechMs,
|
|
2666
|
+
microphoneGateClosed:
|
|
2667
|
+
this.providerTurnEndActive && this.microphoneIdleGate.closed,
|
|
2668
|
+
...details,
|
|
2669
|
+
},
|
|
2670
|
+
"Live voice input diagnostics",
|
|
2671
|
+
);
|
|
2672
|
+
}
|
|
2673
|
+
|
|
2380
2674
|
private async handleClassifiedVadAudio(
|
|
2381
2675
|
detector: MediaTurnDetector,
|
|
2382
2676
|
classified: VadClassifiedChunk,
|
|
2383
2677
|
): Promise<void> {
|
|
2384
|
-
const {
|
|
2678
|
+
const { classification: energyClassification } = classified;
|
|
2679
|
+
let { chunk } = classified;
|
|
2680
|
+
const inputCycle = this.currentUtterance;
|
|
2681
|
+
this.trackPreDialBargeInGuard(inputCycle, classified);
|
|
2385
2682
|
const hasSpeech = energyClassification === "speech";
|
|
2386
2683
|
detector.onMediaChunk(hasSpeech);
|
|
2387
2684
|
this.trackBargeInGuard(energyClassification, chunk);
|
|
@@ -2389,11 +2686,13 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2389
2686
|
this.localSpeechStopAtMs = Date.now();
|
|
2390
2687
|
}
|
|
2391
2688
|
|
|
2392
|
-
//
|
|
2393
|
-
//
|
|
2394
|
-
// follow-up turn.
|
|
2689
|
+
// Flux needs elapsed audio through pauses. Confirmed playback echo keeps
|
|
2690
|
+
// its duration but carries no speaker content into transcription.
|
|
2395
2691
|
if (energyClassification === "echo") {
|
|
2396
|
-
|
|
2692
|
+
if (!this.providerTurnEndActive) {
|
|
2693
|
+
return;
|
|
2694
|
+
}
|
|
2695
|
+
chunk = Buffer.alloc(chunk.byteLength);
|
|
2397
2696
|
}
|
|
2398
2697
|
// Measured past the echo gate, so a greeting heard through the speaker
|
|
2399
2698
|
// cannot stand in for the user on a silent close.
|
|
@@ -2402,11 +2701,20 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2402
2701
|
this.peakChunkAmplitude = meanAmplitude;
|
|
2403
2702
|
}
|
|
2404
2703
|
|
|
2405
|
-
|
|
2406
|
-
|
|
2407
|
-
|
|
2408
|
-
|
|
2409
|
-
|
|
2704
|
+
if (this.providerTurnEndActive) {
|
|
2705
|
+
const wasClosed = this.microphoneIdleGate.closed;
|
|
2706
|
+
chunk = this.microphoneIdleGate.process(chunk, hasSpeech);
|
|
2707
|
+
if (wasClosed !== this.microphoneIdleGate.closed) {
|
|
2708
|
+
this.logInputDiagnostic("voice_input_microphone_gate_changed");
|
|
2709
|
+
}
|
|
2710
|
+
if (chunk.byteLength === 0) {
|
|
2711
|
+
return;
|
|
2712
|
+
}
|
|
2713
|
+
}
|
|
2714
|
+
|
|
2715
|
+
// Locally endpointed streams keep idle audio in pre-roll.
|
|
2716
|
+
if (!hasSpeech && !detector.isActive && !this.providerTurnEndActive) {
|
|
2717
|
+
this.pushVadPreRoll(chunk, "silence");
|
|
2410
2718
|
return;
|
|
2411
2719
|
}
|
|
2412
2720
|
|
|
@@ -2437,7 +2745,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2437
2745
|
// keeps its old-language stream and the language change applies from
|
|
2438
2746
|
// the following utterance.
|
|
2439
2747
|
utterance.finalTranscriptSegments.length === 0 &&
|
|
2440
|
-
utterance.latestPartialText ===
|
|
2748
|
+
(utterance.latestPartialText?.trim() ?? "").length === 0 &&
|
|
2441
2749
|
this.sharedStreamLanguageIsStale()
|
|
2442
2750
|
) {
|
|
2443
2751
|
this.retireSharedTranscriberForRedial(sharedForLanguage);
|
|
@@ -2448,13 +2756,17 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2448
2756
|
if (utterance.released || utterance.completed) {
|
|
2449
2757
|
// Parked speech makes silent chunks arm-worthy too: the parked
|
|
2450
2758
|
// utterance must flush without requiring more speech.
|
|
2451
|
-
if (
|
|
2759
|
+
if (
|
|
2760
|
+
!hasSpeech &&
|
|
2761
|
+
!this.vadPreRollHasSpeech &&
|
|
2762
|
+
!this.providerTurnEndActive
|
|
2763
|
+
) {
|
|
2452
2764
|
return;
|
|
2453
2765
|
}
|
|
2454
2766
|
if (!this.canArmNextUtterance(utterance)) {
|
|
2455
2767
|
// Speech in the release→turn-start window: hold it in the pre-roll
|
|
2456
2768
|
// ring so it flushes into the next utterance once it arms.
|
|
2457
|
-
this.pushVadPreRoll(chunk,
|
|
2769
|
+
this.pushVadPreRoll(chunk, energyClassification);
|
|
2458
2770
|
return;
|
|
2459
2771
|
}
|
|
2460
2772
|
// Sets currentUtterance synchronously; the transcriber resolves async
|
|
@@ -2466,6 +2778,10 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2466
2778
|
}
|
|
2467
2779
|
}
|
|
2468
2780
|
|
|
2781
|
+
if (utterance !== inputCycle) {
|
|
2782
|
+
this.trackPreDialBargeInGuard(utterance, classified);
|
|
2783
|
+
}
|
|
2784
|
+
|
|
2469
2785
|
// Speech is now reaching the cycle, either in this chunk or parked in
|
|
2470
2786
|
// the pre-roll about to flush; read the flag before takeVadPreRoll
|
|
2471
2787
|
// resets it.
|
|
@@ -2489,8 +2805,10 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2489
2805
|
* onset is neither learned as echo nor lost. Once seeded, the EMA follows
|
|
2490
2806
|
* confirmed echo while speech above the learned margin remains frozen out.
|
|
2491
2807
|
*/
|
|
2492
|
-
private classifyVadEnergy(
|
|
2493
|
-
|
|
2808
|
+
private classifyVadEnergy(
|
|
2809
|
+
chunk: Buffer,
|
|
2810
|
+
meanAmplitude: number,
|
|
2811
|
+
): VadClassifiedChunk[] {
|
|
2494
2812
|
const chunkMs = pcm16DurationMs(
|
|
2495
2813
|
chunk.byteLength,
|
|
2496
2814
|
this.context.startFrame.audio.sampleRate,
|
|
@@ -2612,12 +2930,12 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2612
2930
|
chunk.byteLength,
|
|
2613
2931
|
Math.ceil((sampleRate * ECHO_CORRELATION_PROBE_MS * 2) / 1_000),
|
|
2614
2932
|
);
|
|
2615
|
-
|
|
2616
|
-
|
|
2617
|
-
|
|
2618
|
-
this.echoReferenceAudio,
|
|
2619
|
-
) >= ECHO_CORRELATION_THRESHOLD
|
|
2933
|
+
const correlation = pcm16MaxNormalizedCorrelation(
|
|
2934
|
+
chunk.subarray(0, probeByteLength),
|
|
2935
|
+
this.echoReferenceAudio,
|
|
2620
2936
|
);
|
|
2937
|
+
this.diagnosticEchoCorrelation = correlation;
|
|
2938
|
+
return correlation >= ECHO_CORRELATION_THRESHOLD;
|
|
2621
2939
|
}
|
|
2622
2940
|
|
|
2623
2941
|
private updateEchoEnergy(meanAmplitude: number, chunkMs: number): void {
|
|
@@ -2682,6 +3000,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2682
3000
|
}
|
|
2683
3001
|
|
|
2684
3002
|
private appendEchoReference(chunk: LiveVoiceTtsAudioChunk): void {
|
|
3003
|
+
this.diagnosticTtsSampleRate = chunk.sampleRate ?? null;
|
|
2685
3004
|
if (
|
|
2686
3005
|
chunk.contentType.split(";", 1)[0]?.trim().toLowerCase() !==
|
|
2687
3006
|
"audio/pcm" ||
|
|
@@ -2704,7 +3023,11 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2704
3023
|
utterance: UtteranceCycle,
|
|
2705
3024
|
chunk: Buffer,
|
|
2706
3025
|
): Promise<void> {
|
|
2707
|
-
|
|
3026
|
+
// Continuous idle input belongs to the STT stream, not an unbounded
|
|
3027
|
+
// recording of the room before the caller starts a request.
|
|
3028
|
+
if (utterance.speechRouted) {
|
|
3029
|
+
this.collectUserAudio(utterance, chunk);
|
|
3030
|
+
}
|
|
2708
3031
|
if (this.vadSpeechStartPending) {
|
|
2709
3032
|
this.vadSpeechStartPending = false;
|
|
2710
3033
|
this.markSpeechStart(utterance);
|
|
@@ -2724,7 +3047,21 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2724
3047
|
return utterance.completed || utterance.assistantTurnStarted;
|
|
2725
3048
|
}
|
|
2726
3049
|
|
|
2727
|
-
private pushVadPreRoll(
|
|
3050
|
+
private pushVadPreRoll(
|
|
3051
|
+
chunk: Buffer,
|
|
3052
|
+
classification: VadEnergyClassification,
|
|
3053
|
+
): void {
|
|
3054
|
+
const hasSpeech = classification === "speech";
|
|
3055
|
+
if (hasSpeech && !this.vadPreRollBargeInGuard) {
|
|
3056
|
+
this.vadPreRollBargeInGuard = createBargeInGuard(this.bargeInMinSpeechMs);
|
|
3057
|
+
}
|
|
3058
|
+
this.vadPreRollBargeInGuard?.track(
|
|
3059
|
+
classification,
|
|
3060
|
+
pcm16DurationMs(
|
|
3061
|
+
chunk.byteLength,
|
|
3062
|
+
this.context.startFrame.audio.sampleRate,
|
|
3063
|
+
),
|
|
3064
|
+
);
|
|
2728
3065
|
// A full ring never lets idle silence evict parked speech.
|
|
2729
3066
|
if (
|
|
2730
3067
|
!hasSpeech &&
|
|
@@ -2744,6 +3081,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2744
3081
|
|
|
2745
3082
|
private takeVadPreRoll(): Buffer[] {
|
|
2746
3083
|
this.vadPreRollHasSpeech = false;
|
|
3084
|
+
this.vadPreRollBargeInGuard = null;
|
|
2747
3085
|
return this.vadPreRollChunks.splice(0);
|
|
2748
3086
|
}
|
|
2749
3087
|
|
|
@@ -2753,11 +3091,13 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2753
3091
|
// Read before takeVadPreRoll resets it: a ring holding parked speech
|
|
2754
3092
|
// makes this cycle speech-bearing, a silence-only ring does not.
|
|
2755
3093
|
const preRollHadSpeech = this.vadPreRollHasSpeech;
|
|
3094
|
+
const preRollBargeInGuard = this.vadPreRollBargeInGuard;
|
|
2756
3095
|
for (const chunk of this.takeVadPreRoll()) {
|
|
2757
3096
|
this.collectUserAudio(utterance, chunk);
|
|
2758
3097
|
this.bufferPendingUtteranceAudio(utterance, chunk);
|
|
2759
3098
|
}
|
|
2760
3099
|
if (preRollHadSpeech) {
|
|
3100
|
+
utterance.preDialBargeInGuard = preRollBargeInGuard;
|
|
2761
3101
|
utterance.speechRouted = true;
|
|
2762
3102
|
this.detectedSpeech = true;
|
|
2763
3103
|
}
|
|
@@ -2783,10 +3123,19 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2783
3123
|
chunk: Buffer,
|
|
2784
3124
|
): Promise<void> {
|
|
2785
3125
|
try {
|
|
2786
|
-
utterance.transcriber
|
|
2787
|
-
|
|
2788
|
-
|
|
2789
|
-
|
|
3126
|
+
if (utterance.transcriber) {
|
|
3127
|
+
utterance.transcriber.sendAudio(
|
|
3128
|
+
chunk,
|
|
3129
|
+
this.context.startFrame.audio.mimeType,
|
|
3130
|
+
);
|
|
3131
|
+
this.inputDiagnostics.recordSttSubmission(
|
|
3132
|
+
Date.now(),
|
|
3133
|
+
pcm16DurationMs(
|
|
3134
|
+
chunk.byteLength,
|
|
3135
|
+
this.context.startFrame.audio.sampleRate,
|
|
3136
|
+
),
|
|
3137
|
+
);
|
|
3138
|
+
}
|
|
2790
3139
|
await this.drainOutboundFrames();
|
|
2791
3140
|
} catch (err) {
|
|
2792
3141
|
await this.sendFrame({
|
|
@@ -2810,15 +3159,8 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2810
3159
|
}
|
|
2811
3160
|
}
|
|
2812
3161
|
|
|
2813
|
-
//
|
|
2814
|
-
//
|
|
2815
|
-
// non-finalized turn — including a pre-TTS "thinking" turn whose reply is
|
|
2816
|
-
// still being generated, so a user can cut in before the assistant starts
|
|
2817
|
-
// talking (JARVIS-1266). Speaking over a thinking or audibly speaking turn
|
|
2818
|
-
// is deferred behind the same sustained-speech guard, so a cough or noise
|
|
2819
|
-
// blip cannot kill an unspoken reply or clip a spoken one; sustained speech
|
|
2820
|
-
// aborts the turn. Onset while listening keeps the instant speech_started
|
|
2821
|
-
// (turn-taking latency is untouched).
|
|
3162
|
+
// Local onset tracks audio routing and silence-boundary freshness even
|
|
3163
|
+
// when the provider owns the decision to interrupt playback.
|
|
2822
3164
|
private handleVadSpeechStart(): void {
|
|
2823
3165
|
if (this.isClosed || this.state === "failed") {
|
|
2824
3166
|
return;
|
|
@@ -2826,7 +3168,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2826
3168
|
|
|
2827
3169
|
this.vadSpeechStartPending = true;
|
|
2828
3170
|
// Speech resumed: an endpoint decision still in flight is stale (the
|
|
2829
|
-
// generation bump defers it), and a pending hold replay is moot
|
|
3171
|
+
// generation bump defers it), and a pending hold replay is moot: the
|
|
2830
3172
|
// utterance keeps accumulating and the detector fires a fresh turn-end.
|
|
2831
3173
|
this.vadSpeechGeneration += 1;
|
|
2832
3174
|
this.clearEndpointExtensionTimer();
|
|
@@ -2834,9 +3176,18 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2834
3176
|
// spoke through. The next silence boundary arms a fresh one.
|
|
2835
3177
|
this.clearProviderTurnEndTimer();
|
|
2836
3178
|
|
|
3179
|
+
if (!this.providerTurnStartActive) {
|
|
3180
|
+
this.handleSpeechStart("local");
|
|
3181
|
+
}
|
|
3182
|
+
}
|
|
3183
|
+
|
|
3184
|
+
private handleSpeechStart(
|
|
3185
|
+
source: "local" | "provider",
|
|
3186
|
+
preDialGuard: BargeInGuard | null = null,
|
|
3187
|
+
): void {
|
|
2837
3188
|
// Speech resumed while a speculative leg was awaiting its verdict: the
|
|
2838
3189
|
// pause was mid-thought after all. Discard silently (no frames were ever
|
|
2839
|
-
// sent for it) and let the utterance keep accumulating
|
|
3190
|
+
// sent for it) and let the utterance keep accumulating. This is the
|
|
2840
3191
|
// hold outcome decided by the caller's own voice instead of the model.
|
|
2841
3192
|
const speculative = this.activeAssistantTurn;
|
|
2842
3193
|
if (speculative?.speculativePending) {
|
|
@@ -2849,20 +3200,37 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2849
3200
|
// assistant starts talking.
|
|
2850
3201
|
const bargeableTurn = turn && !turn.finalized ? turn : null;
|
|
2851
3202
|
// The client can still be draining audible playback after tts_done
|
|
2852
|
-
// (the turn is already cleared server-side)
|
|
2853
|
-
//
|
|
3203
|
+
// (the turn is already cleared server-side). Local onset still needs
|
|
3204
|
+
// the guard, or a noise blip clips the reply's last words.
|
|
2854
3205
|
const drainingPlayback = this.isAssistantPlaybackEchoPossible();
|
|
2855
3206
|
|
|
2856
|
-
if (
|
|
2857
|
-
|
|
2858
|
-
|
|
2859
|
-
this.
|
|
2860
|
-
|
|
2861
|
-
|
|
2862
|
-
|
|
2863
|
-
|
|
3207
|
+
if (
|
|
3208
|
+
source === "local" &&
|
|
3209
|
+
(bargeableTurn || drainingPlayback) &&
|
|
3210
|
+
this.bargeInMinSpeechMs > 0
|
|
3211
|
+
) {
|
|
3212
|
+
const guard = preDialGuard ?? createBargeInGuard(this.bargeInMinSpeechMs);
|
|
3213
|
+
if (!guard.fired) {
|
|
3214
|
+
// A released pre-dial run below the threshold has already expired.
|
|
3215
|
+
if (preDialGuard && this.currentUtterance?.released) {
|
|
3216
|
+
return;
|
|
3217
|
+
}
|
|
3218
|
+
this.pendingBargeIn = {
|
|
3219
|
+
turn: bargeableTurn,
|
|
3220
|
+
guard,
|
|
3221
|
+
armedAtMs: Date.now(),
|
|
3222
|
+
resets: 0,
|
|
3223
|
+
};
|
|
3224
|
+
this.logInputDiagnostic("voice_input_barge_in_armed");
|
|
3225
|
+
return;
|
|
3226
|
+
}
|
|
2864
3227
|
}
|
|
2865
3228
|
|
|
3229
|
+
this.logInputDiagnostic("voice_input_speech_started", {
|
|
3230
|
+
source,
|
|
3231
|
+
interruptsTurn: bargeableTurn !== null,
|
|
3232
|
+
trace: this.inputDiagnostics.snapshot(Date.now()),
|
|
3233
|
+
});
|
|
2866
3234
|
this.pendingBargeIn = null;
|
|
2867
3235
|
this.assistantPlaybackTailUntilMs = 0;
|
|
2868
3236
|
this.sendSpeechStarted();
|
|
@@ -2871,11 +3239,36 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2871
3239
|
}
|
|
2872
3240
|
}
|
|
2873
3241
|
|
|
3242
|
+
private trackPreDialBargeInGuard(
|
|
3243
|
+
utterance: UtteranceCycle | null,
|
|
3244
|
+
{ chunk, classification }: VadClassifiedChunk,
|
|
3245
|
+
): void {
|
|
3246
|
+
if (
|
|
3247
|
+
!this.providerTurnStartActive ||
|
|
3248
|
+
utterance?.phase !== "pending" ||
|
|
3249
|
+
utterance.released ||
|
|
3250
|
+
utterance.transcriber !== null
|
|
3251
|
+
) {
|
|
3252
|
+
return;
|
|
3253
|
+
}
|
|
3254
|
+
const guard = (utterance.preDialBargeInGuard ??= createBargeInGuard(
|
|
3255
|
+
this.bargeInMinSpeechMs,
|
|
3256
|
+
));
|
|
3257
|
+
guard.track(
|
|
3258
|
+
classification,
|
|
3259
|
+
pcm16DurationMs(
|
|
3260
|
+
chunk.byteLength,
|
|
3261
|
+
this.context.startFrame.audio.sampleRate,
|
|
3262
|
+
),
|
|
3263
|
+
);
|
|
3264
|
+
}
|
|
3265
|
+
|
|
2874
3266
|
/**
|
|
2875
3267
|
* Tell the client the caller started speaking, and remember when, so a
|
|
2876
3268
|
* camera frame that follows can be logged against the onset it answers.
|
|
2877
3269
|
*/
|
|
2878
3270
|
private sendSpeechStarted(): void {
|
|
3271
|
+
this.taskOutcomes.interruptPlayback(Date.now());
|
|
2879
3272
|
this.lastSpeechStartedAtMs = Date.now();
|
|
2880
3273
|
void this.sendFrame({ type: "speech_started" });
|
|
2881
3274
|
}
|
|
@@ -2899,6 +3292,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2899
3292
|
);
|
|
2900
3293
|
const step = pending.guard.track(classification, chunkMs);
|
|
2901
3294
|
if (step === "reset") {
|
|
3295
|
+
pending.resets += 1;
|
|
2902
3296
|
if (this.echoWindowGuardCarryover) {
|
|
2903
3297
|
this.echoWindowGuardCarryover = false;
|
|
2904
3298
|
this.echoEnergyEma = 0;
|
|
@@ -2909,6 +3303,12 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2909
3303
|
if (step !== "fired") {
|
|
2910
3304
|
return;
|
|
2911
3305
|
}
|
|
3306
|
+
this.logInputDiagnostic("voice_input_barge_in_fired", {
|
|
3307
|
+
armedAtMs: pending.armedAtMs,
|
|
3308
|
+
guardResets: pending.resets,
|
|
3309
|
+
accumulatedSpeechMs: pending.guard.speechMs,
|
|
3310
|
+
trace: this.inputDiagnostics.snapshot(Date.now()),
|
|
3311
|
+
});
|
|
2912
3312
|
this.pendingBargeIn = null;
|
|
2913
3313
|
this.assistantPlaybackTailUntilMs = 0;
|
|
2914
3314
|
this.sendSpeechStarted();
|
|
@@ -2919,6 +3319,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2919
3319
|
}
|
|
2920
3320
|
|
|
2921
3321
|
private bargeIn(turn: ActiveAssistantTurn): void {
|
|
3322
|
+
this.taskOutcomes.interruptPlayback(Date.now());
|
|
2922
3323
|
// Abort synchronously so no tts_audio frame can follow turn_cancelled,
|
|
2923
3324
|
// and settle the cancelled turn's metrics so the next utterance's marks
|
|
2924
3325
|
// do not collide with it in the collector. turn_cancelled flushes
|
|
@@ -2933,38 +3334,20 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2933
3334
|
.trim();
|
|
2934
3335
|
this.pendingInterruptedRequest =
|
|
2935
3336
|
interruptedRequest.length > 0 ? interruptedRequest : null;
|
|
2936
|
-
//
|
|
2937
|
-
//
|
|
2938
|
-
//
|
|
2939
|
-
// full-ability background writers never share the workspace. The stashed
|
|
2940
|
-
// result is dropped with them so the barge-in follow-up (or any later)
|
|
2941
|
-
// turn can't consume an older answer before this barge-in's own
|
|
2942
|
-
// continuation completes. This barge-in's detach snapshots the stop
|
|
2943
|
-
// generation AFTER this bump (below), so it is unaffected. The abort is
|
|
2944
|
-
// signal-level, with the same accepted residual as the foreground-wins
|
|
2945
|
-
// gate (see the tool_use_start handler): a tool call already executing in
|
|
2946
|
-
// the superseded run is not awaited. The replacement's detach still waits
|
|
2947
|
-
// out the interrupted TURN's teardown before forking, which bounds the
|
|
2948
|
-
// overlap to that one abandoned call.
|
|
3337
|
+
// Supersede running continuations before starting another writer. Finished
|
|
3338
|
+
// outcomes stay in the delivery queue. Signal cancellation does not await
|
|
3339
|
+
// a tool call already executing in the superseded run.
|
|
2949
3340
|
this.abortDetachedRuns({ reason: "superseded_by_new_barge_in" });
|
|
2950
|
-
// An announcement turn is the one interruption with nothing to continue:
|
|
2951
|
-
// the work is already finished, which is why detachInterruptedTurn skips it
|
|
2952
|
-
// as `announcement_turn`. The analog of "the interrupted work survives" is
|
|
2953
|
-
// therefore the stash — the answer goes back into it so the user's next real
|
|
2954
|
-
// turn still carries it, and the announcement (cut short, at best partly
|
|
2955
|
-
// spoken) is not the delivery it was supposed to be. The restore sits after
|
|
2956
|
-
// the abort above, which clears the stash, so nothing in this sequence
|
|
2957
|
-
// re-drops it. Only a barge-in restores: interrupt/close/a superseding new
|
|
2958
|
-
// continuation are resets the user asked for, and their answer stays gone.
|
|
2959
|
-
if (turn.continuationDelivery !== null) {
|
|
2960
|
-
this.pendingContinuationResult = turn.continuationDelivery.answer;
|
|
2961
|
-
}
|
|
2962
3341
|
// Order this barge-in among concurrent detaches SYNCHRONOUSLY, in barge
|
|
2963
3342
|
// order — the actual detach runs after an async teardown chain, and those
|
|
2964
3343
|
// chains can interleave, so bumping there could assign sequences out of
|
|
2965
3344
|
// order and let an older continuation re-stash. A higher sequence here
|
|
2966
3345
|
// immediately invalidates every earlier still-running continuation.
|
|
2967
3346
|
const detachSeq = ++this.detachSequence;
|
|
3347
|
+
this.openBargeInInterruption(detachSeq);
|
|
3348
|
+
if (keepForegroundTaskOnParent) {
|
|
3349
|
+
this.judgeSuspendedForegroundTask(detachSeq);
|
|
3350
|
+
}
|
|
2968
3351
|
this.clearFillerTimers(turn);
|
|
2969
3352
|
// Tagged reason: provider catch-sites classify untagged caller aborts as
|
|
2970
3353
|
// retryable transport failures (ERROR log + futile retry against the
|
|
@@ -3052,14 +3435,11 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3052
3435
|
}
|
|
3053
3436
|
const skipReason = this.isClosed
|
|
3054
3437
|
? "session_closed"
|
|
3055
|
-
: //
|
|
3056
|
-
|
|
3057
|
-
// request behind an announcement turn to continue. Its answer is
|
|
3058
|
-
// already finished, so bargeIn returns it to the stash instead.
|
|
3059
|
-
turn.continuationDelivery !== null
|
|
3438
|
+
: // Announcement work is finished; only its delivery remains pending.
|
|
3439
|
+
turn.taskOutcome !== null
|
|
3060
3440
|
? "announcement_turn"
|
|
3061
|
-
: //
|
|
3062
|
-
//
|
|
3441
|
+
: // Look follow-ups use a transient view; a barge-in leaves any
|
|
3442
|
+
// unfinished host work with the parent conversation.
|
|
3063
3443
|
turn.lookFollowUp !== null
|
|
3064
3444
|
? "look_follow_up"
|
|
3065
3445
|
: // The model already finished generating (barge-in during TTS playback
|
|
@@ -3100,6 +3480,13 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3100
3480
|
interruptedRequest,
|
|
3101
3481
|
AbortSignal.any([controller.signal, labelAbort.signal]),
|
|
3102
3482
|
);
|
|
3483
|
+
// Judge, in parallel with the teardown wait and the label, whether the
|
|
3484
|
+
// caller's interruption still wants this request finished.
|
|
3485
|
+
const judgementPromise = this.requestContinuationJudgement(
|
|
3486
|
+
interruptedRequest,
|
|
3487
|
+
detachSeq,
|
|
3488
|
+
AbortSignal.any([controller.signal, labelAbort.signal]),
|
|
3489
|
+
);
|
|
3103
3490
|
void (async () => {
|
|
3104
3491
|
try {
|
|
3105
3492
|
// Wait for the interrupted turn's teardown to settle its partial into
|
|
@@ -3141,6 +3528,29 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3141
3528
|
return;
|
|
3142
3529
|
}
|
|
3143
3530
|
}
|
|
3531
|
+
const judgement = await judgementPromise;
|
|
3532
|
+
if (judgement !== null && judgement.outcome !== "unavailable") {
|
|
3533
|
+
log.info(
|
|
3534
|
+
{
|
|
3535
|
+
turnId: turn.turnId,
|
|
3536
|
+
outcome: judgement.outcome,
|
|
3537
|
+
noul: judgement.noul,
|
|
3538
|
+
latencyMs: judgement.latencyMs,
|
|
3539
|
+
},
|
|
3540
|
+
"Voice duplex continuation judged",
|
|
3541
|
+
);
|
|
3542
|
+
}
|
|
3543
|
+
if (judgement !== null && !judgement.keep) {
|
|
3544
|
+
log.info(
|
|
3545
|
+
{
|
|
3546
|
+
turnId: turn.turnId,
|
|
3547
|
+
skipReason: "caller_moved_on",
|
|
3548
|
+
waitedMs: Date.now() - detachStartedAtMs,
|
|
3549
|
+
},
|
|
3550
|
+
"Voice duplex continuation skipped",
|
|
3551
|
+
);
|
|
3552
|
+
return;
|
|
3553
|
+
}
|
|
3144
3554
|
// A closed session is NOT a reason to skip: the work outlives the
|
|
3145
3555
|
// call, and its result is delivered into the conversation below.
|
|
3146
3556
|
// Only an explicit invalidation (stop/interrupt/supersede) stops it.
|
|
@@ -3192,48 +3602,40 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3192
3602
|
!controller.signal.aborted &&
|
|
3193
3603
|
this.detachStopGeneration === stopGeneration &&
|
|
3194
3604
|
detachSeq === this.detachSequence;
|
|
3195
|
-
|
|
3196
|
-
|
|
3197
|
-
|
|
3198
|
-
|
|
3199
|
-
|
|
3200
|
-
|
|
3201
|
-
|
|
3202
|
-
|
|
3203
|
-
|
|
3204
|
-
|
|
3205
|
-
|
|
3206
|
-
|
|
3207
|
-
|
|
3208
|
-
|
|
3605
|
+
let accepted = false;
|
|
3606
|
+
if (notInvalidated && answer.length > 0) {
|
|
3607
|
+
const outcome: VoiceTaskOutcome = {
|
|
3608
|
+
taskId: `voice-continuation:${turn.turnId}`,
|
|
3609
|
+
message: buildContinuationResult(interruptedRequest, answer),
|
|
3610
|
+
metadata: {},
|
|
3611
|
+
source: "continuation",
|
|
3612
|
+
};
|
|
3613
|
+
accepted = this.receiveTaskOutcome(outcome);
|
|
3614
|
+
if (!accepted) {
|
|
3615
|
+
await Promise.allSettled(this.pendingAssistantStarts);
|
|
3616
|
+
await this.getTurnTeardown?.(this.conversationId);
|
|
3617
|
+
const { injectMessageIntoParent } =
|
|
3618
|
+
await import("../subagent/notify.js");
|
|
3619
|
+
injectMessageIntoParent(
|
|
3620
|
+
this.conversationId,
|
|
3621
|
+
outcome.message,
|
|
3622
|
+
outcome.metadata,
|
|
3623
|
+
{ bypassLiveVoice: true },
|
|
3624
|
+
);
|
|
3209
3625
|
}
|
|
3210
3626
|
}
|
|
3211
|
-
if (deliverToConversation) {
|
|
3212
|
-
const { injectMessageIntoParent } =
|
|
3213
|
-
await import("../subagent/notify.js");
|
|
3214
|
-
injectMessageIntoParent(
|
|
3215
|
-
this.conversationId,
|
|
3216
|
-
buildClosedSessionDeliveryPrompt(interruptedRequest, answer),
|
|
3217
|
-
);
|
|
3218
|
-
}
|
|
3219
3627
|
log.info(
|
|
3220
3628
|
{
|
|
3221
3629
|
turnId: turn.turnId,
|
|
3222
3630
|
ranMs: Date.now() - detachStartedAtMs,
|
|
3223
3631
|
resultChars: answer.length,
|
|
3224
|
-
// Where the answer went. "stashed" = queued for the next voice
|
|
3225
|
-
// turn and for an announcement into silence, whichever comes first
|
|
3226
|
-
// (the announcement logs "announced" when it wins);
|
|
3227
|
-
// "conversation" = the call had ended, so it was delivered into the
|
|
3228
|
-
// thread; "dropped" = a stop/interrupt or a newer barge-in landed
|
|
3229
|
-
// while it ran.
|
|
3230
3632
|
resultDestination: !notInvalidated
|
|
3231
3633
|
? "dropped"
|
|
3232
3634
|
: answer.length === 0
|
|
3233
3635
|
? "empty"
|
|
3234
|
-
:
|
|
3235
|
-
? "
|
|
3236
|
-
: "
|
|
3636
|
+
: accepted
|
|
3637
|
+
? "queued"
|
|
3638
|
+
: "conversation",
|
|
3237
3639
|
},
|
|
3238
3640
|
"Voice duplex continuation finished",
|
|
3239
3641
|
);
|
|
@@ -3259,6 +3661,105 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3259
3661
|
})();
|
|
3260
3662
|
}
|
|
3261
3663
|
|
|
3664
|
+
// Opens the wait for this barge-in's interrupting words, settling any older
|
|
3665
|
+
// barge-in's wait with null: its continuation was already superseded.
|
|
3666
|
+
private openBargeInInterruption(detachSeq: number): void {
|
|
3667
|
+
this.bargeInInterruption?.settle(null);
|
|
3668
|
+
let settle!: (words: string | null) => void;
|
|
3669
|
+
const promise = new Promise<string | null>((resolve) => {
|
|
3670
|
+
let settled = false;
|
|
3671
|
+
const timer = setTimeout(() => {
|
|
3672
|
+
finish(null);
|
|
3673
|
+
}, BARGE_IN_INTERRUPTION_WAIT_MS);
|
|
3674
|
+
const finish = (words: string | null): void => {
|
|
3675
|
+
if (settled) {
|
|
3676
|
+
return;
|
|
3677
|
+
}
|
|
3678
|
+
settled = true;
|
|
3679
|
+
clearTimeout(timer);
|
|
3680
|
+
if (this.bargeInInterruption?.detachSeq === detachSeq) {
|
|
3681
|
+
this.bargeInInterruption = null;
|
|
3682
|
+
}
|
|
3683
|
+
resolve(words);
|
|
3684
|
+
};
|
|
3685
|
+
settle = finish;
|
|
3686
|
+
});
|
|
3687
|
+
this.bargeInInterruption = { detachSeq, promise, settle };
|
|
3688
|
+
}
|
|
3689
|
+
|
|
3690
|
+
// Asks the continuation judge whether the foreground task this barge-in
|
|
3691
|
+
// suspended should still be finished. A drop clears the task, so the
|
|
3692
|
+
// automatic resume never fires; a keep leaves the resume as it was. The
|
|
3693
|
+
// interrupting turn may claim the task first, in which case it owns the
|
|
3694
|
+
// outcome and the verdict is only logged.
|
|
3695
|
+
private judgeSuspendedForegroundTask(detachSeq: number): void {
|
|
3696
|
+
const task = this.foregroundTaskState;
|
|
3697
|
+
if (task?.phase !== "suspended" || !this.judgeBackgroundContinuation) {
|
|
3698
|
+
return;
|
|
3699
|
+
}
|
|
3700
|
+
const epoch = task.epoch;
|
|
3701
|
+
const entry: { epoch: number; settled: Promise<void> } = {
|
|
3702
|
+
epoch,
|
|
3703
|
+
settled: Promise.resolve(),
|
|
3704
|
+
};
|
|
3705
|
+
entry.settled = this.requestContinuationJudgement(
|
|
3706
|
+
task.request,
|
|
3707
|
+
detachSeq,
|
|
3708
|
+
new AbortController().signal,
|
|
3709
|
+
).then((judgement) => {
|
|
3710
|
+
if (this.foregroundTaskJudgement === entry) {
|
|
3711
|
+
this.foregroundTaskJudgement = null;
|
|
3712
|
+
}
|
|
3713
|
+
if (judgement === null || judgement.outcome === "unavailable") {
|
|
3714
|
+
return;
|
|
3715
|
+
}
|
|
3716
|
+
log.info(
|
|
3717
|
+
{
|
|
3718
|
+
epoch,
|
|
3719
|
+
outcome: judgement.outcome,
|
|
3720
|
+
noul: judgement.noul,
|
|
3721
|
+
latencyMs: judgement.latencyMs,
|
|
3722
|
+
},
|
|
3723
|
+
"Voice foreground task judged",
|
|
3724
|
+
);
|
|
3725
|
+
const current = this.foregroundTaskState;
|
|
3726
|
+
if (
|
|
3727
|
+
!judgement.keep &&
|
|
3728
|
+
current?.phase === "suspended" &&
|
|
3729
|
+
current.epoch === epoch
|
|
3730
|
+
) {
|
|
3731
|
+
this.clearForegroundTask("caller_moved_on");
|
|
3732
|
+
}
|
|
3733
|
+
});
|
|
3734
|
+
this.foregroundTaskJudgement = entry;
|
|
3735
|
+
}
|
|
3736
|
+
|
|
3737
|
+
// The continuation judge's verdict for one barge-in, or null when no judge
|
|
3738
|
+
// is wired. The judge only waits on the interruption when it can run.
|
|
3739
|
+
private async requestContinuationJudgement(
|
|
3740
|
+
interruptedRequest: string,
|
|
3741
|
+
detachSeq: number,
|
|
3742
|
+
signal: AbortSignal,
|
|
3743
|
+
): Promise<ContinuationJudgement | null> {
|
|
3744
|
+
const judge = this.judgeBackgroundContinuation;
|
|
3745
|
+
if (!judge) {
|
|
3746
|
+
return null;
|
|
3747
|
+
}
|
|
3748
|
+
const waiter = this.bargeInInterruption;
|
|
3749
|
+
const interruption =
|
|
3750
|
+
waiter?.detachSeq === detachSeq ? waiter.promise : Promise.resolve(null);
|
|
3751
|
+
try {
|
|
3752
|
+
return await judge({
|
|
3753
|
+
parentConversationId: this.conversationId,
|
|
3754
|
+
interruptedRequest,
|
|
3755
|
+
interruption,
|
|
3756
|
+
signal,
|
|
3757
|
+
});
|
|
3758
|
+
} catch {
|
|
3759
|
+
return null;
|
|
3760
|
+
}
|
|
3761
|
+
}
|
|
3762
|
+
|
|
3262
3763
|
// Resolves the model-phrased label for a continuation, or null when there
|
|
3263
3764
|
// is nothing to name it from, no labeler is wired, or the call missed
|
|
3264
3765
|
// (timeout, abort, provider error, model declined). Never rejects: the
|
|
@@ -3294,7 +3795,12 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3294
3795
|
const currentRequest = turn.utterance.finalTranscriptSegments
|
|
3295
3796
|
.join(" ")
|
|
3296
3797
|
.trim();
|
|
3297
|
-
return
|
|
3798
|
+
return (
|
|
3799
|
+
currentRequest ||
|
|
3800
|
+
turn.lookFollowUp?.callerUtterance ||
|
|
3801
|
+
turn.interruptedRequest ||
|
|
3802
|
+
""
|
|
3803
|
+
);
|
|
3298
3804
|
}
|
|
3299
3805
|
|
|
3300
3806
|
private createForegroundTaskOwnership(
|
|
@@ -3315,6 +3821,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3315
3821
|
|
|
3316
3822
|
private noteEscalatedForegroundTask(turn: ActiveAssistantTurn): void {
|
|
3317
3823
|
if (
|
|
3824
|
+
turn.taskOutcome !== null ||
|
|
3318
3825
|
this.activeAssistantTurn?.token !== turn.token ||
|
|
3319
3826
|
turn.discardRequested ||
|
|
3320
3827
|
turn.abortController.signal.aborted
|
|
@@ -3339,6 +3846,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3339
3846
|
turn: ActiveAssistantTurn,
|
|
3340
3847
|
effectiveToolName: string,
|
|
3341
3848
|
): void {
|
|
3849
|
+
if (turn.taskOutcome !== null) {
|
|
3850
|
+
return;
|
|
3851
|
+
}
|
|
3342
3852
|
const current = this.foregroundTaskState;
|
|
3343
3853
|
if (current?.phase === "owned" && current.ownerToken === turn.token) {
|
|
3344
3854
|
if (!current.hostToolStarted) {
|
|
@@ -3608,6 +4118,11 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3608
4118
|
private async resumeForegroundTask(
|
|
3609
4119
|
task: SuspendedForegroundTask,
|
|
3610
4120
|
): Promise<void> {
|
|
4121
|
+
// A verdict still out on this task decides whether it resumes at all.
|
|
4122
|
+
const judgement = this.foregroundTaskJudgement;
|
|
4123
|
+
if (judgement?.epoch === task.epoch) {
|
|
4124
|
+
await judgement.settled;
|
|
4125
|
+
}
|
|
3611
4126
|
const current = this.foregroundTaskState;
|
|
3612
4127
|
if (
|
|
3613
4128
|
current?.phase !== "suspended" ||
|
|
@@ -3635,178 +4150,139 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3635
4150
|
}
|
|
3636
4151
|
|
|
3637
4152
|
// Abort every background continuation this session started and drop its
|
|
3638
|
-
// handle.
|
|
3639
|
-
// newer barge-in supersedes the detached runs, and on a foreground-wins
|
|
3640
|
-
// abort; the continuation's own `.finally` removes it from the set too.
|
|
3641
|
-
// `keepPendingResult` is the foreground-wins variant: the foreground turn is
|
|
3642
|
-
// claiming the workspace, so running continuations must die (and pending
|
|
3643
|
-
// detaches must be skipped), but an already-completed continuation's stashed
|
|
3644
|
-
// answer stays — it cannot race anything, and the next turn's "use only if
|
|
3645
|
-
// relevant" framing makes a stale one harmless.
|
|
4153
|
+
// handle. Finished outcomes have independent delivery state.
|
|
3646
4154
|
private abortDetachedRuns(opts?: {
|
|
3647
|
-
keepPendingResult?: boolean;
|
|
3648
|
-
// Why the runs are being invalidated, for the log line below. Every
|
|
3649
|
-
// caller passes one: an unexplained dead continuation is the single
|
|
3650
|
-
// hardest thing to debug about this feature.
|
|
3651
4155
|
reason?: string;
|
|
3652
|
-
// The foreground tool whose start tripped the contention gate, if any.
|
|
3653
4156
|
toolName?: string;
|
|
3654
4157
|
}): void {
|
|
3655
4158
|
const aborted = this.detachControllers.size;
|
|
3656
|
-
|
|
3657
|
-
if (aborted > 0 || hadPendingResult) {
|
|
4159
|
+
if (aborted > 0) {
|
|
3658
4160
|
log.debug(
|
|
3659
4161
|
{
|
|
3660
4162
|
conversationId: this.conversationId,
|
|
3661
4163
|
reason: opts?.reason ?? "unspecified",
|
|
3662
4164
|
...(opts?.toolName ? { toolName: opts.toolName } : {}),
|
|
3663
4165
|
abortedRuns: aborted,
|
|
3664
|
-
droppedPendingResult: hadPendingResult && !opts?.keepPendingResult,
|
|
3665
4166
|
},
|
|
3666
4167
|
"Voice duplex continuations invalidated",
|
|
3667
4168
|
);
|
|
3668
4169
|
}
|
|
3669
|
-
// Bump the generation so a barge-in whose async teardown is still in flight
|
|
3670
|
-
// (its detach not yet spawned) sees the stop and skips the continuation.
|
|
3671
4170
|
this.detachStopGeneration += 1;
|
|
3672
4171
|
for (const controller of this.detachControllers) {
|
|
3673
4172
|
controller.abort();
|
|
3674
4173
|
}
|
|
3675
4174
|
this.detachControllers.clear();
|
|
3676
|
-
|
|
3677
|
-
|
|
3678
|
-
|
|
3679
|
-
|
|
4175
|
+
}
|
|
4176
|
+
|
|
4177
|
+
receiveSubagentNotification(
|
|
4178
|
+
notification: SubagentParentNotification,
|
|
4179
|
+
): boolean {
|
|
4180
|
+
return this.receiveTaskOutcome({ ...notification, source: "subagent" });
|
|
4181
|
+
}
|
|
4182
|
+
|
|
4183
|
+
private receiveTaskOutcome(notification: VoiceTaskOutcome): boolean {
|
|
4184
|
+
if (this.isClosed && this.taskOutcomesClosing) {
|
|
4185
|
+
this.taskOutcomes.enqueue(notification);
|
|
4186
|
+
return true;
|
|
4187
|
+
}
|
|
4188
|
+
if (this.isClosed || this.state === "failed" || !this.startVoiceTurn) {
|
|
4189
|
+
return false;
|
|
3680
4190
|
}
|
|
3681
|
-
|
|
3682
|
-
|
|
3683
|
-
|
|
3684
|
-
this.
|
|
4191
|
+
this.taskOutcomes.acknowledgePlayback(Date.now());
|
|
4192
|
+
this.taskOutcomes.enqueue(notification);
|
|
4193
|
+
this.taskAnnouncementsDeferred = false;
|
|
4194
|
+
this.scheduleTaskAnnouncement();
|
|
4195
|
+
return true;
|
|
3685
4196
|
}
|
|
3686
4197
|
|
|
3687
|
-
private
|
|
3688
|
-
this.
|
|
3689
|
-
|
|
3690
|
-
|
|
3691
|
-
this.announcementTimer = null;
|
|
4198
|
+
private clearTaskAnnouncementTimer(): void {
|
|
4199
|
+
if (this.taskAnnouncementTimer !== null) {
|
|
4200
|
+
clearTimeout(this.taskAnnouncementTimer);
|
|
4201
|
+
this.taskAnnouncementTimer = null;
|
|
3692
4202
|
}
|
|
3693
4203
|
}
|
|
3694
4204
|
|
|
3695
|
-
|
|
3696
|
-
|
|
3697
|
-
|
|
3698
|
-
|
|
3699
|
-
|
|
3700
|
-
|
|
3701
|
-
|
|
3702
|
-
|
|
3703
|
-
|
|
3704
|
-
|
|
3705
|
-
|
|
3706
|
-
|
|
3707
|
-
|
|
3708
|
-
|
|
3709
|
-
|
|
3710
|
-
|
|
3711
|
-
|
|
3712
|
-
|
|
3713
|
-
|
|
3714
|
-
|
|
3715
|
-
|
|
3716
|
-
|
|
3717
|
-
|
|
3718
|
-
|
|
4205
|
+
private scheduleTaskAnnouncement(): void {
|
|
4206
|
+
this.taskOutcomes.acknowledgePlayback(Date.now());
|
|
4207
|
+
if (
|
|
4208
|
+
!this.taskOutcomes.hasPending ||
|
|
4209
|
+
this.taskAnnouncementsDeferred ||
|
|
4210
|
+
this.state === "initializing" ||
|
|
4211
|
+
this.isClosed ||
|
|
4212
|
+
this.state === "failed" ||
|
|
4213
|
+
this.activeAssistantTurn !== null
|
|
4214
|
+
) {
|
|
4215
|
+
this.clearTaskAnnouncementTimer();
|
|
4216
|
+
return;
|
|
4217
|
+
}
|
|
4218
|
+
this.taskAnnouncementTimer = this.scheduleFloorCheck(
|
|
4219
|
+
this.taskAnnouncementTimer,
|
|
4220
|
+
this.continuationAnnounceSilenceMs,
|
|
4221
|
+
(blockedBy) => {
|
|
4222
|
+
this.taskAnnouncementTimer = null;
|
|
4223
|
+
this.taskOutcomes.acknowledgePlayback(Date.now());
|
|
4224
|
+
if (blockedBy === null) {
|
|
4225
|
+
void this.announceTaskOutcome();
|
|
4226
|
+
} else if (
|
|
4227
|
+
blockedBy !== "turn_active" &&
|
|
4228
|
+
blockedBy !== "session_unavailable"
|
|
4229
|
+
) {
|
|
4230
|
+
this.scheduleTaskAnnouncement();
|
|
4231
|
+
}
|
|
4232
|
+
},
|
|
4233
|
+
);
|
|
4234
|
+
}
|
|
4235
|
+
|
|
4236
|
+
private async announceTaskOutcome(): Promise<void> {
|
|
4237
|
+
const notification = this.taskOutcomes.next();
|
|
4238
|
+
if (notification === undefined) {
|
|
3719
4239
|
return;
|
|
3720
4240
|
}
|
|
4241
|
+
let started = false;
|
|
3721
4242
|
try {
|
|
3722
|
-
|
|
3723
|
-
|
|
3724
|
-
|
|
3725
|
-
|
|
3726
|
-
|
|
3727
|
-
log.info(
|
|
4243
|
+
started = await this.launchAssistantTurn(
|
|
4244
|
+
createSyntheticUtterance(),
|
|
4245
|
+
notification.source === "continuation"
|
|
4246
|
+
? CONTINUATION_DELIVERY_CONTENT
|
|
4247
|
+
: notification.message,
|
|
3728
4248
|
{
|
|
3729
|
-
|
|
3730
|
-
|
|
3731
|
-
|
|
4249
|
+
taskOutcome: notification,
|
|
4250
|
+
hiddenPrompt: true,
|
|
4251
|
+
initialLeg: "escalated",
|
|
3732
4252
|
},
|
|
3733
|
-
"Voice duplex continuation delivered on session close",
|
|
3734
4253
|
);
|
|
3735
4254
|
} catch (err) {
|
|
3736
4255
|
log.warn(
|
|
3737
|
-
{ err,
|
|
3738
|
-
"Voice
|
|
4256
|
+
{ err, taskId: notification.taskId },
|
|
4257
|
+
"Voice task announcement failed to start",
|
|
3739
4258
|
);
|
|
3740
4259
|
}
|
|
4260
|
+
if (!started) {
|
|
4261
|
+
// A failed launch waits for the next user turn or update before retrying.
|
|
4262
|
+
this.taskAnnouncementsDeferred = true;
|
|
4263
|
+
this.clearTaskAnnouncementTimer();
|
|
4264
|
+
}
|
|
3741
4265
|
}
|
|
3742
4266
|
|
|
3743
|
-
|
|
3744
|
-
|
|
3745
|
-
|
|
3746
|
-
|
|
3747
|
-
|
|
3748
|
-
|
|
3749
|
-
|
|
3750
|
-
|
|
3751
|
-
|
|
3752
|
-
|
|
3753
|
-
|
|
3754
|
-
|
|
3755
|
-
|
|
3756
|
-
|
|
3757
|
-
|
|
3758
|
-
|
|
3759
|
-
|
|
3760
|
-
|
|
3761
|
-
|
|
3762
|
-
): void {
|
|
3763
|
-
this.announcementTimer = this.scheduleFloorCheck(
|
|
3764
|
-
this.announcementTimer,
|
|
3765
|
-
this.continuationAnnounceSilenceMs,
|
|
3766
|
-
() => {
|
|
3767
|
-
this.announcementTimer = null;
|
|
3768
|
-
const blockedBy = this.continuationAnnouncementBlocker();
|
|
3769
|
-
if (blockedBy === null) {
|
|
3770
|
-
void this.announceContinuation().catch((err: unknown) => {
|
|
3771
|
-
log.warn({ err }, "Voice duplex continuation announcement failed");
|
|
3772
|
-
});
|
|
3773
|
-
return;
|
|
3774
|
-
}
|
|
3775
|
-
log.debug(
|
|
3776
|
-
{
|
|
3777
|
-
conversationId: this.conversationId,
|
|
3778
|
-
blockedBy,
|
|
3779
|
-
retry,
|
|
3780
|
-
drainRearms,
|
|
3781
|
-
},
|
|
3782
|
-
"Voice duplex continuation announcement deferred",
|
|
3783
|
-
);
|
|
3784
|
-
// A tail that grew while the timer ran (the reply was still streaming
|
|
3785
|
-
// TTS when it was armed) is a wait of known length rather than a busy
|
|
3786
|
-
// call, so it re-arms against the new deadline without spending the
|
|
3787
|
-
// single retry the busy-call blockers get. Otherwise a long reply's
|
|
3788
|
-
// playback alone outlasts the budget and the announcement is dropped
|
|
3789
|
-
// even though the call goes idle right after. The tail only extends
|
|
3790
|
-
// while a turn is active, which blocks earlier, so the re-arms converge;
|
|
3791
|
-
// the cap bounds the interleaving where a fresh turn's audio lands
|
|
3792
|
-
// between two checks.
|
|
3793
|
-
if (
|
|
3794
|
-
blockedBy === "playback_draining" &&
|
|
3795
|
-
drainRearms < CONTINUATION_ANNOUNCE_MAX_DRAIN_REARMS
|
|
3796
|
-
) {
|
|
3797
|
-
this.scheduleContinuationAnnouncement(retry, drainRearms + 1);
|
|
3798
|
-
return;
|
|
3799
|
-
}
|
|
3800
|
-
if (blockedBy === "turn_active") {
|
|
3801
|
-
return;
|
|
3802
|
-
}
|
|
3803
|
-
if (retry || this.pendingAnnouncement === null) {
|
|
3804
|
-
this.pendingAnnouncement = null;
|
|
3805
|
-
return;
|
|
3806
|
-
}
|
|
3807
|
-
this.scheduleContinuationAnnouncement(true, drainRearms);
|
|
3808
|
-
},
|
|
3809
|
-
);
|
|
4267
|
+
private async deliverTaskOutcomesToConversation(
|
|
4268
|
+
turnTeardown?: Promise<void>,
|
|
4269
|
+
): Promise<void> {
|
|
4270
|
+
this.clearTaskAnnouncementTimer();
|
|
4271
|
+
// Voice aborts discard the parent queue. Deliver only after its turn settles.
|
|
4272
|
+
await Promise.allSettled(this.pendingAssistantStarts);
|
|
4273
|
+
await turnTeardown;
|
|
4274
|
+
await this.getTurnTeardown?.(this.conversationId);
|
|
4275
|
+
const { injectMessageIntoParent } = await import("../subagent/notify.js");
|
|
4276
|
+
const notifications = this.taskOutcomes.drain(Date.now());
|
|
4277
|
+
this.taskOutcomesClosing = false;
|
|
4278
|
+
for (const notification of notifications) {
|
|
4279
|
+
injectMessageIntoParent(
|
|
4280
|
+
this.conversationId,
|
|
4281
|
+
notification.message,
|
|
4282
|
+
notification.metadata,
|
|
4283
|
+
{ cronRunId: notification.cronRunId, bypassLiveVoice: true },
|
|
4284
|
+
);
|
|
4285
|
+
}
|
|
3810
4286
|
}
|
|
3811
4287
|
|
|
3812
4288
|
/**
|
|
@@ -3853,9 +4329,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3853
4329
|
}
|
|
3854
4330
|
|
|
3855
4331
|
/**
|
|
3856
|
-
* Publish whatever the turn's activity
|
|
3857
|
-
*
|
|
3858
|
-
*
|
|
4332
|
+
* Publish whatever the turn's activity should be right now: the decision it
|
|
4333
|
+
* is waiting on, its newest running tool, or the structured escalation state
|
|
4334
|
+
* underneath those temporary overlays.
|
|
3859
4335
|
*
|
|
3860
4336
|
* The single entry point for every caller that would otherwise reach for
|
|
3861
4337
|
* `currentActivityLabel` directly. A turn can start and finish other ops
|
|
@@ -3873,7 +4349,18 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3873
4349
|
);
|
|
3874
4350
|
return;
|
|
3875
4351
|
}
|
|
3876
|
-
|
|
4352
|
+
const label = this.currentActivityLabel(turn);
|
|
4353
|
+
if (
|
|
4354
|
+
label.length === 0 &&
|
|
4355
|
+
turn.escalationTarget !== null &&
|
|
4356
|
+
!turn.assistantCompleted
|
|
4357
|
+
) {
|
|
4358
|
+
if (turn.publishedActivityKind !== "escalation") {
|
|
4359
|
+
this.publishEscalationActivity(turn, turn.escalationTarget);
|
|
4360
|
+
}
|
|
4361
|
+
return;
|
|
4362
|
+
}
|
|
4363
|
+
this.publishActivity(turn, label);
|
|
3877
4364
|
}
|
|
3878
4365
|
|
|
3879
4366
|
/**
|
|
@@ -3951,14 +4438,17 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3951
4438
|
* conversation may wait on one.
|
|
3952
4439
|
*
|
|
3953
4440
|
* Callers whose line depends on turn state go through
|
|
3954
|
-
* {@link refreshActivity}
|
|
3955
|
-
*
|
|
3956
|
-
* was waiting on.
|
|
4441
|
+
* {@link refreshActivity}. Structured transition events call this directly
|
|
4442
|
+
* because their meaning is not captured by the label alone.
|
|
3957
4443
|
*/
|
|
3958
4444
|
private publishActivity(
|
|
3959
4445
|
turn: ActiveAssistantTurn,
|
|
3960
4446
|
label: string,
|
|
3961
4447
|
approvalRequestId?: string,
|
|
4448
|
+
detail?: Pick<
|
|
4449
|
+
Extract<LiveVoiceServerFramePayload, { type: "activity" }>,
|
|
4450
|
+
"kind" | "profile" | "profileSource"
|
|
4451
|
+
>,
|
|
3962
4452
|
): void {
|
|
3963
4453
|
// De-duplicated on the request id as well as the wording. The two move
|
|
3964
4454
|
// independently: a wait can be entered and left without the tool line
|
|
@@ -3966,6 +4456,8 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3966
4456
|
// retires the approval — leaving the island's buttons up with nothing
|
|
3967
4457
|
// behind them.
|
|
3968
4458
|
if (
|
|
4459
|
+
detail === undefined &&
|
|
4460
|
+
turn.publishedActivityKind === null &&
|
|
3969
4461
|
turn.activityLabel === label &&
|
|
3970
4462
|
turn.publishedApprovalRequestId === (approvalRequestId ?? null)
|
|
3971
4463
|
) {
|
|
@@ -3973,29 +4465,36 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3973
4465
|
}
|
|
3974
4466
|
turn.activityLabel = label;
|
|
3975
4467
|
turn.publishedApprovalRequestId = approvalRequestId ?? null;
|
|
4468
|
+
turn.publishedActivityKind = detail?.kind ?? null;
|
|
3976
4469
|
void this.sendFrame(
|
|
3977
4470
|
{
|
|
3978
4471
|
type: "activity",
|
|
3979
4472
|
turnId: turn.turnId,
|
|
3980
4473
|
label,
|
|
3981
4474
|
...(approvalRequestId !== undefined ? { approvalRequestId } : {}),
|
|
4475
|
+
...detail,
|
|
3982
4476
|
},
|
|
3983
4477
|
() => !this.isClosed,
|
|
3984
4478
|
);
|
|
3985
4479
|
}
|
|
3986
4480
|
|
|
4481
|
+
private publishEscalationActivity(
|
|
4482
|
+
turn: ActiveAssistantTurn,
|
|
4483
|
+
target: VoiceEscalationTarget,
|
|
4484
|
+
): void {
|
|
4485
|
+
this.publishActivity(turn, "", undefined, {
|
|
4486
|
+
kind: "escalation",
|
|
4487
|
+
profile: target.profile,
|
|
4488
|
+
profileSource: target.source,
|
|
4489
|
+
});
|
|
4490
|
+
}
|
|
4491
|
+
|
|
3987
4492
|
private clearActiveAssistantTurn(token: symbol): void {
|
|
3988
4493
|
if (this.activeAssistantTurn?.token !== token) {
|
|
3989
4494
|
return;
|
|
3990
4495
|
}
|
|
3991
4496
|
this.activeAssistantTurn = null;
|
|
3992
|
-
|
|
3993
|
-
this.pendingAnnouncement !== null &&
|
|
3994
|
-
!this.isClosed &&
|
|
3995
|
-
this.state !== "failed"
|
|
3996
|
-
) {
|
|
3997
|
-
this.scheduleContinuationAnnouncement();
|
|
3998
|
-
}
|
|
4497
|
+
this.scheduleTaskAnnouncement();
|
|
3999
4498
|
if (
|
|
4000
4499
|
this.foregroundTaskState?.phase === "suspended" &&
|
|
4001
4500
|
this.foregroundTaskState.resumePending &&
|
|
@@ -4006,18 +4505,6 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
4006
4505
|
}
|
|
4007
4506
|
}
|
|
4008
4507
|
|
|
4009
|
-
// Why the session must not speak up right now, or null when the call is
|
|
4010
|
-
// genuinely idle: live, no turn running, and no user speech anywhere in
|
|
4011
|
-
// flight (onset latch, barge-in guard, detector, or a cycle that has already
|
|
4012
|
-
// captured something). The server-VAD re-arm keeps an empty cycle armed
|
|
4013
|
-
// between turns, so an untouched `currentUtterance` IS the idle state.
|
|
4014
|
-
private continuationAnnouncementBlocker(): string | null {
|
|
4015
|
-
if (this.pendingAnnouncement === null) {
|
|
4016
|
-
return "nothing_pending";
|
|
4017
|
-
}
|
|
4018
|
-
return this.sessionTurnFloorBlocker();
|
|
4019
|
-
}
|
|
4020
|
-
|
|
4021
4508
|
/**
|
|
4022
4509
|
* Why the session may not start a turn of its own right now, or null when it
|
|
4023
4510
|
* may.
|
|
@@ -4101,7 +4588,10 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
4101
4588
|
* that would not open, a share the desktop refused) cannot turn up minutes
|
|
4102
4589
|
* later as a reply to nothing.
|
|
4103
4590
|
*/
|
|
4104
|
-
private awaitLookFrame(
|
|
4591
|
+
private awaitLookFrame(
|
|
4592
|
+
action: LookSessionControl,
|
|
4593
|
+
turn: ActiveAssistantTurn,
|
|
4594
|
+
): void {
|
|
4105
4595
|
if (!this.lookFrames) {
|
|
4106
4596
|
return;
|
|
4107
4597
|
}
|
|
@@ -4116,7 +4606,20 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
4116
4606
|
"Live voice look dropped: no frame arrived",
|
|
4117
4607
|
);
|
|
4118
4608
|
}, LOOK_FRAME_WAIT_MS);
|
|
4119
|
-
this.pendingLook = {
|
|
4609
|
+
this.pendingLook = {
|
|
4610
|
+
action,
|
|
4611
|
+
callerUtterance: this.foregroundTaskRequestForTurn(turn),
|
|
4612
|
+
armedAtMs: Date.now(),
|
|
4613
|
+
timer,
|
|
4614
|
+
frameLanded: false,
|
|
4615
|
+
};
|
|
4616
|
+
}
|
|
4617
|
+
|
|
4618
|
+
private appendPendingLookRequest(content: string): void {
|
|
4619
|
+
if (this.pendingLook !== null && content.trim().length > 0) {
|
|
4620
|
+
this.pendingLook.callerUtterance =
|
|
4621
|
+
`${this.pendingLook.callerUtterance}\n${content}`.trim();
|
|
4622
|
+
}
|
|
4120
4623
|
}
|
|
4121
4624
|
|
|
4122
4625
|
private clearPendingLook(): void {
|
|
@@ -4138,11 +4641,11 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
4138
4641
|
*/
|
|
4139
4642
|
private lookFrameLanded(): void {
|
|
4140
4643
|
const pending = this.pendingLook;
|
|
4141
|
-
if (pending === null || this.isClosed) {
|
|
4644
|
+
if (pending === null || pending.frameLanded || this.isClosed) {
|
|
4142
4645
|
return;
|
|
4143
4646
|
}
|
|
4144
4647
|
clearTimeout(pending.timer);
|
|
4145
|
-
|
|
4648
|
+
pending.frameLanded = true;
|
|
4146
4649
|
this.answerLookWhenFloorIsFree(pending, this.turnsLaunched, 0);
|
|
4147
4650
|
}
|
|
4148
4651
|
|
|
@@ -4163,11 +4666,14 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
4163
4666
|
turnsAtFrame: number,
|
|
4164
4667
|
rearms: number,
|
|
4165
4668
|
): void {
|
|
4669
|
+
if (this.pendingLook !== look) {
|
|
4670
|
+
return;
|
|
4671
|
+
}
|
|
4166
4672
|
if (this.lookFollowUpTimer !== null) {
|
|
4167
4673
|
clearTimeout(this.lookFollowUpTimer);
|
|
4168
4674
|
this.lookFollowUpTimer = null;
|
|
4169
4675
|
}
|
|
4170
|
-
const { action, armedAtMs } = look;
|
|
4676
|
+
const { action, callerUtterance, armedAtMs } = look;
|
|
4171
4677
|
// A turn launched since the frame landed read it already. Speech that
|
|
4172
4678
|
// never became a turn (a cough, noise that transcribed to nothing) is only
|
|
4173
4679
|
// waited out.
|
|
@@ -4176,10 +4682,11 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
4176
4682
|
? "turn_since_look"
|
|
4177
4683
|
: this.sessionTurnFloorBlocker();
|
|
4178
4684
|
if (blockedBy === null) {
|
|
4685
|
+
this.clearPendingLook();
|
|
4179
4686
|
void this.launchAssistantTurn(
|
|
4180
4687
|
createSyntheticUtterance(),
|
|
4181
4688
|
LOOK_FOLLOW_UP_CONTENT,
|
|
4182
|
-
{ lookFollowUp: action, hiddenPrompt: true },
|
|
4689
|
+
{ lookFollowUp: { action, callerUtterance }, hiddenPrompt: true },
|
|
4183
4690
|
).catch((err: unknown) => {
|
|
4184
4691
|
log.warn(
|
|
4185
4692
|
{ err, conversationId: this.conversationId, action },
|
|
@@ -4191,6 +4698,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
4191
4698
|
const waitable =
|
|
4192
4699
|
blockedBy !== "turn_since_look" && blockedBy !== "session_unavailable";
|
|
4193
4700
|
if (!waitable || Date.now() - armedAtMs >= LOOK_ANSWER_DEADLINE_MS) {
|
|
4701
|
+
this.clearPendingLook();
|
|
4194
4702
|
log.info(
|
|
4195
4703
|
{ conversationId: this.conversationId, action, blockedBy, rearms },
|
|
4196
4704
|
"Live voice look follow-up skipped",
|
|
@@ -4207,77 +4715,6 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
4207
4715
|
);
|
|
4208
4716
|
}
|
|
4209
4717
|
|
|
4210
|
-
// Speak the finished continuation's result on a turn the session starts
|
|
4211
|
-
// itself. The turn rides a synthetic cycle rather than the armed VAD one —
|
|
4212
|
-
// `currentUtterance` belongs to the capture loop, and overwriting it would
|
|
4213
|
-
// collide with the user's next real utterance.
|
|
4214
|
-
private async announceContinuation(): Promise<void> {
|
|
4215
|
-
const pending = this.pendingAnnouncement;
|
|
4216
|
-
if (!pending) {
|
|
4217
|
-
return;
|
|
4218
|
-
}
|
|
4219
|
-
this.pendingAnnouncement = null;
|
|
4220
|
-
// The two routes for one answer are mutually exclusive: while this turn is
|
|
4221
|
-
// in flight the stash must not deliver the same answer again. It comes back
|
|
4222
|
-
// only if this turn provably never started (below) or a barge-in cuts it
|
|
4223
|
-
// short before it delivers (see bargeIn).
|
|
4224
|
-
this.pendingContinuationResult = null;
|
|
4225
|
-
// Pins the invalidation state this hand-off was decided in: a hard stop
|
|
4226
|
-
// landing while the turn starts owns the drop, and must not be undone by
|
|
4227
|
-
// the restore below.
|
|
4228
|
-
const stopGeneration = this.detachStopGeneration;
|
|
4229
|
-
let started: boolean;
|
|
4230
|
-
try {
|
|
4231
|
-
started = await this.launchAssistantTurn(
|
|
4232
|
-
createSyntheticUtterance(),
|
|
4233
|
-
CONTINUATION_DELIVERY_CONTENT,
|
|
4234
|
-
{ continuationDelivery: pending },
|
|
4235
|
-
);
|
|
4236
|
-
} catch (err) {
|
|
4237
|
-
// A throw before the leg's own error handling (an outbound frame write
|
|
4238
|
-
// rejecting, say) leaves the same state a false return does: nothing
|
|
4239
|
-
// persisted, nothing spoken, and the stash already emptied above. Take
|
|
4240
|
-
// the identical fallback rather than letting the answer die with the
|
|
4241
|
-
// rejection.
|
|
4242
|
-
started = false;
|
|
4243
|
-
log.warn(
|
|
4244
|
-
{ err, conversationId: this.conversationId },
|
|
4245
|
-
"Voice duplex continuation announcement threw while starting",
|
|
4246
|
-
);
|
|
4247
|
-
}
|
|
4248
|
-
if (started) {
|
|
4249
|
-
log.info(
|
|
4250
|
-
{
|
|
4251
|
-
conversationId: this.conversationId,
|
|
4252
|
-
resultChars: pending.answer.length,
|
|
4253
|
-
resultDestination: "announced",
|
|
4254
|
-
},
|
|
4255
|
-
"Voice duplex continuation announced",
|
|
4256
|
-
);
|
|
4257
|
-
return;
|
|
4258
|
-
}
|
|
4259
|
-
// The turn never reached the bridge (the conversation is busy from another
|
|
4260
|
-
// surface, or the bridge threw), so nothing was persisted or spoken. Hand
|
|
4261
|
-
// the answer back to the stash — the user's next turn is the fallback route
|
|
4262
|
-
// that exists for exactly this. A newer continuation's answer already in
|
|
4263
|
-
// the stash wins over this older one rather than being overwritten.
|
|
4264
|
-
const restashed =
|
|
4265
|
-
!this.isClosed &&
|
|
4266
|
-
this.detachStopGeneration === stopGeneration &&
|
|
4267
|
-
this.pendingContinuationResult === null;
|
|
4268
|
-
if (restashed) {
|
|
4269
|
-
this.pendingContinuationResult = pending.answer;
|
|
4270
|
-
}
|
|
4271
|
-
log.warn(
|
|
4272
|
-
{
|
|
4273
|
-
conversationId: this.conversationId,
|
|
4274
|
-
resultChars: pending.answer.length,
|
|
4275
|
-
resultDestination: restashed ? "stashed" : "dropped",
|
|
4276
|
-
},
|
|
4277
|
-
"Voice duplex continuation announcement could not start",
|
|
4278
|
-
);
|
|
4279
|
-
}
|
|
4280
|
-
|
|
4281
4718
|
// VAD closed the utterance — the analog of ptt_release: emit
|
|
4282
4719
|
// utterance_end, then run the standard release path.
|
|
4283
4720
|
private handleVadUtteranceEnd(reason: "silence" | "max-duration"): void {
|
|
@@ -4294,6 +4731,14 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
4294
4731
|
this.vadSpeechStartPending = false;
|
|
4295
4732
|
// The detector turn is over: an untripped guard was noise, not
|
|
4296
4733
|
// barge-in — leave playback untouched.
|
|
4734
|
+
if (this.pendingBargeIn) {
|
|
4735
|
+
this.logInputDiagnostic("voice_input_barge_in_expired", {
|
|
4736
|
+
reason,
|
|
4737
|
+
armedAtMs: this.pendingBargeIn.armedAtMs,
|
|
4738
|
+
guardResets: this.pendingBargeIn.resets,
|
|
4739
|
+
accumulatedSpeechMs: this.pendingBargeIn.guard.speechMs,
|
|
4740
|
+
});
|
|
4741
|
+
}
|
|
4297
4742
|
this.pendingBargeIn = null;
|
|
4298
4743
|
if (reason === "max-duration") {
|
|
4299
4744
|
// A max-duration boundary always releases: drop any pending hold
|
|
@@ -4372,7 +4817,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
4372
4817
|
// end-of-turn is coming and the utterance replays this boundary on the
|
|
4373
4818
|
// silence path.
|
|
4374
4819
|
// `waitMsOverride` collapses that wait when the caller already knows no
|
|
4375
|
-
// end-of-turn is coming (see
|
|
4820
|
+
// end-of-turn is coming (see setProviderTurnDetectionActive).
|
|
4376
4821
|
private armProviderTurnEndFallbackTimer(
|
|
4377
4822
|
utterance: UtteranceCycle,
|
|
4378
4823
|
waitMsOverride?: number,
|
|
@@ -4405,32 +4850,22 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
4405
4850
|
}
|
|
4406
4851
|
|
|
4407
4852
|
/**
|
|
4408
|
-
*
|
|
4409
|
-
*
|
|
4410
|
-
*
|
|
4411
|
-
*
|
|
4412
|
-
* provider by the time the resolved one says otherwise. That deferred
|
|
4413
|
-
* boundary is parked on the fail-open deadline, a whole end-of-turn budget
|
|
4414
|
-
* away, waiting for an event that will now never arrive. Collapse the wait to
|
|
4415
|
-
* zero rather than burn the budget: the deadline body re-checks the cycle
|
|
4416
|
-
* and replays the silence boundary, which with the latch down takes the
|
|
4417
|
-
* ordinary hold path. Replaying through the deadline instead of calling
|
|
4418
|
-
* `handleVadUtteranceEnd` directly keeps the release off the arming
|
|
4419
|
-
* caller's stack, which is still mid-dial.
|
|
4420
|
-
*
|
|
4421
|
-
* A cleared latch with no deadline armed needs no unwind: either no
|
|
4422
|
-
* boundary ever deferred, or the caller resumed speaking and
|
|
4423
|
-
* `handleVadSpeechStart` already dropped the deadline, leaving the detector
|
|
4424
|
-
* owning the next boundary. The cycle's `turnBoundaryGeneration` stamp is
|
|
4425
|
-
* left as it is: `isStaleProviderTurnEnd` is only ever consulted from
|
|
4426
|
-
* `handleProviderTurnEnd`, which returns immediately once the latch is down.
|
|
4853
|
+
* The pre-dial configuration can defer a silence boundary to a provider
|
|
4854
|
+
* that resolves without turn detection. Collapse its fallback deadline
|
|
4855
|
+
* rather than wait for an event that cannot arrive. Replaying through the
|
|
4856
|
+
* timer keeps release off the resolver's stack and rechecks cycle ownership.
|
|
4427
4857
|
*/
|
|
4428
|
-
private
|
|
4429
|
-
|
|
4858
|
+
private setProviderTurnDetectionActive(active: boolean): void {
|
|
4859
|
+
const wasEndActive = this.providerTurnEndActive;
|
|
4860
|
+
this.providerTurnStartActive = active;
|
|
4861
|
+
if (active) {
|
|
4862
|
+
this.pendingBargeIn = null;
|
|
4863
|
+
}
|
|
4864
|
+
if (wasEndActive === this.providerTurnEndActive) {
|
|
4430
4865
|
return;
|
|
4431
4866
|
}
|
|
4432
|
-
this.
|
|
4433
|
-
if (
|
|
4867
|
+
this.microphoneIdleGate.reset();
|
|
4868
|
+
if (this.providerTurnEndActive || this.providerTurnEndTimer === null) {
|
|
4434
4869
|
return;
|
|
4435
4870
|
}
|
|
4436
4871
|
const utterance = this.currentUtterance;
|
|
@@ -4441,18 +4876,40 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
4441
4876
|
}
|
|
4442
4877
|
|
|
4443
4878
|
/**
|
|
4444
|
-
*
|
|
4445
|
-
*
|
|
4446
|
-
*
|
|
4879
|
+
* Turn indices belong to the stream, so duplicates cannot interrupt the
|
|
4880
|
+
* reply to an earlier cycle. The cycle also keeps its own index for matching
|
|
4881
|
+
* end-of-turn events; unnumbered events use local speech generation there.
|
|
4447
4882
|
*/
|
|
4448
|
-
private
|
|
4883
|
+
private handleProviderTurnStart(
|
|
4449
4884
|
utterance: UtteranceCycle,
|
|
4450
4885
|
turnIndex: number | undefined,
|
|
4451
4886
|
): void {
|
|
4452
|
-
|
|
4887
|
+
this.logInputDiagnostic("voice_input_provider_turn_start", {
|
|
4888
|
+
inputTurnId: this.ensureTurnId(utterance),
|
|
4889
|
+
turnIndex: turnIndex ?? null,
|
|
4890
|
+
sttProvider: utterance.dialedSttProvider,
|
|
4891
|
+
localDetectorActive: this.turnDetector?.isActive ?? false,
|
|
4892
|
+
});
|
|
4893
|
+
if (
|
|
4894
|
+
this.currentUtterance !== utterance ||
|
|
4895
|
+
(utterance.released && utterance.assistantTurnStarted) ||
|
|
4896
|
+
utterance.completed ||
|
|
4897
|
+
(turnIndex !== undefined &&
|
|
4898
|
+
this.latestProviderTurnStartIndex !== null &&
|
|
4899
|
+
turnIndex <= this.latestProviderTurnStartIndex)
|
|
4900
|
+
) {
|
|
4453
4901
|
return;
|
|
4454
4902
|
}
|
|
4455
|
-
|
|
4903
|
+
if (turnIndex !== undefined) {
|
|
4904
|
+
this.latestProviderTurnStartIndex = turnIndex;
|
|
4905
|
+
utterance.openProviderTurnIndex = turnIndex;
|
|
4906
|
+
}
|
|
4907
|
+
if (this.providerTurnStartActive) {
|
|
4908
|
+
utterance.speechRouted = true;
|
|
4909
|
+
this.detectedSpeech = true;
|
|
4910
|
+
this.markSpeechStart(utterance);
|
|
4911
|
+
this.handleSpeechStart("provider");
|
|
4912
|
+
}
|
|
4456
4913
|
}
|
|
4457
4914
|
|
|
4458
4915
|
/**
|
|
@@ -4520,8 +4977,25 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
4520
4977
|
*/
|
|
4521
4978
|
private async handleProviderTurnEnd(
|
|
4522
4979
|
utterance: UtteranceCycle,
|
|
4523
|
-
|
|
4980
|
+
event: Extract<SttStreamServerEvent, { type: "turn-end" }>,
|
|
4524
4981
|
): Promise<void> {
|
|
4982
|
+
const { turnIndex } = event;
|
|
4983
|
+
this.logInputDiagnostic("voice_input_provider_turn_end", {
|
|
4984
|
+
inputTurnId: utterance.turnId,
|
|
4985
|
+
sttProvider: utterance.dialedSttProvider,
|
|
4986
|
+
turnIndex: turnIndex ?? null,
|
|
4987
|
+
confidence: event.confidence ?? null,
|
|
4988
|
+
trigger: event.trigger ?? null,
|
|
4989
|
+
audioWindowEndSeconds: event.audioWindowEndSeconds ?? null,
|
|
4990
|
+
transcriptChars: event.text.length,
|
|
4991
|
+
localDetectorActive: this.turnDetector?.isActive ?? false,
|
|
4992
|
+
msSinceLocalSpeechStop:
|
|
4993
|
+
this.localSpeechStopAtMs === null
|
|
4994
|
+
? null
|
|
4995
|
+
: this.msSinceLocalSpeechStop(),
|
|
4996
|
+
providerTurnEndActive: this.providerTurnEndActive,
|
|
4997
|
+
providerTurnEndTimedOut: utterance.providerTurnEndTimedOut,
|
|
4998
|
+
});
|
|
4525
4999
|
if (
|
|
4526
5000
|
!this.providerTurnEndActive ||
|
|
4527
5001
|
this.currentUtterance !== utterance ||
|
|
@@ -4572,12 +5046,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
4572
5046
|
// closing the stream.
|
|
4573
5047
|
utterance.providerClosedTurn = true;
|
|
4574
5048
|
await this.releaseUtterance();
|
|
4575
|
-
//
|
|
4576
|
-
//
|
|
4577
|
-
//
|
|
4578
|
-
// ONSET: a detector left mid-turn reports no onset, so the caller could
|
|
4579
|
-
// not interrupt the reply they just triggered. The forced boundary
|
|
4580
|
-
// reaches an already-released utterance and returns.
|
|
5049
|
+
// The provider can commit before the local silence countdown expires.
|
|
5050
|
+
// Reset the detector so the next speech run gets its own local boundary.
|
|
5051
|
+
// This forced boundary reaches an already-released utterance and returns.
|
|
4581
5052
|
this.turnDetector?.forceEnd();
|
|
4582
5053
|
}
|
|
4583
5054
|
|
|
@@ -4642,6 +5113,13 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
4642
5113
|
// are skipped; the thinking frame and timers still apply.
|
|
4643
5114
|
const alreadyReleased = utterance.released;
|
|
4644
5115
|
turn.speculativePending = false;
|
|
5116
|
+
// Caller context is final once the speculative turn commits.
|
|
5117
|
+
if (!turn.hiddenPrompt && turn.speculativeContent !== null) {
|
|
5118
|
+
this.appendPendingLookRequest(turn.speculativeContent);
|
|
5119
|
+
if (turn.interruptedRequest !== null) {
|
|
5120
|
+
this.bargeInInterruption?.settle(turn.speculativeContent);
|
|
5121
|
+
}
|
|
5122
|
+
}
|
|
4645
5123
|
// Finals can land between the speculative dispatch and this verdict.
|
|
4646
5124
|
// Fill the language only when dispatch had none: the model request was
|
|
4647
5125
|
// already issued with the dispatch language, so overwriting here would
|
|
@@ -4723,11 +5201,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
4723
5201
|
reason: string,
|
|
4724
5202
|
): void {
|
|
4725
5203
|
turn.speculativePending = false;
|
|
4726
|
-
//
|
|
4727
|
-
// continuation's answer, its queued announcement, and any camera frame it
|
|
4728
|
-
// claimed all go back to the session. The turn that would have delivered
|
|
4729
|
-
// them is gone, and the utterance they belong to is about to be sent by
|
|
4730
|
-
// another.
|
|
5204
|
+
// Return claimed user context and camera frames for the replay.
|
|
4731
5205
|
this.restorePendingTurnContext(turn);
|
|
4732
5206
|
// Latched before the handle check: when the discard beats the bridge
|
|
4733
5207
|
// handle's resolution (startVoiceTurn still persisting), the handle's
|
|
@@ -4737,6 +5211,8 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
4737
5211
|
this.activeAssistantTurn = null;
|
|
4738
5212
|
}
|
|
4739
5213
|
this.clearFillerTimers(turn);
|
|
5214
|
+
this.releaseModeSessionRequests(turn);
|
|
5215
|
+
this.releaseModeSessionDelivery(turn);
|
|
4740
5216
|
turn.abortController.abort(
|
|
4741
5217
|
createAbortReason("voice_session_aborted", `live-voice-${reason}`),
|
|
4742
5218
|
);
|
|
@@ -5037,12 +5513,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5037
5513
|
// completion signal has no cycle to advance here.
|
|
5038
5514
|
return;
|
|
5039
5515
|
case "turn-start":
|
|
5040
|
-
|
|
5041
|
-
// because a provider roundtrip cannot beat a local energy gate on an
|
|
5042
|
-
// interrupt during playback (see DEFAULT_BARGE_IN_MIN_SPEECH_MS). The
|
|
5043
|
-
// index is recorded so a later end-of-turn can be told apart from one
|
|
5044
|
-
// this turn superseded (see isStaleProviderTurnEnd).
|
|
5045
|
-
this.recordProviderTurnStart(utterance, event.turnIndex);
|
|
5516
|
+
this.handleProviderTurnStart(utterance, event.turnIndex);
|
|
5046
5517
|
return;
|
|
5047
5518
|
case "eager-turn-end":
|
|
5048
5519
|
case "turn-resumed":
|
|
@@ -5052,7 +5523,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5052
5523
|
// follow-up.
|
|
5053
5524
|
return;
|
|
5054
5525
|
case "turn-end":
|
|
5055
|
-
await this.handleProviderTurnEnd(utterance, event
|
|
5526
|
+
await this.handleProviderTurnEnd(utterance, event);
|
|
5056
5527
|
return;
|
|
5057
5528
|
case "error":
|
|
5058
5529
|
await this.sendTranscriberErrorFrame(event);
|
|
@@ -5183,14 +5654,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5183
5654
|
return;
|
|
5184
5655
|
}
|
|
5185
5656
|
case "turn-start": {
|
|
5186
|
-
// Barge-in is deliberately untouched: local VAD still owns it,
|
|
5187
|
-
// because a provider roundtrip cannot beat a local energy gate on an
|
|
5188
|
-
// interrupt during playback (see DEFAULT_BARGE_IN_MIN_SPEECH_MS). The
|
|
5189
|
-
// index is recorded so a later end-of-turn can be told apart from one
|
|
5190
|
-
// this turn superseded (see isStaleProviderTurnEnd).
|
|
5191
5657
|
const target = this.pendingTranscriptCycle();
|
|
5192
5658
|
if (target) {
|
|
5193
|
-
this.
|
|
5659
|
+
this.handleProviderTurnStart(target, event.turnIndex);
|
|
5194
5660
|
}
|
|
5195
5661
|
return;
|
|
5196
5662
|
}
|
|
@@ -5206,7 +5672,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5206
5672
|
// transcript owns the turn the provider just closed.
|
|
5207
5673
|
const target = this.pendingTranscriptCycle();
|
|
5208
5674
|
if (target) {
|
|
5209
|
-
await this.handleProviderTurnEnd(target, event
|
|
5675
|
+
await this.handleProviderTurnEnd(target, event);
|
|
5210
5676
|
}
|
|
5211
5677
|
return;
|
|
5212
5678
|
}
|
|
@@ -5224,8 +5690,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5224
5690
|
}
|
|
5225
5691
|
// The shared stream closed under the session: fall back to the
|
|
5226
5692
|
// per-cycle path — the next arm resolves a fresh transcriber.
|
|
5227
|
-
this.
|
|
5228
|
-
this.sharedTranscriberLanguage = undefined;
|
|
5693
|
+
this.releaseSharedTranscriber(transcriber);
|
|
5229
5694
|
const drained = this.drainFinalizeQueueFor(transcriber);
|
|
5230
5695
|
const current = this.currentUtterance;
|
|
5231
5696
|
if (
|
|
@@ -5378,6 +5843,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5378
5843
|
}
|
|
5379
5844
|
|
|
5380
5845
|
private async interrupt(): Promise<void> {
|
|
5846
|
+
this.taskOutcomes.interruptPlayback(Date.now());
|
|
5847
|
+
this.taskAnnouncementsDeferred = true;
|
|
5848
|
+
this.clearTaskAnnouncementTimer();
|
|
5381
5849
|
if (this.isClosed || this.state === "failed") {
|
|
5382
5850
|
return;
|
|
5383
5851
|
}
|
|
@@ -5421,6 +5889,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5421
5889
|
// per-cycle) and clears the finalize bookkeeping. Used by interrupt() and
|
|
5422
5890
|
// close(); persistent mode re-establishes itself on the next arm.
|
|
5423
5891
|
private stopSessionTranscriber(): void {
|
|
5892
|
+
this.microphoneIdleGate.reset();
|
|
5424
5893
|
this.clearFinalizeGraceTimer();
|
|
5425
5894
|
this.finalizeQueue = [];
|
|
5426
5895
|
const shared = this.sharedTranscriber;
|
|
@@ -5505,64 +5974,32 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5505
5974
|
await this.launchAssistantTurn(utterance, content);
|
|
5506
5975
|
}
|
|
5507
5976
|
|
|
5508
|
-
|
|
5509
|
-
* Take the barge-in merge context, the completed-continuation context, the
|
|
5510
|
-
* handoff note, and the parked camera frame for the turn being launched
|
|
5511
|
-
* (each feeds exactly the next launched turn) and cancel the queued
|
|
5512
|
-
* announcement, because the turn this context rides delivers the same answer
|
|
5513
|
-
* and would otherwise say it twice.
|
|
5514
|
-
*
|
|
5515
|
-
* Every real user turn consumes here, speculative or not: a speculative
|
|
5516
|
-
* dispatch latches `assistantTurnStarted`, so a turn that skipped this would
|
|
5517
|
-
* never pick the context up on any later pass. A rolled-back dispatch hands
|
|
5518
|
-
* it all back (restorePendingTurnContext).
|
|
5519
|
-
*/
|
|
5977
|
+
// User-turn context is consumable; finished outcomes are only peeked.
|
|
5520
5978
|
private consumePendingTurnContext(): {
|
|
5521
5979
|
interruptedRequest: string | null;
|
|
5522
|
-
continuationResult: string | null;
|
|
5523
5980
|
handedOffRequest: string | null;
|
|
5524
|
-
announcement: ContinuationDelivery | null;
|
|
5525
5981
|
turnAttachmentId: string | null;
|
|
5526
5982
|
} {
|
|
5527
5983
|
const consumed = {
|
|
5528
5984
|
interruptedRequest: this.pendingInterruptedRequest,
|
|
5529
|
-
continuationResult: this.pendingContinuationResult,
|
|
5530
5985
|
handedOffRequest: this.pendingHandoffRequest,
|
|
5531
|
-
announcement: this.pendingAnnouncement,
|
|
5532
5986
|
turnAttachmentId: this.pendingTurnAttachmentId,
|
|
5533
5987
|
};
|
|
5534
5988
|
this.pendingInterruptedRequest = null;
|
|
5535
|
-
this.pendingContinuationResult = null;
|
|
5536
5989
|
this.pendingHandoffRequest = null;
|
|
5537
5990
|
this.pendingTurnAttachmentId = null;
|
|
5538
|
-
this.clearContinuationAnnouncement();
|
|
5539
5991
|
return consumed;
|
|
5540
5992
|
}
|
|
5541
5993
|
|
|
5542
|
-
|
|
5543
|
-
* Hand a turn's consumed pending context back to the session because the
|
|
5544
|
-
* turn never happened: a speculative dispatch was discarded or held, or the
|
|
5545
|
-
* launch never reached the bridge. Without this a hold verdict would destroy
|
|
5546
|
-
* a finished continuation's answer outright — the stash was already emptied
|
|
5547
|
-
* and no later turn would ever see it.
|
|
5548
|
-
*
|
|
5549
|
-
* Restores are conservative. A hard stop (interrupt/close/supersede) bumps
|
|
5550
|
-
* `detachStopGeneration`, and that drop is deliberate — never undone here.
|
|
5551
|
-
* Anything newer already sitting in a slot wins over the older value being
|
|
5552
|
-
* returned to it.
|
|
5553
|
-
*/
|
|
5994
|
+
// Rollbacks return consumed context without changing queued outcome delivery.
|
|
5554
5995
|
private restorePendingTurnContext(turn: ActiveAssistantTurn): void {
|
|
5555
5996
|
const interruptedRequest = turn.interruptedRequest;
|
|
5556
|
-
const continuationResult = turn.continuationResult;
|
|
5557
5997
|
const handedOffRequest = turn.handedOffRequest;
|
|
5558
|
-
const announcement = turn.consumedAnnouncement;
|
|
5559
5998
|
const turnAttachmentId = turn.turnAttachmentId;
|
|
5560
5999
|
// One restore per turn: the rollback paths overlap (a discarded turn whose
|
|
5561
6000
|
// leg start also fails), and the second pass must be a no-op.
|
|
5562
6001
|
turn.interruptedRequest = null;
|
|
5563
|
-
turn.continuationResult = null;
|
|
5564
6002
|
turn.handedOffRequest = null;
|
|
5565
|
-
turn.consumedAnnouncement = null;
|
|
5566
6003
|
turn.turnAttachmentId = null;
|
|
5567
6004
|
// Nobody's to send unless the slot already holds it. A client that
|
|
5568
6005
|
// retransmits the id this turn claimed parked the SAME frame rather than a
|
|
@@ -5593,39 +6030,26 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5593
6030
|
} else {
|
|
5594
6031
|
turn.staleFrameId = orphanedFrameId;
|
|
5595
6032
|
}
|
|
5596
|
-
if (this.pendingContinuationResult === null) {
|
|
5597
|
-
this.pendingContinuationResult = continuationResult;
|
|
5598
|
-
}
|
|
5599
|
-
// "Still running" is stale the moment a result exists, which is why the
|
|
5600
|
-
// detach clears the handoff note when it finishes — don't reinstate it
|
|
5601
|
-
// against a continuation that has since completed.
|
|
5602
6033
|
if (
|
|
5603
6034
|
this.pendingHandoffRequest === null &&
|
|
5604
|
-
this.
|
|
6035
|
+
this.detachControllers.size > 0
|
|
5605
6036
|
) {
|
|
5606
6037
|
this.pendingHandoffRequest = handedOffRequest;
|
|
5607
6038
|
}
|
|
5608
|
-
if (announcement !== null && this.pendingAnnouncement === null) {
|
|
5609
|
-
this.pendingAnnouncement = announcement;
|
|
5610
|
-
this.scheduleContinuationAnnouncement();
|
|
5611
|
-
}
|
|
5612
6039
|
}
|
|
5613
6040
|
|
|
5614
6041
|
// Build the ActiveAssistantTurn for a released utterance and drive its model
|
|
5615
6042
|
// leg. Resolves true once the leg has a live turn handle, false when the
|
|
5616
|
-
// dispatch never reached the bridge
|
|
5617
|
-
// route (
|
|
6043
|
+
// dispatch never reached the bridge: the signal a caller with a fallback
|
|
6044
|
+
// route (announceTaskOutcome) needs, since a failed start is otherwise
|
|
5618
6045
|
// handled entirely inside startAssistantLeg.
|
|
5619
6046
|
private async launchAssistantTurn(
|
|
5620
6047
|
utterance: UtteranceCycle,
|
|
5621
6048
|
content: string,
|
|
5622
6049
|
opts?: {
|
|
5623
|
-
|
|
5624
|
-
//
|
|
5625
|
-
|
|
5626
|
-
// Set on the turn that answers a look: which look. Its instruction goes
|
|
5627
|
-
// in the control prompt, not in `content`.
|
|
5628
|
-
lookFollowUp?: LookSessionControl;
|
|
6050
|
+
taskOutcome?: VoiceTaskOutcome;
|
|
6051
|
+
// The original request accompanies the fresh view in the control prompt.
|
|
6052
|
+
lookFollowUp?: LookFollowUp;
|
|
5629
6053
|
// Unified front-door: dispatch without releasing the utterance. The
|
|
5630
6054
|
// thinking frame and floor-holding timers are deferred until the leg's
|
|
5631
6055
|
// leading verdict commits the turn (see commitSpeculativeTurn); a hold
|
|
@@ -5642,16 +6066,70 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5642
6066
|
foregroundTaskEpoch?: number;
|
|
5643
6067
|
},
|
|
5644
6068
|
): Promise<boolean> {
|
|
6069
|
+
if (opts?.hiddenPrompt !== true) {
|
|
6070
|
+
this.taskOutcomes.acknowledgePlayback(Date.now());
|
|
6071
|
+
this.taskAnnouncementsDeferred = false;
|
|
6072
|
+
}
|
|
5645
6073
|
utterance.assistantTurnStarted = true;
|
|
5646
|
-
//
|
|
5647
|
-
//
|
|
5648
|
-
// answer twice. Every other turn is a real user turn and takes the context.
|
|
6074
|
+
// Task announcements own their updates. Pending user-turn context stays
|
|
6075
|
+
// available for the next user turn instead of being consumed by an update.
|
|
5649
6076
|
const pending =
|
|
5650
|
-
opts?.
|
|
5651
|
-
|
|
5652
|
-
|
|
6077
|
+
opts?.taskOutcome == null ? this.consumePendingTurnContext() : null;
|
|
6078
|
+
// A speculative dispatch may hold only the start of the interruption
|
|
6079
|
+
// ("Actually..."): its words reach the continuation judge when it
|
|
6080
|
+
// commits (commitSpeculativeTurn), and a discard restores the wait for
|
|
6081
|
+
// the redispatch.
|
|
6082
|
+
if (
|
|
6083
|
+
pending?.interruptedRequest != null &&
|
|
6084
|
+
opts?.hiddenPrompt !== true &&
|
|
6085
|
+
opts?.speculative !== true
|
|
6086
|
+
) {
|
|
6087
|
+
this.bargeInInterruption?.settle(content);
|
|
6088
|
+
}
|
|
5653
6089
|
const token = Symbol("live-voice-assistant-turn");
|
|
5654
6090
|
const turnId = this.ensureTurnId(utterance);
|
|
6091
|
+
const initialLeg = opts?.initialLeg ?? "front-door";
|
|
6092
|
+
const modeSessionRequestId = randomUUID();
|
|
6093
|
+
const attemptedModeSessionRequestIds: string[] = [modeSessionRequestId];
|
|
6094
|
+
let modeSessionSource: ModeSessionSourceHandle | undefined;
|
|
6095
|
+
let modeSessionRequestIds: string[] = [];
|
|
6096
|
+
let modeSessionDeliveryTurnId: string | null = null;
|
|
6097
|
+
try {
|
|
6098
|
+
modeSessionSource =
|
|
6099
|
+
this.cameraModeSessions?.holdDelivery(modeSessionRequestId);
|
|
6100
|
+
if (!modeSessionSource) {
|
|
6101
|
+
this.releaseModeSessionTurnId(modeSessionRequestId);
|
|
6102
|
+
} else {
|
|
6103
|
+
modeSessionRequestIds.push(modeSessionRequestId);
|
|
6104
|
+
if (initialLeg === "front-door") {
|
|
6105
|
+
const escalatedRequestId = randomUUID();
|
|
6106
|
+
attemptedModeSessionRequestIds.push(escalatedRequestId);
|
|
6107
|
+
const escalatedSource =
|
|
6108
|
+
this.cameraModeSessions?.holdDelivery(escalatedRequestId);
|
|
6109
|
+
if (!sameModeSessionSource(escalatedSource, modeSessionSource)) {
|
|
6110
|
+
throw new Error("Camera session changed during voice admission");
|
|
6111
|
+
}
|
|
6112
|
+
modeSessionRequestIds.push(escalatedRequestId);
|
|
6113
|
+
}
|
|
6114
|
+
const deliveryTurnId = `live-voice-delivery:${this.context.sessionId}:${++this.modeSessionDeliveryCounter}`;
|
|
6115
|
+
attemptedModeSessionRequestIds.push(deliveryTurnId);
|
|
6116
|
+
const deliverySource =
|
|
6117
|
+
this.cameraModeSessions?.holdDelivery(deliveryTurnId);
|
|
6118
|
+
if (!sameModeSessionSource(deliverySource, modeSessionSource)) {
|
|
6119
|
+
throw new Error("Camera session changed during voice admission");
|
|
6120
|
+
}
|
|
6121
|
+
modeSessionDeliveryTurnId = deliveryTurnId;
|
|
6122
|
+
}
|
|
6123
|
+
} catch (err) {
|
|
6124
|
+
log.warn(
|
|
6125
|
+
{ err, turnId },
|
|
6126
|
+
"Could not attach camera session ownership to voice turn",
|
|
6127
|
+
);
|
|
6128
|
+
this.releaseModeSessionTurnIds(attemptedModeSessionRequestIds);
|
|
6129
|
+
modeSessionSource = undefined;
|
|
6130
|
+
modeSessionRequestIds = [];
|
|
6131
|
+
modeSessionDeliveryTurnId = null;
|
|
6132
|
+
}
|
|
5655
6133
|
this.startMetricsTurnIfNeeded(utterance, turnId);
|
|
5656
6134
|
this.markAssistantDispatch(utterance, turnId);
|
|
5657
6135
|
const abortController = new AbortController();
|
|
@@ -5671,6 +6149,11 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5671
6149
|
language: this.turnLanguageFor(utterance),
|
|
5672
6150
|
abortController,
|
|
5673
6151
|
handle: null,
|
|
6152
|
+
modeSessionSource,
|
|
6153
|
+
modeSessionRequestIds,
|
|
6154
|
+
modeSessionDeliveryTurnId: modeSessionSource
|
|
6155
|
+
? modeSessionDeliveryTurnId
|
|
6156
|
+
: null,
|
|
5674
6157
|
launchedAtMs: Date.now(),
|
|
5675
6158
|
progress: createProgressCadence({
|
|
5676
6159
|
config: progressConfigForCadence(
|
|
@@ -5714,8 +6197,10 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5714
6197
|
sessionControlRequested: null,
|
|
5715
6198
|
activityLabel: "",
|
|
5716
6199
|
publishedApprovalRequestId: null,
|
|
6200
|
+
publishedActivityKind: null,
|
|
5717
6201
|
pendingApproval: null,
|
|
5718
6202
|
ttsAudioStarted: false,
|
|
6203
|
+
ttsFailed: false,
|
|
5719
6204
|
finalized: false,
|
|
5720
6205
|
speculativePending: opts?.speculative === true,
|
|
5721
6206
|
speculativeGeneration: this.vadSpeechGeneration,
|
|
@@ -5727,13 +6212,16 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5727
6212
|
discardRequested: false,
|
|
5728
6213
|
speculativeBuffer: "",
|
|
5729
6214
|
interruptedRequest: pending?.interruptedRequest ?? null,
|
|
5730
|
-
|
|
6215
|
+
contextOutcomes: pending ? this.taskOutcomes.pendingOutcomes() : [],
|
|
5731
6216
|
handedOffRequest: pending?.handedOffRequest ?? null,
|
|
5732
6217
|
turnAttachmentId: pending?.turnAttachmentId ?? null,
|
|
5733
6218
|
staleFrameId: null,
|
|
5734
|
-
consumedAnnouncement: pending?.announcement ?? null,
|
|
5735
6219
|
pendingContextStopGeneration: this.detachStopGeneration,
|
|
5736
|
-
|
|
6220
|
+
taskOutcome: opts?.taskOutcome ?? null,
|
|
6221
|
+
taskDeliveryContext: opts?.taskOutcome
|
|
6222
|
+
? this.taskOutcomes.deliveryContext(opts.taskOutcome)
|
|
6223
|
+
: null,
|
|
6224
|
+
notificationHandledSilently: false,
|
|
5737
6225
|
lookFollowUp: opts?.lookFollowUp ?? null,
|
|
5738
6226
|
hiddenPrompt: opts?.hiddenPrompt === true,
|
|
5739
6227
|
foregroundTaskEpoch,
|
|
@@ -5743,6 +6231,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5743
6231
|
: null,
|
|
5744
6232
|
deltaEpoch: 0,
|
|
5745
6233
|
frontDoor: null,
|
|
6234
|
+
escalationTarget: null,
|
|
5746
6235
|
ttsBuffer: "",
|
|
5747
6236
|
spokenSegments: 0,
|
|
5748
6237
|
spokenChars: 0,
|
|
@@ -5756,6 +6245,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5756
6245
|
};
|
|
5757
6246
|
this.activeAssistantTurn = activeTurn;
|
|
5758
6247
|
this.turnsLaunched += 1;
|
|
6248
|
+
if (!activeTurn.hiddenPrompt && !activeTurn.speculativePending) {
|
|
6249
|
+
this.appendPendingLookRequest(content);
|
|
6250
|
+
}
|
|
5759
6251
|
|
|
5760
6252
|
// A speculative turn defers the thinking frame and both floor-holding
|
|
5761
6253
|
// timers to commitSpeculativeTurn: until the verdict arrives, the pause
|
|
@@ -5763,6 +6255,8 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5763
6255
|
if (!opts?.speculative) {
|
|
5764
6256
|
await this.sendFrame({ type: "thinking", turnId });
|
|
5765
6257
|
if (!this.isActiveAssistantTurn(token)) {
|
|
6258
|
+
this.releaseModeSessionRequests(activeTurn);
|
|
6259
|
+
this.releaseModeSessionDelivery(activeTurn);
|
|
5766
6260
|
this.restorePendingTurnContext(activeTurn);
|
|
5767
6261
|
// Nothing was persisted, so a frame the restore could not hand back
|
|
5768
6262
|
// links to no row and is collectible right here.
|
|
@@ -5805,7 +6299,6 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5805
6299
|
activeTurn.progress.arm();
|
|
5806
6300
|
}
|
|
5807
6301
|
|
|
5808
|
-
const initialLeg = opts?.initialLeg ?? "front-door";
|
|
5809
6302
|
const started = await this.startAssistantLeg(activeTurn, {
|
|
5810
6303
|
content,
|
|
5811
6304
|
routingLeg: initialLeg,
|
|
@@ -5819,6 +6312,8 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5819
6312
|
: {}),
|
|
5820
6313
|
});
|
|
5821
6314
|
if (!started) {
|
|
6315
|
+
this.releaseModeSessionRequests(activeTurn);
|
|
6316
|
+
this.releaseModeSessionDelivery(activeTurn);
|
|
5822
6317
|
// Nothing was persisted or spoken, so the context this turn consumed goes
|
|
5823
6318
|
// back to the session rather than dying with the failed dispatch. A frame
|
|
5824
6319
|
// a newer one displaced meanwhile links to no row, so it is collected
|
|
@@ -5861,10 +6356,15 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5861
6356
|
attachments?: readonly string[];
|
|
5862
6357
|
},
|
|
5863
6358
|
): Promise<boolean> {
|
|
5864
|
-
if (
|
|
6359
|
+
if (
|
|
6360
|
+
!this.startVoiceTurn ||
|
|
6361
|
+
this.isClosed ||
|
|
6362
|
+
activeTurn.abortController.signal.aborted
|
|
6363
|
+
) {
|
|
5865
6364
|
return false;
|
|
5866
6365
|
}
|
|
5867
6366
|
const { token, utterance, turnId } = activeTurn;
|
|
6367
|
+
const modeSessionRequestId = activeTurn.modeSessionRequestIds.shift();
|
|
5868
6368
|
|
|
5869
6369
|
// `rawText` accumulates this leg's spoken stream. A front-door leg runs
|
|
5870
6370
|
// through the shared coordinator, which reads its stream through the
|
|
@@ -5934,6 +6434,15 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5934
6434
|
activeTurn.handle?.abort();
|
|
5935
6435
|
activeTurn.handle = null;
|
|
5936
6436
|
},
|
|
6437
|
+
overruleLeg: () => {
|
|
6438
|
+
const handle = activeTurn.handle;
|
|
6439
|
+
activeTurn.handle = null;
|
|
6440
|
+
if (handle?.overrule) {
|
|
6441
|
+
handle.overrule();
|
|
6442
|
+
} else {
|
|
6443
|
+
handle?.abort();
|
|
6444
|
+
}
|
|
6445
|
+
},
|
|
5937
6446
|
speakBridge: (bridge) =>
|
|
5938
6447
|
this.speakEscalationBridge(activeTurn, bridge),
|
|
5939
6448
|
startEscalatedLeg: (escalated) => {
|
|
@@ -5947,6 +6456,107 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5947
6456
|
activeTurn.frontDoor = coordinator;
|
|
5948
6457
|
}
|
|
5949
6458
|
|
|
6459
|
+
// The leg's completion. A front-door answer held for the escalation judge
|
|
6460
|
+
// is spoken or overruled first, so the turn never closes its TTS buffer
|
|
6461
|
+
// under text that has not been released yet.
|
|
6462
|
+
const completeLeg = (
|
|
6463
|
+
msg: Parameters<NonNullable<VoiceTurnCallbacks["message_complete"]>>[0],
|
|
6464
|
+
): void => {
|
|
6465
|
+
if (
|
|
6466
|
+
coordinator?.awaitingJudge === true &&
|
|
6467
|
+
msg.type === "message_complete"
|
|
6468
|
+
) {
|
|
6469
|
+
void coordinator.settled().then(() => completeLeg(msg));
|
|
6470
|
+
return;
|
|
6471
|
+
}
|
|
6472
|
+
const current = this.activeAssistantTurn;
|
|
6473
|
+
if (
|
|
6474
|
+
current?.token !== token ||
|
|
6475
|
+
current.assistantCompleted ||
|
|
6476
|
+
// A barged-in turn finalizes through cancelAssistantTurn.
|
|
6477
|
+
current.abortController.signal.aborted ||
|
|
6478
|
+
this.isClosed
|
|
6479
|
+
) {
|
|
6480
|
+
return;
|
|
6481
|
+
}
|
|
6482
|
+
if (!leg.frontDoor) {
|
|
6483
|
+
this.claimSuspendedForegroundTask(current);
|
|
6484
|
+
}
|
|
6485
|
+
// A speculative leg that finished without a single delta (empty
|
|
6486
|
+
// output, provider hiccup) carries no verdict, so fail open to a
|
|
6487
|
+
// committed turn so the utterance releases and finalizes like a
|
|
6488
|
+
// normal empty completion instead of dangling un-released.
|
|
6489
|
+
if (current.speculativePending) {
|
|
6490
|
+
this.commitSpeculativeTurn(current);
|
|
6491
|
+
}
|
|
6492
|
+
// A front-door leg that stopped mid-bridge hands off now with
|
|
6493
|
+
// whatever arrived. A cancellation mid-bridge falls through to
|
|
6494
|
+
// normal cancelled finalization instead: a dead turn must not
|
|
6495
|
+
// spawn an escalated leg. A front-door leg that handed off is
|
|
6496
|
+
// finished; the escalated leg drives completion, so this leg's
|
|
6497
|
+
// own trailing completion (including the generation_cancelled
|
|
6498
|
+
// from its abort) is a no-op.
|
|
6499
|
+
if (coordinator !== null) {
|
|
6500
|
+
if (msg.type === "message_complete") {
|
|
6501
|
+
coordinator.complete();
|
|
6502
|
+
}
|
|
6503
|
+
if (coordinator.handedOff) {
|
|
6504
|
+
return;
|
|
6505
|
+
}
|
|
6506
|
+
}
|
|
6507
|
+
// A held "[…"-tail that never completed a marker is real text:
|
|
6508
|
+
// force-flush it before assistantCompleted closes the TTS buffer
|
|
6509
|
+
// and completeTtsForTurn signals the drain, so it is spoken and
|
|
6510
|
+
// emitted rather than dropped.
|
|
6511
|
+
if (!leg.frontDoor && msg.type === "message_complete") {
|
|
6512
|
+
flushLegText(rawText, { force: true });
|
|
6513
|
+
}
|
|
6514
|
+
// Read off the leg that finished the reply: a front-door answer
|
|
6515
|
+
// or the escalated leg. A handed-off front-door leg returned
|
|
6516
|
+
// above, so its holding phrase can never end a call.
|
|
6517
|
+
if (msg.type === "message_complete") {
|
|
6518
|
+
current.notificationHandledSilently =
|
|
6519
|
+
current.taskOutcome !== null &&
|
|
6520
|
+
rawText.trim() === TASK_UPDATE_SILENT_MARKER;
|
|
6521
|
+
const request = requestedSessionControl(rawText, this.sessionControls);
|
|
6522
|
+
if (request?.action === "updates") {
|
|
6523
|
+
// The session's own control: nothing to send, and nothing to
|
|
6524
|
+
// wait for, since it shapes turns that have not started yet.
|
|
6525
|
+
this.progressCadence = request.cadence;
|
|
6526
|
+
log.info(
|
|
6527
|
+
{ turnId, cadence: request.cadence },
|
|
6528
|
+
"Live voice progress cadence changed",
|
|
6529
|
+
);
|
|
6530
|
+
} else if (request?.action === "task_stop") {
|
|
6531
|
+
this.stopOutstandingInterruptedWork("spoken_task_stop");
|
|
6532
|
+
} else {
|
|
6533
|
+
if (request?.action === "end") {
|
|
6534
|
+
this.clearForegroundTask("call_end_requested");
|
|
6535
|
+
}
|
|
6536
|
+
current.sessionControlRequested = request;
|
|
6537
|
+
}
|
|
6538
|
+
}
|
|
6539
|
+
if (leg.frontDoor) {
|
|
6540
|
+
this.queueForegroundTaskResume(current);
|
|
6541
|
+
}
|
|
6542
|
+
current.assistantCompleted = true;
|
|
6543
|
+
if (msg.type === "generation_cancelled") {
|
|
6544
|
+
void this.finalizeAssistantTurn(
|
|
6545
|
+
current,
|
|
6546
|
+
"cancelled",
|
|
6547
|
+
"generation_cancelled",
|
|
6548
|
+
);
|
|
6549
|
+
return;
|
|
6550
|
+
}
|
|
6551
|
+
current.assistantMessageId = msg.messageId ?? null;
|
|
6552
|
+
this.completeTtsForTurn(token);
|
|
6553
|
+
};
|
|
6554
|
+
|
|
6555
|
+
let finishStart!: () => void;
|
|
6556
|
+
const startup = new Promise<void>((resolve) => {
|
|
6557
|
+
finishStart = resolve;
|
|
6558
|
+
});
|
|
6559
|
+
this.pendingAssistantStarts.add(startup);
|
|
5950
6560
|
try {
|
|
5951
6561
|
// Latched before the await, not after: this flag only decides whether the
|
|
5952
6562
|
// end event carries a silence classification, and the dashboard decides
|
|
@@ -5969,7 +6579,18 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5969
6579
|
const handle = await this.startVoiceTurn({
|
|
5970
6580
|
conversationId: this.conversationId,
|
|
5971
6581
|
voiceSessionId: this.context.sessionId,
|
|
6582
|
+
...(activeTurn.modeSessionSource && modeSessionRequestId
|
|
6583
|
+
? {
|
|
6584
|
+
preacceptedModeSession: {
|
|
6585
|
+
requestId: modeSessionRequestId,
|
|
6586
|
+
source: activeTurn.modeSessionSource,
|
|
6587
|
+
},
|
|
6588
|
+
}
|
|
6589
|
+
: {}),
|
|
5972
6590
|
...(activeTurn.hiddenPrompt ? { hiddenSyntheticPrompt: true } : {}),
|
|
6591
|
+
...(activeTurn.lookFollowUp !== null
|
|
6592
|
+
? { routingUtterance: activeTurn.lookFollowUp.callerUtterance }
|
|
6593
|
+
: {}),
|
|
5973
6594
|
userMessageChannel: "vellum",
|
|
5974
6595
|
assistantMessageChannel: "vellum",
|
|
5975
6596
|
// Fixed, and NOT the originating client: this pair resolves the turn's
|
|
@@ -6008,16 +6629,25 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
6008
6629
|
onApprovalsResolved: () => {
|
|
6009
6630
|
this.clearAwaitingApproval(activeTurn);
|
|
6010
6631
|
},
|
|
6632
|
+
...(leg.routingLeg === "escalated"
|
|
6633
|
+
? {
|
|
6634
|
+
onEscalationTargetResolved: (target) => {
|
|
6635
|
+
if (!this.isActiveAssistantTurn(token)) {
|
|
6636
|
+
return;
|
|
6637
|
+
}
|
|
6638
|
+
activeTurn.escalationTarget = target;
|
|
6639
|
+
this.publishEscalationActivity(activeTurn, target);
|
|
6640
|
+
},
|
|
6641
|
+
}
|
|
6642
|
+
: {}),
|
|
6011
6643
|
content: leg.content,
|
|
6012
6644
|
...(leg.attachments ? { attachments: leg.attachments } : {}),
|
|
6013
6645
|
isInbound: true,
|
|
6014
6646
|
launchedAtMs: activeTurn.launchedAtMs,
|
|
6015
6647
|
signal: activeTurn.abortController.signal,
|
|
6016
|
-
|
|
6017
|
-
|
|
6018
|
-
|
|
6019
|
-
...(activeTurn.continuationDelivery !== null
|
|
6020
|
-
? { hiddenSyntheticPrompt: true }
|
|
6648
|
+
...(activeTurn.taskOutcome !== null &&
|
|
6649
|
+
activeTurn.taskOutcome.source === "subagent"
|
|
6650
|
+
? { subagentNotification: activeTurn.taskOutcome }
|
|
6021
6651
|
: {}),
|
|
6022
6652
|
...(leg.overrideProfile != null
|
|
6023
6653
|
? { overrideProfile: leg.overrideProfile }
|
|
@@ -6070,90 +6700,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
6070
6700
|
rawText += msg.text;
|
|
6071
6701
|
flushLegText(rawText);
|
|
6072
6702
|
},
|
|
6073
|
-
message_complete:
|
|
6074
|
-
const current = this.activeAssistantTurn;
|
|
6075
|
-
if (
|
|
6076
|
-
current?.token !== token ||
|
|
6077
|
-
current.assistantCompleted ||
|
|
6078
|
-
// A barged-in turn finalizes through cancelAssistantTurn.
|
|
6079
|
-
current.abortController.signal.aborted ||
|
|
6080
|
-
this.isClosed
|
|
6081
|
-
) {
|
|
6082
|
-
return;
|
|
6083
|
-
}
|
|
6084
|
-
if (!leg.frontDoor) {
|
|
6085
|
-
this.claimSuspendedForegroundTask(current);
|
|
6086
|
-
}
|
|
6087
|
-
// A speculative leg that finished without a single delta (empty
|
|
6088
|
-
// output, provider hiccup) carries no verdict — fail open to a
|
|
6089
|
-
// committed turn so the utterance releases and finalizes like a
|
|
6090
|
-
// normal empty completion instead of dangling un-released.
|
|
6091
|
-
if (current.speculativePending) {
|
|
6092
|
-
this.commitSpeculativeTurn(current);
|
|
6093
|
-
}
|
|
6094
|
-
// A front-door leg that stopped mid-bridge hands off now with
|
|
6095
|
-
// whatever arrived. A cancellation mid-bridge falls through to
|
|
6096
|
-
// normal cancelled finalization instead: a dead turn must not
|
|
6097
|
-
// spawn an escalated leg. A front-door leg that handed off is
|
|
6098
|
-
// finished; the escalated leg drives completion, so this leg's
|
|
6099
|
-
// own trailing completion (including the generation_cancelled
|
|
6100
|
-
// from its abort) is a no-op.
|
|
6101
|
-
if (coordinator !== null) {
|
|
6102
|
-
if (msg.type === "message_complete") {
|
|
6103
|
-
coordinator.complete();
|
|
6104
|
-
}
|
|
6105
|
-
if (coordinator.handedOff) {
|
|
6106
|
-
return;
|
|
6107
|
-
}
|
|
6108
|
-
}
|
|
6109
|
-
// A held "[…"-tail that never completed a marker is real text —
|
|
6110
|
-
// force-flush it before assistantCompleted closes the TTS buffer
|
|
6111
|
-
// and completeTtsForTurn signals the drain, so it is spoken and
|
|
6112
|
-
// emitted rather than dropped.
|
|
6113
|
-
if (!leg.frontDoor && msg.type === "message_complete") {
|
|
6114
|
-
flushLegText(rawText, { force: true });
|
|
6115
|
-
}
|
|
6116
|
-
// Read off the leg that finished the reply: a front-door answer
|
|
6117
|
-
// or the escalated leg. A handed-off front-door leg returned
|
|
6118
|
-
// above, so its holding phrase can never end a call.
|
|
6119
|
-
if (msg.type === "message_complete") {
|
|
6120
|
-
const request = requestedSessionControl(
|
|
6121
|
-
rawText,
|
|
6122
|
-
this.sessionControls,
|
|
6123
|
-
);
|
|
6124
|
-
if (request?.action === "updates") {
|
|
6125
|
-
// The session's own control: nothing to send, and nothing to
|
|
6126
|
-
// wait for, since it shapes turns that have not started yet.
|
|
6127
|
-
this.progressCadence = request.cadence;
|
|
6128
|
-
log.info(
|
|
6129
|
-
{ turnId, cadence: request.cadence },
|
|
6130
|
-
"Live voice progress cadence changed",
|
|
6131
|
-
);
|
|
6132
|
-
} else if (request?.action === "task_stop") {
|
|
6133
|
-
this.stopOutstandingInterruptedWork("spoken_task_stop");
|
|
6134
|
-
} else {
|
|
6135
|
-
if (request?.action === "end") {
|
|
6136
|
-
this.clearForegroundTask("call_end_requested");
|
|
6137
|
-
}
|
|
6138
|
-
current.sessionControlRequested = request;
|
|
6139
|
-
}
|
|
6140
|
-
}
|
|
6141
|
-
if (leg.frontDoor) {
|
|
6142
|
-
this.queueForegroundTaskResume(current);
|
|
6143
|
-
}
|
|
6144
|
-
current.assistantCompleted = true;
|
|
6145
|
-
if (msg.type === "generation_cancelled") {
|
|
6146
|
-
void this.finalizeAssistantTurn(
|
|
6147
|
-
current,
|
|
6148
|
-
"cancelled",
|
|
6149
|
-
"generation_cancelled",
|
|
6150
|
-
);
|
|
6151
|
-
return;
|
|
6152
|
-
}
|
|
6153
|
-
current.assistantMessageId = msg.messageId ?? null;
|
|
6154
|
-
current.continuationDelivery = null;
|
|
6155
|
-
this.completeTtsForTurn(token);
|
|
6156
|
-
},
|
|
6703
|
+
message_complete: completeLeg,
|
|
6157
6704
|
persisted_user_message_id: (messageId) => {
|
|
6158
6705
|
const current = this.activeAssistantTurn;
|
|
6159
6706
|
// Only the first leg's user row is the real caller utterance; the
|
|
@@ -6212,7 +6759,6 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
6212
6759
|
foregroundToolContendsWithContinuation(toolName, detail?.input)
|
|
6213
6760
|
) {
|
|
6214
6761
|
this.abortDetachedRuns({
|
|
6215
|
-
keepPendingResult: true,
|
|
6216
6762
|
reason: "foreground_tool_contends",
|
|
6217
6763
|
toolName,
|
|
6218
6764
|
});
|
|
@@ -6281,7 +6827,6 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
6281
6827
|
})();
|
|
6282
6828
|
},
|
|
6283
6829
|
});
|
|
6284
|
-
|
|
6285
6830
|
const current = this.activeAssistantTurn;
|
|
6286
6831
|
if (current?.token !== token) {
|
|
6287
6832
|
// A discard that beat this handle's resolution still owes the
|
|
@@ -6324,8 +6869,14 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
6324
6869
|
}
|
|
6325
6870
|
|
|
6326
6871
|
current.handle = handle;
|
|
6872
|
+
if (coordinator !== null && handle.escalationJudgement) {
|
|
6873
|
+
coordinator.attachEscalationJudge(handle.escalationJudgement);
|
|
6874
|
+
}
|
|
6327
6875
|
return true;
|
|
6328
6876
|
} catch (err) {
|
|
6877
|
+
if (modeSessionRequestId) {
|
|
6878
|
+
this.releaseModeSessionTurnId(modeSessionRequestId);
|
|
6879
|
+
}
|
|
6329
6880
|
if (!this.isActiveAssistantTurn(token)) {
|
|
6330
6881
|
return false;
|
|
6331
6882
|
}
|
|
@@ -6338,6 +6889,8 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
6338
6889
|
this.rearmForegroundTaskAfterLegFailure(activeTurn);
|
|
6339
6890
|
this.clearFillerTimers(activeTurn);
|
|
6340
6891
|
this.clearActiveAssistantTurn(token);
|
|
6892
|
+
this.releaseModeSessionRequests(activeTurn);
|
|
6893
|
+
this.releaseModeSessionDelivery(activeTurn);
|
|
6341
6894
|
if (foregroundTaskResumeRearmed) {
|
|
6342
6895
|
this.scheduleForegroundTaskResume();
|
|
6343
6896
|
}
|
|
@@ -6351,6 +6904,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
6351
6904
|
await this.finalizePendingUtterance(utterance, "assistant_start_error");
|
|
6352
6905
|
this.scheduleRearmAfterTurn();
|
|
6353
6906
|
return false;
|
|
6907
|
+
} finally {
|
|
6908
|
+
this.pendingAssistantStarts.delete(startup);
|
|
6909
|
+
finishStart();
|
|
6354
6910
|
}
|
|
6355
6911
|
}
|
|
6356
6912
|
|
|
@@ -6377,15 +6933,40 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
6377
6933
|
// otherwise sit buffered until a sentence boundary and leave the
|
|
6378
6934
|
// caller in silence during the escalated model's call.
|
|
6379
6935
|
this.flushTtsBuffer(activeTurn.token, true);
|
|
6380
|
-
|
|
6381
|
-
|
|
6382
|
-
|
|
6383
|
-
|
|
6384
|
-
|
|
6385
|
-
|
|
6386
|
-
|
|
6387
|
-
|
|
6936
|
+
} else {
|
|
6937
|
+
// The canned bridge is a fixed localized-table phrase, enqueued
|
|
6938
|
+
// directly (it is already one complete sentence) so the segment can
|
|
6939
|
+
// carry the "en" override when the table lacks the turn's language.
|
|
6940
|
+
const speakable = sanitizeForTts(spokenBridge).trim();
|
|
6941
|
+
if (speakable.length > 0) {
|
|
6942
|
+
this.enqueueTtsSegment(activeTurn.token, speakable, { language });
|
|
6943
|
+
}
|
|
6388
6944
|
}
|
|
6945
|
+
|
|
6946
|
+
// Target resolution can land before the bridge's queued audio finishes.
|
|
6947
|
+
// Its activity frame moves the system surface to thinking, then a late
|
|
6948
|
+
// bridge tts_audio frame moves it back to speaking. Restore the neutral
|
|
6949
|
+
// waiting phase at the bridge's emission boundary, ahead of any answer
|
|
6950
|
+
// audio queued behind it.
|
|
6951
|
+
const bridgeDrain = activeTurn.ttsQueue;
|
|
6952
|
+
void bridgeDrain.then(
|
|
6953
|
+
() => {
|
|
6954
|
+
const target = activeTurn.escalationTarget;
|
|
6955
|
+
if (
|
|
6956
|
+
target === null ||
|
|
6957
|
+
!this.isActiveAssistantTurn(activeTurn.token) ||
|
|
6958
|
+
activeTurn.assistantCompleted
|
|
6959
|
+
) {
|
|
6960
|
+
return;
|
|
6961
|
+
}
|
|
6962
|
+
if (activeTurn.publishedActivityKind === "escalation") {
|
|
6963
|
+
this.publishEscalationActivity(activeTurn, target);
|
|
6964
|
+
return;
|
|
6965
|
+
}
|
|
6966
|
+
this.liveActivityReporter.restoreThinkingPhase();
|
|
6967
|
+
},
|
|
6968
|
+
() => undefined,
|
|
6969
|
+
);
|
|
6389
6970
|
}
|
|
6390
6971
|
|
|
6391
6972
|
private async cancelAssistantTurn(reason: string): Promise<void> {
|
|
@@ -6607,7 +7188,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
6607
7188
|
isLookSessionControl(sessionControl.action) &&
|
|
6608
7189
|
currentTurn.lookFollowUp === null
|
|
6609
7190
|
) {
|
|
6610
|
-
this.awaitLookFrame(sessionControl.action);
|
|
7191
|
+
this.awaitLookFrame(sessionControl.action, currentTurn);
|
|
6611
7192
|
}
|
|
6612
7193
|
await this.sendFrame(
|
|
6613
7194
|
{
|
|
@@ -6643,6 +7224,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
6643
7224
|
if (!currentTurn.abortController.signal.aborted) {
|
|
6644
7225
|
this.scheduleRearmAfterTurn();
|
|
6645
7226
|
}
|
|
7227
|
+
})
|
|
7228
|
+
.finally(() => {
|
|
7229
|
+
this.releaseModeSessionDelivery(activeTurn);
|
|
6646
7230
|
});
|
|
6647
7231
|
}
|
|
6648
7232
|
|
|
@@ -6696,6 +7280,8 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
6696
7280
|
}
|
|
6697
7281
|
const job: TtsSegmentJob = {
|
|
6698
7282
|
text: segment,
|
|
7283
|
+
isReply: options.countsAsFirstSegment ?? true,
|
|
7284
|
+
audioSent: false,
|
|
6699
7285
|
language: options.language,
|
|
6700
7286
|
started: false,
|
|
6701
7287
|
settled: false,
|
|
@@ -6813,6 +7399,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
6813
7399
|
await job.frames;
|
|
6814
7400
|
|
|
6815
7401
|
if (failed && this.isForwardingTts(token)) {
|
|
7402
|
+
currentTurn.ttsFailed = true;
|
|
6816
7403
|
// Per-segment failure: the turn (and session) continue, so the
|
|
6817
7404
|
// error is recoverable for the client.
|
|
6818
7405
|
await this.sendFrame(
|
|
@@ -6825,6 +7412,12 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
6825
7412
|
() => this.isForwardingTts(token),
|
|
6826
7413
|
);
|
|
6827
7414
|
}
|
|
7415
|
+
} catch (err) {
|
|
7416
|
+
const turn = this.activeAssistantTurn;
|
|
7417
|
+
if (turn?.token === token) {
|
|
7418
|
+
turn.ttsFailed = true;
|
|
7419
|
+
}
|
|
7420
|
+
throw err;
|
|
6828
7421
|
} finally {
|
|
6829
7422
|
job.settled = true;
|
|
6830
7423
|
const settledTurn = this.activeAssistantTurn;
|
|
@@ -6849,16 +7442,25 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
6849
7442
|
if (activeTurn?.token !== token) {
|
|
6850
7443
|
return;
|
|
6851
7444
|
}
|
|
6852
|
-
// Only retain the assistant TTS audio when it will be archived (see
|
|
6853
|
-
// collectUserAudio); the mime/sample-rate are cheap and left unconditional.
|
|
6854
|
-
if (this.archiveAudio) {
|
|
6855
|
-
activeTurn.assistantAudioChunks.push(
|
|
6856
|
-
Buffer.from(chunk.dataBase64, "base64"),
|
|
6857
|
-
);
|
|
6858
|
-
}
|
|
6859
|
-
activeTurn.assistantAudioMimeType = chunk.contentType;
|
|
6860
|
-
activeTurn.assistantAudioSampleRate = chunk.sampleRate;
|
|
6861
7445
|
job.frames = job.frames.then(async () => {
|
|
7446
|
+
// Pace audio before entering the shared outbound queue so control frames
|
|
7447
|
+
// and cancellation remain immediate while prefetched speech waits.
|
|
7448
|
+
const leadMs = this.assistantPlaybackTailUntilMs - Date.now();
|
|
7449
|
+
if (
|
|
7450
|
+
leadMs > TTS_MAX_PLAYBACK_LEAD_MS &&
|
|
7451
|
+
!activeTurn.abortController.signal.aborted
|
|
7452
|
+
) {
|
|
7453
|
+
try {
|
|
7454
|
+
await delay(leadMs - TTS_MAX_PLAYBACK_LEAD_MS, undefined, {
|
|
7455
|
+
signal: activeTurn.abortController.signal,
|
|
7456
|
+
});
|
|
7457
|
+
} catch (error) {
|
|
7458
|
+
if (activeTurn.abortController.signal.aborted) {
|
|
7459
|
+
return;
|
|
7460
|
+
}
|
|
7461
|
+
throw error;
|
|
7462
|
+
}
|
|
7463
|
+
}
|
|
6862
7464
|
const sent = await this.sendFrame(
|
|
6863
7465
|
{
|
|
6864
7466
|
type: "tts_audio",
|
|
@@ -6868,13 +7470,21 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
6868
7470
|
},
|
|
6869
7471
|
() => this.isForwardingTts(token),
|
|
6870
7472
|
);
|
|
6871
|
-
// Skip a frame that wasn't actually written
|
|
7473
|
+
// Skip a frame that wasn't actually written. A backed-up outbound
|
|
6872
7474
|
// queue hasn't reached the client, so it must not extend the
|
|
6873
7475
|
// playback-tail estimate or latch first-audio state. Token match keeps
|
|
6874
7476
|
// a stale turn's late send from latching a newer turn.
|
|
6875
7477
|
if (!sent) {
|
|
6876
7478
|
return;
|
|
6877
7479
|
}
|
|
7480
|
+
if (this.archiveAudio) {
|
|
7481
|
+
activeTurn.assistantAudioChunks.push(
|
|
7482
|
+
Buffer.from(chunk.dataBase64, "base64"),
|
|
7483
|
+
);
|
|
7484
|
+
}
|
|
7485
|
+
activeTurn.assistantAudioMimeType = chunk.contentType;
|
|
7486
|
+
activeTurn.assistantAudioSampleRate = chunk.sampleRate;
|
|
7487
|
+
job.audioSent = true;
|
|
6878
7488
|
// Extend the client playback-tail estimate by this chunk's PCM
|
|
6879
7489
|
// duration (chunks queue gaplessly client-side, so the tail grows
|
|
6880
7490
|
// from whichever is later: now or the current estimate).
|
|
@@ -6914,6 +7524,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
6914
7524
|
this.markUtteranceMetric(utterance, "speechStartAtMs", (turnId) =>
|
|
6915
7525
|
this.metrics.markSpeechStart(turnId),
|
|
6916
7526
|
);
|
|
7527
|
+
this.logInputDiagnostic("voice_input_speech_routed", {
|
|
7528
|
+
inputTurnId: utterance.turnId,
|
|
7529
|
+
});
|
|
6917
7530
|
}
|
|
6918
7531
|
|
|
6919
7532
|
/**
|
|
@@ -6956,6 +7569,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
6956
7569
|
}
|
|
6957
7570
|
|
|
6958
7571
|
private markFirstPartial(utterance: UtteranceCycle): void {
|
|
7572
|
+
if ((utterance.latestPartialText?.trim() ?? "").length === 0) {
|
|
7573
|
+
return;
|
|
7574
|
+
}
|
|
6959
7575
|
this.markUtteranceMetric(utterance, "firstPartialAtMs", (turnId) =>
|
|
6960
7576
|
this.metrics.markFirstPartial(turnId),
|
|
6961
7577
|
);
|
|
@@ -6965,6 +7581,12 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
6965
7581
|
this.markUtteranceMetric(utterance, "finalTranscriptAtMs", (turnId) =>
|
|
6966
7582
|
this.metrics.markFinalTranscript(turnId),
|
|
6967
7583
|
);
|
|
7584
|
+
this.logInputDiagnostic("voice_input_transcript", {
|
|
7585
|
+
inputTurnId: utterance.turnId,
|
|
7586
|
+
sttProvider: utterance.dialedSttProvider,
|
|
7587
|
+
transcriptChars: utterance.finalTranscriptSegments.join(" ").trim()
|
|
7588
|
+
.length,
|
|
7589
|
+
});
|
|
6968
7590
|
}
|
|
6969
7591
|
|
|
6970
7592
|
// Records the mark on the utterance's metrics turn, or — while a previous
|
|
@@ -7105,6 +7727,14 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
7105
7727
|
utterance: UtteranceCycle,
|
|
7106
7728
|
reason: string,
|
|
7107
7729
|
): Promise<void> {
|
|
7730
|
+
this.logInputDiagnostic("voice_input_utterance_discarded", {
|
|
7731
|
+
inputTurnId: utterance.turnId,
|
|
7732
|
+
reason,
|
|
7733
|
+
sttProvider: utterance.dialedSttProvider,
|
|
7734
|
+
transcriptChars: utterance.finalTranscriptSegments.join(" ").trim()
|
|
7735
|
+
.length,
|
|
7736
|
+
hasPartialAtDiscard: utterance.latestPartialText !== null,
|
|
7737
|
+
});
|
|
7108
7738
|
// An utterance that finalizes here never became a turn (empty transcript,
|
|
7109
7739
|
// client interrupt, transcriber close, error), so it ends the window a
|
|
7110
7740
|
// barge-in's merge context was waiting to attach to. Drop that context so
|
|
@@ -7112,13 +7742,6 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
7112
7742
|
// finalizes through finalizeAssistantTurn, not here, so this never clears a
|
|
7113
7743
|
// request that the barge-in follow-up turn is still about to consume.
|
|
7114
7744
|
this.pendingInterruptedRequest = null;
|
|
7115
|
-
// `pendingContinuationResult` is deliberately NOT cleared here. Unlike the
|
|
7116
|
-
// merge context (bound to the immediate barge-in follow-up), a completed
|
|
7117
|
-
// continuation's answer targets the next REAL turn, and hands-free server-VAD
|
|
7118
|
-
// routinely discards noise/empty-transcript utterances between turns; wiping
|
|
7119
|
-
// it on each such discard would frequently drop a valid result before the
|
|
7120
|
-
// user's next question. It is bounded instead by consume-once, the hard-stop
|
|
7121
|
-
// clear (abortDetachedRuns), and the model's "use only if relevant" framing.
|
|
7122
7745
|
utterance.completed = true;
|
|
7123
7746
|
const turnId = utterance.turnId;
|
|
7124
7747
|
if (!turnId) {
|
|
@@ -7147,6 +7770,42 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
7147
7770
|
}
|
|
7148
7771
|
|
|
7149
7772
|
turn.finalized = true;
|
|
7773
|
+
const reply = turn.ttsJobs
|
|
7774
|
+
.filter((job) => job.isReply && job.audioSent)
|
|
7775
|
+
.map((job) => job.text)
|
|
7776
|
+
.join(" ");
|
|
7777
|
+
if (turn.taskOutcome !== null) {
|
|
7778
|
+
if (
|
|
7779
|
+
status === "completed" &&
|
|
7780
|
+
!turn.ttsFailed &&
|
|
7781
|
+
(turn.notificationHandledSilently || reply.length > 0)
|
|
7782
|
+
) {
|
|
7783
|
+
this.taskOutcomes.finish(
|
|
7784
|
+
turn.taskOutcome,
|
|
7785
|
+
turn.notificationHandledSilently
|
|
7786
|
+
? Date.now()
|
|
7787
|
+
: this.assistantPlaybackTailUntilMs,
|
|
7788
|
+
turn.notificationHandledSilently ? "" : reply,
|
|
7789
|
+
);
|
|
7790
|
+
} else {
|
|
7791
|
+
if (
|
|
7792
|
+
reason === "barge_in" ||
|
|
7793
|
+
reason === "interrupt" ||
|
|
7794
|
+
reason === "session_closed"
|
|
7795
|
+
) {
|
|
7796
|
+
this.taskOutcomes.interrupt(turn.taskOutcome);
|
|
7797
|
+
}
|
|
7798
|
+
if (reason !== "barge_in") {
|
|
7799
|
+
this.taskAnnouncementsDeferred = true;
|
|
7800
|
+
}
|
|
7801
|
+
}
|
|
7802
|
+
} else if (status === "completed" && !turn.ttsFailed && reply.length > 0) {
|
|
7803
|
+
this.taskOutcomes.finishContextReply(
|
|
7804
|
+
turn.contextOutcomes,
|
|
7805
|
+
this.assistantPlaybackTailUntilMs,
|
|
7806
|
+
reply,
|
|
7807
|
+
);
|
|
7808
|
+
}
|
|
7150
7809
|
this.clearFillerTimers(turn);
|
|
7151
7810
|
if (
|
|
7152
7811
|
this.foregroundTaskState?.phase === "owned" &&
|
|
@@ -7169,6 +7828,11 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
7169
7828
|
: {}),
|
|
7170
7829
|
});
|
|
7171
7830
|
await this.finishMetricsTurn(turn.utterance, status, reason, turn.turnId);
|
|
7831
|
+
this.releaseModeSessionRequests(turn);
|
|
7832
|
+
|
|
7833
|
+
if (status === "cancelled") {
|
|
7834
|
+
this.releaseModeSessionDelivery(turn);
|
|
7835
|
+
}
|
|
7172
7836
|
|
|
7173
7837
|
if (
|
|
7174
7838
|
(options.clearActive ?? true) &&
|
|
@@ -7183,6 +7847,36 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
7183
7847
|
}
|
|
7184
7848
|
}
|
|
7185
7849
|
|
|
7850
|
+
private releaseModeSessionDelivery(turn: ActiveAssistantTurn): void {
|
|
7851
|
+
const turnId = turn.modeSessionDeliveryTurnId;
|
|
7852
|
+
if (turnId === null) {
|
|
7853
|
+
return;
|
|
7854
|
+
}
|
|
7855
|
+
turn.modeSessionDeliveryTurnId = null;
|
|
7856
|
+
this.releaseModeSessionTurnId(turnId);
|
|
7857
|
+
}
|
|
7858
|
+
|
|
7859
|
+
private releaseModeSessionRequests(turn: ActiveAssistantTurn): void {
|
|
7860
|
+
this.releaseModeSessionTurnIds(turn.modeSessionRequestIds.splice(0));
|
|
7861
|
+
}
|
|
7862
|
+
|
|
7863
|
+
private releaseModeSessionTurnIds(turnIds: readonly string[]): void {
|
|
7864
|
+
for (const turnId of turnIds) {
|
|
7865
|
+
this.releaseModeSessionTurnId(turnId);
|
|
7866
|
+
}
|
|
7867
|
+
}
|
|
7868
|
+
|
|
7869
|
+
private releaseModeSessionTurnId(turnId: string): void {
|
|
7870
|
+
try {
|
|
7871
|
+
this.cameraModeSessions?.releaseDelivery(turnId);
|
|
7872
|
+
} catch (err) {
|
|
7873
|
+
log.warn(
|
|
7874
|
+
{ err, turnId },
|
|
7875
|
+
"Could not release camera session ownership from voice turn",
|
|
7876
|
+
);
|
|
7877
|
+
}
|
|
7878
|
+
}
|
|
7879
|
+
|
|
7186
7880
|
private async archiveBufferedAudio(input: {
|
|
7187
7881
|
turnId: string;
|
|
7188
7882
|
userMessageId: string | null;
|
|
@@ -7330,7 +8024,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
7330
8024
|
}
|
|
7331
8025
|
|
|
7332
8026
|
private async emitMetricsFrame(
|
|
7333
|
-
event:
|
|
8027
|
+
event: VoiceMetricsEvent,
|
|
7334
8028
|
turnId = this.currentUtterance?.turnId ?? this.context.sessionId,
|
|
7335
8029
|
): Promise<void> {
|
|
7336
8030
|
const metrics = this.metrics.getSnapshot();
|
|
@@ -7341,7 +8035,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
7341
8035
|
conversationId: this.conversationId,
|
|
7342
8036
|
turnId,
|
|
7343
8037
|
metrics,
|
|
7344
|
-
...
|
|
8038
|
+
...getVoiceMetricsAggregateFields(metrics, turnId),
|
|
7345
8039
|
});
|
|
7346
8040
|
}
|
|
7347
8041
|
|
|
@@ -7473,6 +8167,8 @@ export function createLiveVoiceSession(
|
|
|
7473
8167
|
options.spawnBackgroundContinuation ?? defaultSpawnBackgroundContinuation,
|
|
7474
8168
|
labelBackgroundContinuation:
|
|
7475
8169
|
options.labelBackgroundContinuation ?? createContinuationLabeler(),
|
|
8170
|
+
judgeBackgroundContinuation:
|
|
8171
|
+
options.judgeBackgroundContinuation ?? createContinuationJudge(),
|
|
7476
8172
|
getTurnTeardown: options.getTurnTeardown ?? getConversationTurnTeardown,
|
|
7477
8173
|
// Off by default (see the `liveVoice.archiveAudio` schema): voice turns
|
|
7478
8174
|
// persist only their transcribed text, so the recorded audio never lands
|
|
@@ -7802,7 +8498,7 @@ async function defaultArchiveLiveVoiceAudio(
|
|
|
7802
8498
|
: linkLiveVoiceAssistantResponseAudioToMessage(input);
|
|
7803
8499
|
}
|
|
7804
8500
|
|
|
7805
|
-
function toSeedMarks(stashed: StashedMetricsMarks):
|
|
8501
|
+
function toSeedMarks(stashed: StashedMetricsMarks): VoiceTurnSeedMarks {
|
|
7806
8502
|
return {
|
|
7807
8503
|
...(stashed.firstAudioAtMs !== null
|
|
7808
8504
|
? { firstAudioAtMs: stashed.firstAudioAtMs }
|