@vellumai/assistant 0.11.2 → 0.11.3-dev.202608080427.92f668a
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +69 -9
- package/docs/flux-turn-detection-spike.md +243 -0
- package/docs/stt-provider-onboarding.md +3 -1
- package/docs/trusted-contact-access.md +9 -5
- package/docs/vellum-doctor.md +195 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/channels.ts +50 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/ingress.ts +10 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/remote-web-pairing.ts +6 -0
- package/node_modules/@vellumai/ces-client/src/index.ts +1 -0
- package/node_modules/@vellumai/ces-client/src/rpc-client.ts +19 -2
- package/node_modules/@vellumai/environments/package.json +2 -1
- package/node_modules/@vellumai/environments/src/__tests__/cloud-assistant-hub-url.test.ts +38 -0
- package/node_modules/@vellumai/environments/src/__tests__/install-layout.test.ts +98 -0
- package/node_modules/@vellumai/environments/src/__tests__/package-boundary.test.ts +5 -5
- package/node_modules/@vellumai/environments/src/index.ts +17 -6
- package/node_modules/@vellumai/environments/src/install-layout.ts +49 -0
- package/node_modules/@vellumai/environments/src/seeds.ts +29 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/channels.ts +50 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/ingress.ts +10 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/remote-web-pairing.ts +6 -0
- package/node_modules/@vellumai/gateway-client/src/index.ts +4 -0
- package/node_modules/@vellumai/gateway-client/src/verification-session-contract.ts +91 -6
- package/node_modules/@vellumai/ipc-server-utils/src/endpoint.test.ts +36 -0
- package/node_modules/@vellumai/ipc-server-utils/src/endpoint.ts +142 -0
- package/node_modules/@vellumai/ipc-server-utils/src/index.ts +2 -0
- package/node_modules/@vellumai/ipc-server-utils/src/listen-options.ts +3 -0
- package/node_modules/@vellumai/ipc-server-utils/src/socket-watchdog.test.ts +15 -1
- package/node_modules/@vellumai/ipc-server-utils/src/socket-watchdog.ts +17 -2
- package/node_modules/@vellumai/service-contracts/src/channels.ts +50 -0
- package/node_modules/@vellumai/service-contracts/src/ingress.ts +10 -0
- package/node_modules/@vellumai/service-contracts/src/remote-web-pairing.ts +6 -0
- package/node_modules/@vellumai/slack-text/src/index.ts +16 -2
- package/openapi.yaml +619 -47
- package/package.json +1 -1
- package/scripts/voice-ttft-spike.ts +3 -3
- package/src/__tests__/agent-image-optimize.test.ts +4 -4
- package/src/__tests__/agent-loop-mutable-latest-user-message.test.ts +113 -66
- package/src/__tests__/agent-loop-override-profile.test.ts +38 -48
- package/src/__tests__/agent-loop-subagent-billing-headers.test.ts +166 -0
- package/src/__tests__/agent-wake-disk-pressure-callsite.test.ts +1 -0
- package/src/__tests__/agent-wake-override-profile.test.ts +1 -0
- package/src/__tests__/answered-question-persistence.test.ts +240 -0
- package/src/__tests__/anthropic-provider.test.ts +165 -30
- package/src/__tests__/app-control-flow.test.ts +9 -8
- package/src/__tests__/assistant-event-hub.test.ts +48 -0
- package/src/__tests__/attachments-store-heic-normalize.test.ts +16 -12
- package/src/__tests__/attachments-store.test.ts +130 -68
- package/src/__tests__/attachments.test.ts +36 -36
- package/src/__tests__/background-workers-disk-pressure.test.ts +1 -1
- package/src/__tests__/call-controller.test.ts +120 -0
- package/src/__tests__/call-start-guardian-guard.test.ts +18 -4
- package/src/__tests__/canned-reply-release.test.ts +86 -0
- package/src/__tests__/channel-availability-routes.test.ts +14 -1
- package/src/__tests__/channel-guardian.test.ts +59 -29
- package/src/__tests__/channel-readiness-telegram-remote.test.ts +154 -0
- package/src/__tests__/channel-setup-panel-ack.test.ts +6 -5
- package/src/__tests__/client-os-metadata-persistence.test.ts +103 -4
- package/src/__tests__/compactor-image-manifest-trust.test.ts +1 -1
- package/src/__tests__/config-loader-backfill.test.ts +19 -0
- package/src/__tests__/config-schema.test.ts +150 -7
- package/src/__tests__/consult-deadline.test.ts +65 -0
- package/src/__tests__/conversation-agent-loop.test.ts +438 -2
- package/src/__tests__/conversation-disk-view-integration.test.ts +5 -1
- package/src/__tests__/conversation-disk-view.test.ts +4 -4
- package/src/__tests__/conversation-error.test.ts +22 -3
- package/src/__tests__/conversation-fork-crud.test.ts +5 -1
- package/src/__tests__/conversation-fork-referential.test.ts +352 -0
- package/src/__tests__/conversation-fork-retrospective.test.ts +5 -1
- package/src/__tests__/conversation-group-tools.test.ts +32 -2
- package/src/__tests__/conversation-inference-profile-route.test.ts +27 -1
- package/src/__tests__/conversation-initial-prompt.test.ts +51 -22
- package/src/__tests__/conversation-key-store-bootstrap-cleanup.test.ts +119 -0
- package/src/__tests__/conversation-key-store-origin.test.ts +105 -0
- package/src/__tests__/conversation-list-recency-ordering.test.ts +104 -0
- package/src/__tests__/conversation-origin-at-creation.test.ts +153 -0
- package/src/__tests__/conversation-origin-channel-filter.test.ts +95 -0
- package/src/__tests__/conversation-placement-promotion.test.ts +172 -0
- package/src/__tests__/conversation-process-app-control-preactivation.test.ts +123 -0
- package/src/__tests__/conversation-queue.test.ts +433 -1
- package/src/__tests__/conversation-retry-route.test.ts +3 -1
- package/src/__tests__/conversation-routes-hidden-queue.test.ts +64 -1
- package/src/__tests__/conversation-store-ephemeral.test.ts +4 -0
- package/src/__tests__/conversation-store.test.ts +26 -10
- package/src/__tests__/conversation-summarize-route.test.ts +1 -1
- package/src/__tests__/conversation-surfaces-action-delivery.test.ts +15 -6
- package/src/__tests__/conversation-surfaces-activation-emit.test.ts +6 -5
- package/src/__tests__/conversation-surfaces-app-control.test.ts +12 -11
- package/src/__tests__/conversation-surfaces-app-open.test.ts +11 -5
- package/src/__tests__/conversation-surfaces-data-persist.test.ts +5 -4
- package/src/__tests__/conversation-surfaces-history-restored-completion.test.ts +967 -0
- package/src/__tests__/conversation-surfaces-persisted-info.test.ts +312 -0
- package/src/__tests__/conversation-surfaces-queued-emit.test.ts +6 -5
- package/src/__tests__/conversation-surfaces-standalone-payloads.test.ts +12 -8
- package/src/__tests__/conversation-surfaces-standalone.test.ts +21 -11
- package/src/__tests__/conversation-surfaces-state-update.test.ts +7 -11
- package/src/__tests__/conversation-surfaces-table-action.test.ts +7 -6
- package/src/__tests__/conversation-surfaces-task-progress.test.ts +11 -6
- package/src/__tests__/conversation-tool-setup-app-refresh.test.ts +9 -6
- package/src/__tests__/conversation-tool-setup-attribution.test.ts +70 -7
- package/src/__tests__/conversation-wait-for-idle.test.ts +6 -11
- package/src/__tests__/credential-prompt-route.test.ts +54 -7
- package/src/__tests__/credential-security-invariants.test.ts +1 -0
- package/src/__tests__/cu-unified-flow.test.ts +89 -28
- package/src/__tests__/custom-group-listing-visibility.test.ts +1 -1
- package/src/__tests__/db-schedule-syntax-migration.test.ts +4 -1
- package/src/__tests__/discord-access-request-privacy.test.ts +366 -0
- package/src/__tests__/discord-requester-notice-privacy.test.ts +87 -0
- package/src/__tests__/disk-pressure-guard.test.ts +55 -55
- package/src/__tests__/disk-pressure-routes.test.ts +10 -10
- package/src/__tests__/disk-pressure-tools.test.ts +2 -2
- package/src/__tests__/disk-usage.test.ts +35 -20
- package/src/__tests__/document-sync-tags.test.ts +379 -0
- package/src/__tests__/document-update-default-surface.test.ts +28 -0
- package/src/__tests__/dynamic-skill-background-guard.test.ts +2 -2
- package/src/__tests__/emit-signal-routing-intent.test.ts +3 -4
- package/src/__tests__/ensure-conversation-exists.test.ts +8 -5
- package/src/__tests__/events-dev-bypass-actor.test.ts +294 -1
- package/src/__tests__/evict-conversations-for-reload.test.ts +90 -0
- package/src/__tests__/file-list-tool.test.ts +12 -10
- package/src/__tests__/file-ops-service.test.ts +151 -44
- package/src/__tests__/file-read-container-boundary.test.ts +137 -0
- package/src/__tests__/file-read-tool.test.ts +21 -9
- package/src/__tests__/file-url-path-guard.test.ts +83 -0
- package/src/__tests__/filesystem-tools.test.ts +82 -66
- package/src/__tests__/guardian-outbound-http.test.ts +8 -8
- package/src/__tests__/helpers/gateway-verification-sessions-stub.ts +14 -1
- package/src/__tests__/helpers/hub-clients.ts +42 -0
- package/src/__tests__/helpers/mock-conversation.ts +46 -0
- package/src/__tests__/helpers/verification-sessions-ipc-sim.ts +61 -57
- package/src/__tests__/history-repair.test.ts +112 -12
- package/src/__tests__/host-bash-proxy.test.ts +10 -5
- package/src/__tests__/host-bash-routes.test.ts +67 -0
- package/src/__tests__/host-browser-proxy.test.ts +2 -2
- package/src/__tests__/host-cu-proxy.test.ts +3 -7
- package/src/__tests__/host-file-proxy.test.ts +3 -7
- package/src/__tests__/host-file-read-tool.test.ts +5 -1
- package/src/__tests__/host-transfer-proxy-targeted.test.ts +6 -4
- package/src/__tests__/http-conversation-lineage.test.ts +2 -5
- package/src/__tests__/http-user-message-parity.test.ts +44 -0
- package/src/__tests__/identity-routes.test.ts +13 -13
- package/src/__tests__/image-conversion.test.ts +29 -26
- package/src/__tests__/image-source-path-reinject.test.ts +2 -2
- package/src/__tests__/inference-profile-session-handler.test.ts +95 -0
- package/src/__tests__/inference-profile-session-ipc.test.ts +25 -0
- package/src/__tests__/introduction-card-resolver.test.ts +6 -1
- package/src/__tests__/list-messages-attachments.test.ts +56 -32
- package/src/__tests__/list-messages-background-tool-completion.test.ts +4 -4
- package/src/__tests__/list-messages-client-message-id.test.ts +2 -2
- package/src/__tests__/list-messages-hidden-metadata.test.ts +16 -16
- package/src/__tests__/list-messages-page-latest.test.ts +61 -61
- package/src/__tests__/list-messages-provider-error.test.ts +6 -6
- package/src/__tests__/list-messages-queued.test.ts +49 -6
- package/src/__tests__/list-messages-system-card.test.ts +4 -4
- package/src/__tests__/list-messages-tool-merge.test.ts +9 -9
- package/src/__tests__/llm-resolver-override-or-default.test.ts +141 -0
- package/src/__tests__/llm-usage-store.test.ts +55 -0
- package/src/__tests__/manual-token-reconciliation.test.ts +45 -0
- package/src/__tests__/mcp-list-plugin-servers.test.ts +250 -0
- package/src/__tests__/mcp-tool-annotations-risk.test.ts +133 -0
- package/src/__tests__/media-stream-output.test.ts +175 -0
- package/src/__tests__/media-stream-server-integration.test.ts +36 -0
- package/src/__tests__/media-stream-stt-session.test.ts +67 -0
- package/src/__tests__/migration-export-http.test.ts +1 -1
- package/src/__tests__/migration-import-commit-http.test.ts +1 -1
- package/src/__tests__/migration-import-preflight-http.test.ts +1 -1
- package/src/__tests__/migration-validate-http.test.ts +1 -1
- package/src/__tests__/mtime-cache.test.ts +65 -3
- package/src/__tests__/notification-platform-adapter.test.ts +104 -0
- package/src/__tests__/notification-schedule-notify-dedup.test.ts +1 -1
- package/src/__tests__/oauth-provider-profiles.test.ts +4 -2
- package/src/__tests__/oauth-provider-seed-logos.test.ts +65 -0
- package/src/__tests__/oauth-provider-serializer.test.ts +17 -0
- package/src/__tests__/oauth-providers-routes.test.ts +1 -0
- package/src/__tests__/openai-responses-prompt-cache.test.ts +5 -2
- package/src/__tests__/path-policy.test.ts +164 -1
- package/src/__tests__/persist-media-references.test.ts +2 -2
- package/src/__tests__/persist-unsendable-image-downscale.test.ts +5 -5
- package/src/__tests__/persist-unsendable-image.test.ts +23 -23
- package/src/__tests__/persona-resolver.test.ts +154 -0
- package/src/__tests__/plugin-api-webhook-url.test.ts +170 -0
- package/src/__tests__/plugin-import-boundary-guard.test.ts +6 -0
- package/src/__tests__/processing-flag-persist-failure.test.ts +15 -10
- package/src/__tests__/profile-availability-incomplete.test.ts +113 -0
- package/src/__tests__/profiler-routes.test.ts +13 -13
- package/src/__tests__/prompt-cache-cross-turn-stability.test.ts +593 -0
- package/src/__tests__/proxy-approval-callback.test.ts +3 -3
- package/src/__tests__/queued-message-cancel-and-steer.test.ts +189 -0
- package/src/__tests__/queued-message-steer-actor-scoping.test.ts +164 -0
- package/src/__tests__/resolve-trust-class.test.ts +20 -0
- package/src/__tests__/run-conversation-turn-persistence.test.ts +1 -1
- package/src/__tests__/run-due-schedules.test.ts +174 -0
- package/src/__tests__/runtime-attachment-metadata.test.ts +15 -7
- package/src/__tests__/scaffold-managed-skill-tool.test.ts +166 -0
- package/src/__tests__/schedule-routes.test.ts +730 -8
- package/src/__tests__/schedule-store.test.ts +540 -0
- package/src/__tests__/schedule-tools.test.ts +103 -0
- package/src/__tests__/scheduler-wake.test.ts +5 -0
- package/src/__tests__/secret-routes-platform-proxy.test.ts +29 -1
- package/src/__tests__/secure-keys.test.ts +91 -0
- package/src/__tests__/server-history-render.test.ts +97 -7
- package/src/__tests__/size-guard.test.ts +8 -8
- package/src/__tests__/skill-docs-sync-guard.test.ts +3 -2
- package/src/__tests__/skill-tool-factory.test.ts +247 -0
- package/src/__tests__/skills-install-staging.test.ts +15 -6
- package/src/__tests__/skills.test.ts +32 -0
- package/src/__tests__/slack-edit-ordering-characterization.test.ts +103 -0
- package/src/__tests__/slack-inbound-verification.test.ts +4 -1
- package/src/__tests__/slack-mention-binding-invariance.test.ts +103 -0
- package/src/__tests__/slack-mention-provider-leak-audit.test.ts +135 -0
- package/src/__tests__/starter-task-flow.test.ts +5 -4
- package/src/__tests__/steer-on-enqueue-question.test.ts +3 -2
- package/src/__tests__/stt-language-catalog-parity.test.ts +38 -0
- package/src/__tests__/subagent-allowlist-validation.test.ts +26 -4
- package/src/__tests__/subagent-cron-run-attribution.test.ts +301 -0
- package/src/__tests__/subagent-disposal.test.ts +10 -0
- package/src/__tests__/subagent-fork-notifications.test.ts +10 -0
- package/src/__tests__/subagent-fork-prompt-role.test.ts +85 -11
- package/src/__tests__/subagent-fork-spawn.test.ts +15 -7
- package/src/__tests__/subagent-manager-notify.test.ts +148 -0
- package/src/__tests__/subagent-notify-parent.test.ts +1 -1
- package/src/__tests__/subagent-persistence.test.ts +141 -0
- package/src/__tests__/subagent-role-registry.test.ts +172 -63
- package/src/__tests__/subagent-role-resolution.test.ts +88 -0
- package/src/__tests__/subagent-spawn-and-await.test.ts +48 -0
- package/src/__tests__/subagent-spawn-tool-fork.test.ts +111 -10
- package/src/__tests__/subagent-terminal-message.test.ts +88 -1
- package/src/__tests__/subagent-tool-gate-mode.test.ts +225 -10
- package/src/__tests__/subagent-tools.test.ts +1901 -108
- package/src/__tests__/surface-completion-compaction-boundary.test.ts +124 -0
- package/src/__tests__/surface-completion-in-flight-snapshot.test.ts +298 -0
- package/src/__tests__/system-prompt.test.ts +82 -7
- package/src/__tests__/tool-grant-request-escalation.test.ts +40 -1
- package/src/__tests__/tool-result-spool.test.ts +74 -18
- package/src/__tests__/tool-side-effects-documents.test.ts +174 -0
- package/src/__tests__/tool-side-effects-hook-failures.test.ts +129 -0
- package/src/__tests__/tools-audio-read.test.ts +10 -10
- package/src/__tests__/tools-get-route.test.ts +85 -12
- package/src/__tests__/tools-list-cli.test.ts +171 -0
- package/src/__tests__/top-level-renderer.test.ts +54 -1
- package/src/__tests__/trusted-contact-inline-approval-integration.test.ts +68 -1
- package/src/__tests__/trusted-contact-multichannel.test.ts +4 -1
- package/src/__tests__/turn-tail-chain.test.ts +135 -0
- package/src/__tests__/ui-choice-copy-surfaces.test.ts +6 -4
- package/src/__tests__/ui-visual-surface.test.ts +5 -4
- package/src/__tests__/ui-work-result-surface.test.ts +5 -4
- package/src/__tests__/unified-turn-context-visible-app.test.ts +7 -8
- package/src/__tests__/usage-routes.test.ts +62 -0
- package/src/__tests__/verification-outbound-delivery.test.ts +176 -0
- package/src/__tests__/visible-app-context.test.ts +20 -4
- package/src/__tests__/web-search-history.test.ts +96 -0
- package/src/__tests__/workspace-migration-139-clear-renamed-cost-profile-label.test.ts +6 -6
- package/src/__tests__/workspace-migration-140-repair-seed-pinned-memory-v3-live.test.ts +330 -0
- package/src/__tests__/workspace-migration-142-consolidate-voice-front-door.test.ts +150 -0
- package/src/acp/failure-error.ts +3 -5
- package/src/agent/attachments.ts +26 -24
- package/src/agent/history-repair/history-repair.ts +27 -26
- package/src/agent/image-optimize.ts +6 -6
- package/src/agent/loop.ts +40 -22
- package/src/agent/message-types.ts +3 -3
- package/src/api/constants/document-tools.ts +55 -0
- package/src/api/events/question-answered.ts +53 -0
- package/src/api/events/tool-result.ts +7 -0
- package/src/api/index.ts +13 -0
- package/src/api/responses/conversation-message.ts +11 -0
- package/src/api/surfaces-oauth-connect.test.ts +96 -0
- package/src/api/surfaces.ts +36 -8
- package/src/approvals/guardian-channel-delivery.ts +83 -0
- package/src/approvals/guardian-expiry-notifier.ts +13 -11
- package/src/approvals/guardian-request-resolvers.ts +222 -135
- package/src/calls/__tests__/tts-text-sanitizer.test.ts +13 -0
- package/src/calls/__tests__/voice-session-bridge.test.ts +131 -12
- package/src/calls/__tests__/voice-triage-escalate.test.ts +106 -0
- package/src/calls/call-controller.ts +30 -2
- package/src/calls/call-speech-output.ts +12 -4
- package/src/calls/call-transport.ts +18 -1
- package/src/calls/media-stream-output.ts +53 -5
- package/src/calls/media-stream-server.ts +21 -1
- package/src/calls/media-stream-stt-session.ts +49 -0
- package/src/calls/telephony-synthesis-language.ts +84 -0
- package/src/calls/tts-text-sanitizer.ts +12 -5
- package/src/calls/voice-session-bridge.ts +86 -17
- package/src/calls/voice-triage-escalate.ts +54 -7
- package/src/channels/__tests__/gateway-verification-sessions.test.ts +3 -3
- package/src/channels/__tests__/plugin-channel-declarations.test.ts +161 -0
- package/src/channels/config.ts +20 -2
- package/src/channels/gateway-verification-sessions.ts +7 -2
- package/src/channels/plugin-channel-declarations.ts +108 -0
- package/src/channels/types.ts +37 -1
- package/src/cli/AGENTS.md +5 -2
- package/src/cli/commands/__tests__/cache.test.ts +7 -4
- package/src/cli/commands/__tests__/cli-test-harness.ts +82 -0
- package/src/cli/commands/__tests__/conversations-wake.test.ts +51 -6
- package/src/cli/commands/__tests__/inference-models.test.ts +14 -0
- package/src/cli/commands/__tests__/inference-profiles.test.ts +44 -1
- package/src/cli/commands/__tests__/inference-providers.test.ts +7 -0
- package/src/cli/commands/__tests__/plugins.test.ts +692 -0
- package/src/cli/commands/__tests__/schedules.test.ts +136 -37
- package/src/cli/commands/conversations.ts +7 -3
- package/src/cli/commands/credentials.help.ts +7 -0
- package/src/cli/commands/inference-profiles.ts +9 -3
- package/src/cli/commands/inference.help.ts +16 -2
- package/src/cli/commands/mcp.help.ts +17 -6
- package/src/cli/commands/mcp.ts +9 -0
- package/src/cli/commands/memory/memory-retrospective.ts +18 -0
- package/src/cli/commands/notifications.ts +3 -6
- package/src/cli/commands/oauth/connect-surface-guidance.test.ts +16 -0
- package/src/cli/commands/oauth/connect-surface-guidance.ts +25 -7
- package/src/cli/commands/oauth/connect.test.ts +26 -1
- package/src/cli/commands/oauth/connect.ts +4 -1
- package/src/cli/commands/oauth/index.help.ts +3 -1
- package/src/cli/commands/oauth/providers.ts +3 -0
- package/src/cli/commands/plugins.help.ts +25 -2
- package/src/cli/commands/plugins.ts +289 -10
- package/src/cli/commands/schedules.help.ts +14 -12
- package/src/cli/commands/schedules.ts +19 -5
- package/src/cli/commands/stt.help.ts +27 -2
- package/src/cli/commands/tools.help.ts +4 -2
- package/src/cli/commands/tools.ts +38 -0
- package/src/cli/lib/__tests__/inspect-plugin.test.ts +1 -0
- package/src/cli/lib/__tests__/install-from-github.test.ts +79 -0
- package/src/cli/lib/__tests__/install-from-platform.test.ts +47 -1
- package/src/cli/lib/__tests__/plugin-surfaces.test.ts +86 -7
- package/src/cli/lib/__tests__/upgrade-plugin.test.ts +161 -0
- package/src/cli/lib/install-from-github.ts +61 -0
- package/src/cli/lib/install-from-platform.ts +13 -2
- package/src/cli/lib/plugin-surfaces.ts +124 -4
- package/src/cli/lib/upgrade-plugin.ts +60 -0
- package/src/cli.ts +1 -1
- package/src/config/__tests__/default-profile-catalog.test.ts +11 -1
- package/src/config/__tests__/deployment-context-defaults.test.ts +73 -0
- package/src/config/__tests__/memory-retrospective-schema.test.ts +40 -0
- package/src/config/__tests__/plugin-updates-schema.test.ts +43 -0
- package/src/config/__tests__/profile-tool-support.test.ts +131 -0
- package/src/config/bundled-skills/AGENTS.md +1 -1
- package/src/config/bundled-skills/acp/SKILL.md +6 -7
- package/src/config/bundled-skills/app-builder/SKILL.md +1 -1
- package/src/config/bundled-skills/app-builder/references/RESPONSIVE.md +3 -3
- package/src/config/bundled-skills/media-processing/services/preprocess.ts +14 -4
- package/src/config/bundled-skills/phone-calls/references/CONFIG.md +15 -15
- package/src/config/bundled-skills/schedule/SKILL.md +5 -1
- package/src/config/bundled-skills/schedule/TOOLS.json +2 -2
- package/src/config/bundled-skills/settings/TOOLS.json +3 -3
- package/src/config/bundled-skills/settings/tools/voice-config-update.ts +1 -0
- package/src/config/bundled-skills/subagent/SKILL.md +53 -22
- package/src/config/bundled-skills/subagent/TOOLS.json +14 -13
- package/src/config/bundled-skills/transcribe/tools/transcribe-media.test.ts +22 -1
- package/src/config/bundled-skills/transcribe/tools/transcribe-media.ts +9 -2
- package/src/config/bundled-skills/visualize/SKILL.md +1 -1
- package/src/config/call-site-defaults.ts +4 -5
- package/src/config/default-profile-catalog.ts +9 -5
- package/src/config/feature-flag-registry.json +25 -17
- package/src/config/llm-resolver.ts +45 -15
- package/src/config/loader.ts +75 -1
- package/src/config/profile-tool-support.ts +110 -0
- package/src/config/sanitize-for-transfer.ts +11 -0
- package/src/config/schema.ts +4 -0
- package/src/config/schemas/__tests__/live-voice.test.ts +142 -4
- package/src/config/schemas/__tests__/stt.test.ts +20 -5
- package/src/config/schemas/call-site-catalog.ts +4 -4
- package/src/config/schemas/calls.ts +0 -4
- package/src/config/schemas/channels.ts +13 -0
- package/src/config/schemas/live-voice.ts +82 -23
- package/src/config/schemas/llm.ts +1 -1
- package/src/config/schemas/memory-retrospective.ts +25 -0
- package/src/config/schemas/plugin-updates.ts +59 -0
- package/src/config/schemas/services.ts +8 -4
- package/src/config/schemas/stt.ts +18 -5
- package/src/config/schemas/timeouts.ts +1 -1
- package/src/config/schemas/tts.ts +63 -0
- package/src/contacts/__tests__/guardian-delivery-reader.test.ts +16 -0
- package/src/contacts/guardian-delivery-reader.ts +29 -2
- package/src/context/compactor.ts +4 -4
- package/src/context/post-turn-tool-result-truncation.ts +32 -16
- package/src/context/strip-injections.ts +42 -0
- package/src/context/tool-result-spool.ts +32 -18
- package/src/credential-execution/ces-runtime.ts +3 -0
- package/src/credential-execution/client.ts +22 -1
- package/src/credential-execution/process-manager.test.ts +99 -0
- package/src/credential-execution/process-manager.ts +62 -16
- package/src/daemon/__tests__/abort-null-controller.test.ts +141 -5
- package/src/daemon/__tests__/conversation-surfaces-launch.test.ts +8 -12
- package/src/daemon/__tests__/install-assistant-command.test.ts +234 -0
- package/src/daemon/__tests__/turn-tail-assistant-reply-notify.test.ts +2 -4
- package/src/daemon/__tests__/turn-tail-deleted-conversation.test.ts +20 -29
- package/src/daemon/channel-ui-capability.ts +4 -4
- package/src/daemon/conversation-agent-loop-handlers.ts +117 -61
- package/src/daemon/conversation-agent-loop.ts +318 -148
- package/src/daemon/conversation-attachments.ts +1 -1
- package/src/daemon/conversation-error.ts +8 -1
- package/src/daemon/conversation-initial-prompt.ts +16 -38
- package/src/daemon/conversation-lifecycle.ts +154 -32
- package/src/daemon/conversation-messaging.ts +106 -43
- package/src/daemon/conversation-process.ts +131 -33
- package/src/daemon/conversation-queue-manager.ts +10 -0
- package/src/daemon/conversation-runtime-assembly.ts +9 -4
- package/src/daemon/conversation-store.ts +52 -3
- package/src/daemon/conversation-surface-state.ts +9 -0
- package/src/daemon/conversation-surfaces.ts +403 -210
- package/src/daemon/conversation-tool-setup.ts +56 -7
- package/src/daemon/conversation-turn-finalize.ts +134 -89
- package/src/daemon/conversation.ts +111 -27
- package/src/daemon/disk-pressure-guard-lifecycle.ts +2 -2
- package/src/daemon/disk-pressure-guard.ts +3 -3
- package/src/daemon/doordash-steps.ts +5 -5
- package/src/daemon/external-plugins-bootstrap.ts +7 -1
- package/src/daemon/handlers/config-channels.ts +48 -5
- package/src/daemon/handlers/conversation-history.ts +1 -1
- package/src/daemon/handlers/conversations.ts +62 -17
- package/src/daemon/handlers/shared.ts +12 -0
- package/src/daemon/handlers/skills.ts +11 -1
- package/src/daemon/identity-helpers.ts +38 -10
- package/src/daemon/install-assistant-command.ts +322 -0
- package/src/daemon/lifecycle.ts +14 -3
- package/src/daemon/message-types/conversations.ts +6 -7
- package/src/daemon/message-types/sync.ts +1 -0
- package/src/daemon/persist-media-references.ts +23 -14
- package/src/daemon/process-message.ts +6 -6
- package/src/daemon/tool-setup-types.ts +19 -122
- package/src/daemon/tool-side-effects.ts +17 -94
- package/src/daemon/trust-context-types.ts +39 -0
- package/src/daemon/trust-context.ts +14 -6
- package/src/daemon/turn-tail-chain.ts +73 -0
- package/src/daemon/web-search-history.ts +47 -33
- package/src/heartbeat/heartbeat-service.ts +6 -6
- package/src/inbound/__tests__/oauth-callback-url.test.ts +83 -0
- package/src/inbound/oauth-callback-url.ts +61 -0
- package/src/ipc/__tests__/socket-path.test.ts +21 -0
- package/src/ipc/assistant-server.ts +17 -10
- package/src/ipc/cli-client.ts +10 -1
- package/src/ipc/routes/__tests__/documents-sync-ipc-routes.test.ts +56 -0
- package/src/ipc/routes/documents-sync-ipc-routes.ts +42 -0
- package/src/ipc/socket-cleanup.ts +9 -7
- package/src/ipc/socket-path.ts +4 -96
- package/src/live-voice/__tests__/live-voice-agent-turn.test.ts +1 -104
- package/src/live-voice/__tests__/live-voice-archive.test.ts +16 -16
- package/src/live-voice/__tests__/live-voice-attach-image.test.ts +209 -0
- package/src/live-voice/__tests__/live-voice-connection.test.ts +20 -0
- package/src/live-voice/__tests__/live-voice-events.test.ts +7 -8
- package/src/live-voice/__tests__/live-voice-flux-turn-end.test.ts +932 -0
- package/src/live-voice/__tests__/live-voice-integration.test.ts +12 -5
- package/src/live-voice/__tests__/live-voice-metrics.test.ts +115 -8
- package/src/live-voice/__tests__/live-voice-photo.test.ts +100 -0
- package/src/live-voice/__tests__/live-voice-progress.test.ts +179 -146
- package/src/live-voice/__tests__/live-voice-session-telemetry.test.ts +265 -0
- package/src/live-voice/__tests__/live-voice-stt.test.ts +324 -2
- package/src/live-voice/__tests__/live-voice-triage-escalate.test.ts +131 -2
- package/src/live-voice/__tests__/live-voice-tts-session.test.ts +0 -483
- package/src/live-voice/__tests__/live-voice-tts.test.ts +148 -2
- package/src/live-voice/__tests__/live-voice-vad.test.ts +313 -32
- package/src/live-voice/__tests__/progress-narration.test.ts +214 -0
- package/src/live-voice/__tests__/progress-phrases.test.ts +167 -0
- package/src/live-voice/__tests__/protocol.test.ts +52 -0
- package/src/live-voice/__tests__/runtime-websocket-shell.test.ts +3 -0
- package/src/live-voice/live-voice-archive.ts +13 -13
- package/src/live-voice/live-voice-connection.ts +7 -1
- package/src/live-voice/live-voice-metrics.ts +57 -32
- package/src/live-voice/live-voice-photo.ts +168 -0
- package/src/live-voice/live-voice-session.ts +1137 -351
- package/src/live-voice/live-voice-tts.ts +18 -2
- package/src/live-voice/progress-narration.ts +277 -0
- package/src/live-voice/progress-phrases.ts +105 -2
- package/src/live-voice/protocol.ts +110 -2
- package/src/mcp/__tests__/mcp-auth-orchestrator.test.ts +0 -1
- package/src/mcp/__tests__/mcp-oauth-client-registration.test.ts +196 -0
- package/src/mcp/__tests__/mcp-oauth-provider.test.ts +9 -9
- package/src/mcp/client.ts +14 -7
- package/src/mcp/mcp-auth-orchestrator.ts +13 -22
- package/src/mcp/mcp-oauth-provider.ts +195 -240
- package/src/messaging/providers/__tests__/callback-routing.test.ts +5 -1
- package/src/messaging/providers/__tests__/transport-dispatch.test.ts +66 -4
- package/src/messaging/providers/callback-routing.ts +1 -0
- package/src/messaging/providers/discord/api.ts +312 -0
- package/src/messaging/providers/discord/dm-delivery.test.ts +204 -0
- package/src/messaging/providers/discord/render.test.ts +204 -0
- package/src/messaging/providers/discord/render.ts +195 -0
- package/src/messaging/providers/discord/send.test.ts +179 -0
- package/src/messaging/providers/discord/send.ts +212 -0
- package/src/messaging/providers/discord/transport.ts +81 -0
- package/src/messaging/providers/index.ts +3 -1
- package/src/messaging/providers/retry-policy.test.ts +247 -0
- package/src/messaging/providers/retry-policy.ts +206 -0
- package/src/messaging/providers/slack/mention-source.test.ts +285 -0
- package/src/messaging/providers/slack/mention-source.ts +287 -0
- package/src/messaging/providers/telegram-bot/api.ts +47 -130
- package/src/messaging/providers/whatsapp/api.ts +47 -136
- package/src/monitoring/__tests__/db-integrity-sample.test.ts +4 -1
- package/src/monitoring/__tests__/plugin-auto-update.test.ts +495 -0
- package/src/monitoring/db-integrity-sample.ts +6 -1
- package/src/monitoring/plugin-auto-update.ts +463 -0
- package/src/monitoring/resource-sampler.ts +21 -5
- package/src/monitoring/worker.ts +13 -0
- package/src/notifications/__tests__/assistant-reply-producer.test.ts +375 -2
- package/src/notifications/__tests__/broadcaster.test.ts +37 -1
- package/src/notifications/__tests__/copy-composer.test.ts +165 -0
- package/src/notifications/__tests__/emit-signal-pipeline-failure.test.ts +205 -0
- package/src/notifications/__tests__/notification-utils.test.ts +249 -0
- package/src/notifications/adapters/platform.ts +31 -2
- package/src/notifications/assistant-reply-producer.ts +79 -32
- package/src/notifications/broadcaster.ts +30 -10
- package/src/notifications/copy-composer.ts +69 -0
- package/src/notifications/emit-signal.ts +82 -15
- package/src/notifications/events-store.ts +10 -2
- package/src/notifications/home-feed-side-effect.ts +6 -3
- package/src/notifications/notification-utils.ts +173 -5
- package/src/notifications/signal.ts +13 -0
- package/src/oauth/AGENTS.md +4 -2
- package/src/oauth/byo-connection.ts +6 -1
- package/src/oauth/credential-token-resolver.ts +3 -10
- package/src/oauth/manual-token-connection.ts +29 -64
- package/src/oauth/manual-token-providers.ts +62 -0
- package/src/oauth/provider-serializer.ts +13 -0
- package/src/oauth/seed-providers.ts +62 -11
- package/src/onboarding/onboarding-events-store.ts +91 -1
- package/src/permissions/question-prompter.test.ts +4 -0
- package/src/permissions/question-prompter.ts +21 -4
- package/src/persistence/__tests__/conversation-lineage.test.ts +248 -0
- package/src/persistence/attachments-store.ts +15 -15
- package/src/persistence/conversation-attention-store.ts +30 -10
- package/src/persistence/conversation-bootstrap.ts +17 -0
- package/src/persistence/conversation-crud.ts +363 -97
- package/src/persistence/conversation-group-migration.ts +1 -1
- package/src/persistence/conversation-key-store.ts +38 -10
- package/src/persistence/conversation-lineage.ts +226 -0
- package/src/persistence/conversation-plugin-facade.ts +28 -3
- package/src/persistence/conversation-queries.ts +207 -33
- package/src/persistence/conversation-types.ts +107 -1
- package/src/persistence/jobs-store.ts +32 -8
- package/src/persistence/llm-usage-store.ts +22 -0
- package/src/persistence/migrations/361-normalize-managed-connection-rows.test.ts +168 -0
- package/src/persistence/migrations/361-normalize-managed-connection-rows.ts +76 -0
- package/src/persistence/migrations/362-add-conversation-subagent-kind.test.ts +187 -0
- package/src/persistence/migrations/362-add-conversation-subagent-kind.ts +93 -0
- package/src/persistence/migrations/363-backfill-schedule-inference-profile.test.ts +229 -0
- package/src/persistence/migrations/363-backfill-schedule-inference-profile.ts +127 -0
- package/src/persistence/migrations/364-add-schedule-source-key.test.ts +120 -0
- package/src/persistence/migrations/364-add-schedule-source-key.ts +43 -0
- package/src/persistence/migrations/365-add-conversation-fork-strategy.ts +27 -0
- package/src/persistence/schema/conversations.ts +32 -0
- package/src/persistence/schema/infrastructure.ts +3 -0
- package/src/persistence/steps.ts +30 -0
- package/src/persistence/subagent-store.ts +148 -6
- package/src/platform/managed-speech.test.ts +20 -0
- package/src/platform/managed-speech.ts +9 -0
- package/src/plugin-api/__tests__/oauth-callback-url-export.test.ts +29 -0
- package/src/plugin-api/index.ts +28 -5
- package/src/plugin-api/webhook-url.ts +151 -0
- package/src/plugins/__tests__/installed-plugin-dirs.test.ts +82 -0
- package/src/plugins/__tests__/mcp-servers.test.ts +370 -0
- package/src/plugins/__tests__/source-fingerprint.test.ts +5 -1
- package/src/plugins/collect-source-versions.ts +4 -27
- package/src/plugins/defaults/exploration-drift/hooks/post-tool-use.ts +5 -5
- package/src/plugins/defaults/exploration-drift/package.json +1 -1
- package/src/plugins/defaults/image-recovery/hooks/post-model-call.ts +4 -2
- package/src/plugins/defaults/image-recovery/recover.ts +67 -57
- package/src/plugins/defaults/index.ts +1 -1
- package/src/plugins/defaults/memory/__tests__/conversation-queries.test.ts +64 -18
- package/src/plugins/defaults/memory/__tests__/jobs-worker-outcome-contract.test.ts +242 -0
- package/src/plugins/defaults/memory/__tests__/jobs-worker-retrospective-sweep.test.ts +24 -1
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-enqueue.test.ts +48 -3
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-job.test.ts +511 -19
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-prompt.test.ts +39 -3
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-provider-path.test.ts +493 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-wake-chain.test.ts +5 -4
- package/src/plugins/defaults/memory/__tests__/memory-tier-boundary-guard.test.ts +5 -1
- package/src/plugins/defaults/memory/job-handlers.ts +87 -5
- package/src/plugins/defaults/memory/jobs-worker.ts +60 -6
- package/src/plugins/defaults/memory/memory-retrospective-constants.ts +11 -0
- package/src/plugins/defaults/memory/memory-retrospective-enqueue.ts +29 -0
- package/src/plugins/defaults/memory/memory-retrospective-job.ts +342 -60
- package/src/plugins/defaults/memory/memory-retrospective-prompt.ts +23 -3
- package/src/plugins/defaults/memory/substrate/consolidation-lock.ts +9 -4
- package/src/plugins/defaults/memory/substrate/skill-content.ts +8 -1
- package/src/plugins/defaults/memory/substrate/sweep-job.ts +7 -0
- package/src/plugins/defaults/memory/v1/graph/injection.ts +1 -1
- package/src/plugins/defaults/memory/v2/router.ts +8 -0
- package/src/plugins/defaults/turn-context/unified-turn-context.ts +1 -2
- package/src/plugins/external-plugin-loader.ts +64 -8
- package/src/plugins/installed-plugin-dirs.ts +101 -0
- package/src/plugins/mcp-servers.ts +343 -0
- package/src/plugins/mtime-cache.ts +34 -22
- package/src/plugins/types.ts +69 -8
- package/src/prompts/persona-resolver.ts +6 -2
- package/src/prompts/system-prompt.ts +72 -17
- package/src/providers/__tests__/inference.test.ts +40 -0
- package/src/providers/__tests__/preflight-resolved-config.test.ts +58 -1
- package/src/providers/__tests__/retry-callsite.test.ts +72 -2
- package/src/providers/__tests__/vellum-connection-routing.test.ts +42 -11
- package/src/providers/anthropic/client.ts +112 -95
- package/src/providers/connection-resolution.ts +35 -1
- package/src/providers/gemini/client.ts +5 -5
- package/src/providers/inference/adapter-factory.ts +16 -10
- package/src/providers/inference/auth.ts +16 -0
- package/src/providers/inference/connection-availability.ts +164 -12
- package/src/providers/inference/connections.ts +25 -1
- package/src/providers/media-resolve.ts +21 -11
- package/src/providers/model-catalog.ts +15 -0
- package/src/providers/openai/chat-completions-provider.ts +4 -4
- package/src/providers/openai/responses-provider.ts +12 -25
- package/src/providers/registry.ts +9 -4
- package/src/providers/retry.ts +25 -0
- package/src/providers/server-tool-pairing.ts +134 -0
- package/src/providers/speech-to-text/__tests__/deepgram-flux-frames.test.ts +433 -0
- package/src/providers/speech-to-text/__tests__/deepgram-flux-realtime.test.ts +620 -0
- package/src/providers/speech-to-text/__tests__/provider-catalog.test.ts +34 -0
- package/src/providers/speech-to-text/__tests__/resolve.test.ts +472 -12
- package/src/providers/speech-to-text/deepgram-flux-frames.ts +395 -0
- package/src/providers/speech-to-text/deepgram-flux-realtime.ts +719 -0
- package/src/providers/speech-to-text/deepgram-realtime.test.ts +283 -1
- package/src/providers/speech-to-text/deepgram-realtime.ts +117 -3
- package/src/providers/speech-to-text/deepgram.ts +1 -0
- package/src/providers/speech-to-text/provider-catalog.ts +137 -8
- package/src/providers/speech-to-text/resolve.ts +105 -12
- package/src/providers/speech-to-text/vellum-managed.ts +7 -1
- package/src/providers/speech-to-text/vellum-speech-relay-connection.ts +3 -1
- package/src/providers/types.ts +26 -6
- package/src/providers/vellum-model-routing.ts +6 -7
- package/src/runtime/AGENTS.md +2 -0
- package/src/runtime/__tests__/agent-wake.test.ts +16 -16
- package/src/runtime/__tests__/background-job-runner.test.ts +0 -144
- package/src/runtime/__tests__/desktop-presence.test.ts +162 -0
- package/src/runtime/__tests__/local-actor-identity-force-refresh.test.ts +104 -0
- package/src/runtime/agent-wake.ts +63 -24
- package/src/runtime/assistant-event-hub.ts +110 -12
- package/src/runtime/auth/__tests__/guard-tests.test.ts +2 -0
- package/src/runtime/auth/__tests__/scopes.test.ts +20 -0
- package/src/runtime/auth/same-actor.ts +56 -8
- package/src/runtime/auth/scopes.ts +12 -0
- package/src/runtime/auth/types.ts +2 -0
- package/src/runtime/background-job-runner.ts +5 -27
- package/src/runtime/channel-readiness-service.ts +59 -3
- package/src/runtime/channel-readiness-types.ts +18 -0
- package/src/runtime/desktop-presence.ts +69 -0
- package/src/runtime/http-server.ts +12 -0
- package/src/runtime/local-actor-identity.ts +29 -14
- package/src/runtime/migrations/__tests__/vbundle-symlink-streaming-importer.test.ts +0 -0
- package/src/runtime/routes/__tests__/channel-verification-routes.test.ts +1 -1
- package/src/runtime/routes/__tests__/client-routes.test.ts +186 -14
- package/src/runtime/routes/__tests__/conversation-list-routes.test.ts +547 -1
- package/src/runtime/routes/__tests__/conversation-management-routes.test.ts +162 -7
- package/src/runtime/routes/__tests__/conversation-query-routes.test.ts +14 -10
- package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +340 -3
- package/src/runtime/routes/__tests__/llm-call-sites-routes.test.ts +28 -1
- package/src/runtime/routes/__tests__/plugins-routes.test.ts +217 -1
- package/src/runtime/routes/__tests__/question-routes.test.ts +64 -0
- package/src/runtime/routes/__tests__/retrospective-routes.test.ts +20 -0
- package/src/runtime/routes/__tests__/skill-history-route.test.ts +116 -0
- package/src/runtime/routes/__tests__/sse-actor-principal-heal.test.ts +165 -0
- package/src/runtime/routes/__tests__/stt-routes.test.ts +25 -0
- package/src/runtime/routes/__tests__/surface-action-routes.test.ts +2 -0
- package/src/runtime/routes/__tests__/wake-conversation-routes.test.ts +60 -2
- package/src/runtime/routes/archive-utils.ts +16 -12
- package/src/runtime/routes/attachment-routes.ts +13 -6
- package/src/runtime/routes/canned-reply-release.ts +54 -0
- package/src/runtime/routes/channel-availability-routes.ts +32 -14
- package/src/runtime/routes/channel-verification-routes.ts +2 -27
- package/src/runtime/routes/chatgpt-subscription-auth-routes.ts +5 -0
- package/src/runtime/routes/client-routes.ts +114 -2
- package/src/runtime/routes/conversation-cli-routes.ts +10 -12
- package/src/runtime/routes/conversation-list-routes.ts +59 -10
- package/src/runtime/routes/conversation-management-routes.ts +47 -18
- package/src/runtime/routes/conversation-query-routes.ts +32 -53
- package/src/runtime/routes/conversation-routes.ts +360 -293
- package/src/runtime/routes/credential-prompt-routes.ts +20 -2
- package/src/runtime/routes/debug-routes.ts +4 -1
- package/src/runtime/routes/default-provider-routes.ts +7 -11
- package/src/runtime/routes/documents-routes.ts +44 -1
- package/src/runtime/routes/events-routes.ts +29 -1
- package/src/runtime/routes/host-app-control-routes.ts +2 -0
- package/src/runtime/routes/host-bash-routes.ts +2 -0
- package/src/runtime/routes/host-browser-routes.ts +2 -0
- package/src/runtime/routes/host-cu-routes.ts +2 -0
- package/src/runtime/routes/host-file-routes.ts +2 -0
- package/src/runtime/routes/host-transfer-routes.ts +4 -0
- package/src/runtime/routes/identity-routes.ts +6 -6
- package/src/runtime/routes/inbound-message-handler.ts +5 -5
- package/src/runtime/routes/inbound-stages/acl-enforcement.ts +7 -3
- package/src/runtime/routes/inbound-stages/background-dispatch.ts +14 -0
- package/src/runtime/routes/inbound-stages/bootstrap-intercept.ts +2 -2
- package/src/runtime/routes/inbound-stages/guardian-activation-intercept.test.ts +14 -11
- package/src/runtime/routes/inbound-stages/guardian-activation-intercept.ts +14 -17
- package/src/runtime/routes/inference-profile-availability-guard.ts +171 -0
- package/src/runtime/routes/inference-profile-session-handler.ts +26 -2
- package/src/runtime/routes/inference-profile-session-routes.ts +2 -1
- package/src/runtime/routes/inference-profiles-routes.ts +152 -73
- package/src/runtime/routes/inference-provider-connection-routes.ts +0 -16
- package/src/runtime/routes/llm-call-sites-routes.ts +13 -0
- package/src/runtime/routes/log-export-routes.ts +1 -1
- package/src/runtime/routes/mcp-auth-routes.ts +141 -57
- package/src/runtime/routes/notification-routes.ts +0 -14
- package/src/runtime/routes/oauth-providers.ts +1 -0
- package/src/runtime/routes/plugins-routes.ts +127 -4
- package/src/runtime/routes/profiler-routes.ts +4 -2
- package/src/runtime/routes/question-routes.ts +28 -10
- package/src/runtime/routes/retrospective-routes.ts +9 -3
- package/src/runtime/routes/schedule-routes.ts +210 -12
- package/src/runtime/routes/secret-routes.ts +40 -23
- package/src/runtime/routes/settings-routes.ts +41 -13
- package/src/runtime/routes/skills-routes.ts +79 -0
- package/src/runtime/routes/sse-actor-principal-heal.ts +113 -0
- package/src/runtime/routes/stt-routes.ts +31 -25
- package/src/runtime/routes/ui-snapshot-routes.ts +1 -0
- package/src/runtime/routes/wake-conversation-routes.ts +23 -0
- package/src/runtime/services/__tests__/conversation-serializer.test.ts +1 -0
- package/src/runtime/services/conversation-serializer.ts +24 -9
- package/src/runtime/sync/documents-sidecar-publish.test.ts +101 -0
- package/src/runtime/sync/resource-sync-events.ts +79 -1
- package/src/runtime/sync/worker-daemon-notify.test.ts +34 -0
- package/src/runtime/sync/worker-daemon-notify.ts +44 -6
- package/src/runtime/verification-outbound-actions.ts +250 -222
- package/src/runtime/verification-templates.ts +41 -30
- package/src/schedule/__tests__/plugin-schedule-declarations.test.ts +614 -0
- package/src/schedule/__tests__/plugin-schedule-reconciler.test.ts +1066 -0
- package/src/schedule/inference-profile.ts +64 -3
- package/src/schedule/plugin-schedule-declarations.ts +518 -0
- package/src/schedule/plugin-schedule-reconciler.ts +605 -0
- package/src/schedule/plugin-schedules-gate.ts +23 -0
- package/src/schedule/schedule-store.ts +520 -26
- package/src/schedule/scheduler.ts +38 -0
- package/src/security/credential-backend.ts +51 -4
- package/src/security/encrypted-store.ts +64 -42
- package/src/security/oauth2.ts +6 -22
- package/src/security/secure-keys.ts +55 -7
- package/src/skills/catalog-install.ts +20 -7
- package/src/skills/inline-command-runner.ts +2 -7
- package/src/skills/skill-history.test.ts +269 -0
- package/src/skills/skill-history.ts +250 -0
- package/src/skills/skillssh-registry.ts +1 -1
- package/src/stt/__tests__/daemon-batch-transcriber.test.ts +22 -0
- package/src/stt/__tests__/language-metadata.test.ts +85 -0
- package/src/stt/__tests__/speech-energy.test.ts +79 -0
- package/src/stt/__tests__/types.test.ts +94 -0
- package/src/stt/daemon-batch-transcriber.ts +18 -6
- package/src/stt/language-metadata.ts +65 -0
- package/src/stt/speech-energy.ts +115 -12
- package/src/stt/stt-stream-session.ts +8 -4
- package/src/stt/types.ts +119 -0
- package/src/subagent/__tests__/consult-prompt.test.ts +25 -0
- package/src/subagent/consult-prompt.ts +3 -1
- package/src/subagent/consult-transcript.ts +1 -1
- package/src/subagent/manager.ts +304 -44
- package/src/subagent/progress-events.ts +35 -0
- package/src/subagent/role-resolution.ts +88 -0
- package/src/subagent/types.ts +303 -67
- package/src/subagent/validate-allowlists.ts +3 -4
- package/src/telegram/__tests__/webhook-health.test.ts +73 -1
- package/src/telegram/webhook-health.ts +101 -3
- package/src/telemetry/__tests__/telemetry-event-fixtures.ts +2 -0
- package/src/telemetry/activation-funnel.ts +6 -1
- package/src/telemetry/live-voice-funnel.ts +75 -0
- package/src/telemetry/telemetry-event-sources.ts +6 -0
- package/src/telemetry/telemetry-wire-source.json +1 -1
- package/src/telemetry/telemetry-wire.generated.ts +2 -0
- package/src/telemetry/types.ts +23 -4
- package/src/telemetry/usage-telemetry-reporter.test.ts +4 -0
- package/src/tools/__tests__/subagent-workflow-tool-input-schemas.test.ts +1 -1
- package/src/tools/ask-question/ask-question-tool.test.ts +118 -3
- package/src/tools/ask-question/ask-question-tool.ts +56 -2
- package/src/tools/browser/browser-execution.ts +1 -1
- package/src/tools/calls/call-start.ts +8 -5
- package/src/tools/conversation-groups/move_to_group.ts +10 -12
- package/src/tools/document/document-tool.ts +11 -4
- package/src/tools/execution-timeout.ts +9 -2
- package/src/tools/executor.ts +7 -6
- package/src/tools/filesystem/edit.ts +1 -1
- package/src/tools/filesystem/list.ts +1 -1
- package/src/tools/filesystem/read.ts +7 -13
- package/src/tools/filesystem/search.ts +23 -21
- package/src/tools/filesystem/write.ts +4 -1
- package/src/tools/host-filesystem/edit.ts +7 -2
- package/src/tools/host-filesystem/read.ts +21 -8
- package/src/tools/host-filesystem/write.ts +10 -2
- package/src/tools/host-terminal/host-shell.ts +2 -2
- package/src/tools/mcp/mcp-tool-factory.ts +25 -1
- package/src/tools/permission-checker.ts +8 -3
- package/src/tools/schedule/delete.ts +13 -1
- package/src/tools/schedule/list.ts +14 -3
- package/src/tools/schedule/update.ts +56 -33
- package/src/tools/shared/filesystem/audio-read.ts +6 -4
- package/src/tools/shared/filesystem/file-ops-service.ts +176 -116
- package/src/tools/shared/filesystem/image-read.ts +12 -12
- package/src/tools/shared/filesystem/path-policy.ts +70 -5
- package/src/tools/shared/filesystem/size-guard.ts +6 -6
- package/src/tools/shared/input-misuse.ts +72 -0
- package/src/tools/skills/scaffold-managed.ts +82 -0
- package/src/tools/skills/skill-tool-factory.ts +14 -1
- package/src/tools/subagent/consult-deadline.ts +12 -7
- package/src/tools/subagent/message.ts +3 -1
- package/src/tools/subagent/read.ts +66 -4
- package/src/tools/subagent/spawn.ts +500 -78
- package/src/tools/tool-approval-handler.ts +45 -4
- package/src/tools/types.ts +22 -3
- package/src/tools/ui-surface/surface-shape-docs.ts +1 -1
- package/src/tools/ui-surface/visual-validation.ts +2 -0
- package/src/tts/__tests__/provider-adapters.test.ts +147 -0
- package/src/tts/__tests__/speakable-segments.test.ts +470 -0
- package/src/tts/language-voices.ts +23 -0
- package/src/tts/providers/deepgram-provider.ts +3 -1
- package/src/tts/providers/elevenlabs-provider.ts +73 -1
- package/src/tts/providers/xai-provider.ts +28 -2
- package/src/tts/speakable-segments.ts +293 -23
- package/src/tts/synthesis-stream.ts +7 -0
- package/src/tts/types.ts +7 -0
- package/src/usage/__tests__/subagent-attribution.test.ts +109 -0
- package/src/usage/subagent-attribution.ts +108 -0
- package/src/util/__tests__/language-subtag.test.ts +54 -0
- package/src/util/__tests__/sqlite-retry.test.ts +112 -0
- package/src/util/abort-reasons.ts +16 -0
- package/src/util/ansi.ts +44 -0
- package/src/util/disk-usage.ts +17 -13
- package/src/util/image-conversion.ts +20 -13
- package/src/util/language-subtag.ts +43 -0
- package/src/util/schedule-source-key.ts +23 -0
- package/src/util/unicode.ts +1 -1
- package/src/util/worker-process.ts +7 -1
- package/src/workspace/git-service.ts +33 -9
- package/src/workspace/migrations/140-repair-seed-pinned-memory-v3-live.ts +260 -0
- package/src/workspace/migrations/141-stt-english-default-to-multilingual.ts +111 -0
- package/src/workspace/migrations/142-consolidate-voice-front-door.ts +70 -0
- package/src/workspace/migrations/__tests__/141-stt-english-default-to-multilingual.test.ts +116 -0
- package/src/workspace/migrations/registry.ts +6 -0
- package/src/workspace/top-level-renderer.ts +15 -2
- package/src/__tests__/tool-side-effects-slack-dm.test.ts +0 -357
- package/src/daemon/install-symlink.ts +0 -207
- package/src/live-voice/__tests__/front-decision.test.ts +0 -525
- package/src/live-voice/front-decision.ts +0 -429
- package/src/notifications/deferred-emit.ts +0 -147
|
@@ -17,7 +17,6 @@ import type {
|
|
|
17
17
|
import {
|
|
18
18
|
getConversationTurnTeardown,
|
|
19
19
|
resolveProcessingWaitMs,
|
|
20
|
-
VOICE_NO_SETUP_FLOWS_RULE,
|
|
21
20
|
waitForPriorTurnTeardown,
|
|
22
21
|
} from "../calls/voice-session-bridge.js";
|
|
23
22
|
import {
|
|
@@ -25,38 +24,58 @@ import {
|
|
|
25
24
|
classifyFrontDoorLeading,
|
|
26
25
|
ESCALATE_VERDICT_TOKEN,
|
|
27
26
|
ESCALATION_CONTINUATION_CONTENT,
|
|
28
|
-
|
|
27
|
+
FALLBACK_ESCALATION_BRIDGE_BY_LANGUAGE,
|
|
28
|
+
fallbackEscalationBridgeFor,
|
|
29
29
|
isEscalationBridgeComplete,
|
|
30
30
|
MIN_SPOKEN_BRIDGE_CHARS,
|
|
31
31
|
type VoiceRoutingLeg,
|
|
32
32
|
} from "../calls/voice-triage-escalate.js";
|
|
33
33
|
import { getConfig } from "../config/loader.js";
|
|
34
34
|
import {
|
|
35
|
+
type LiveVoiceFluxConfig,
|
|
36
|
+
LiveVoiceFluxConfigSchema,
|
|
35
37
|
type LiveVoiceFrontModelConfig,
|
|
36
38
|
LiveVoiceFrontModelConfigSchema,
|
|
37
39
|
} from "../config/schemas/live-voice.js";
|
|
38
40
|
import { ABORT_WATCHDOG_MS } from "../daemon/abort-watchdog.js";
|
|
39
41
|
import { isRefusedInReadOnlyPass } from "../daemon/conversation-tool-setup.js";
|
|
40
42
|
import type { TrustContext } from "../daemon/trust-context-types.js";
|
|
43
|
+
import {
|
|
44
|
+
recordLiveVoiceSessionEnded,
|
|
45
|
+
recordLiveVoiceSessionStarted,
|
|
46
|
+
} from "../onboarding/onboarding-events-store.js";
|
|
41
47
|
import { isInstalledStaticSkillLoad } from "../permissions/checker.js";
|
|
42
48
|
import { ensureConversationExists } from "../persistence/conversation-crud.js";
|
|
43
49
|
import {
|
|
44
50
|
listProviderIds,
|
|
51
|
+
pinnedListeningLanguage,
|
|
45
52
|
supportsBoundary,
|
|
53
|
+
supportsProviderTurnDetection,
|
|
46
54
|
} from "../providers/speech-to-text/provider-catalog.js";
|
|
47
55
|
import type { ResolveStreamingTranscriberOptions } from "../providers/speech-to-text/resolve.js";
|
|
48
56
|
import { broadcastMessage } from "../runtime/assistant-event-hub.js";
|
|
49
57
|
import { publishConversationListAndMetadataChanged } from "../runtime/sync/resource-sync-events.js";
|
|
50
|
-
import {
|
|
58
|
+
import {
|
|
59
|
+
dominantLanguageTag,
|
|
60
|
+
voteDominantLanguage,
|
|
61
|
+
} from "../stt/language-metadata.js";
|
|
62
|
+
import {
|
|
63
|
+
DEFAULT_SPEECH_ENERGY_THRESHOLD,
|
|
64
|
+
pcm16MaxNormalizedCorrelation,
|
|
65
|
+
pcm16MeanAmplitude,
|
|
66
|
+
} from "../stt/speech-energy.js";
|
|
51
67
|
import type {
|
|
52
68
|
StreamingTranscriber,
|
|
69
|
+
SttProviderId,
|
|
53
70
|
SttStreamServerErrorEvent,
|
|
54
71
|
SttStreamServerEvent,
|
|
55
72
|
} from "../stt/types.js";
|
|
56
73
|
import { getSubagentManager } from "../subagent/index.js";
|
|
74
|
+
import { liveVoiceEndScreen } from "../telemetry/live-voice-funnel.js";
|
|
57
75
|
import { getToolOwner } from "../tools/registry.js";
|
|
58
76
|
import { extractSpeakableSegments } from "../tts/speakable-segments.js";
|
|
59
77
|
import { createAbortReason } from "../util/abort-reasons.js";
|
|
78
|
+
import { hasLocalizedEntry } from "../util/language-subtag.js";
|
|
60
79
|
import { getLogger } from "../util/logger.js";
|
|
61
80
|
import {
|
|
62
81
|
activityLabelForTool,
|
|
@@ -64,11 +83,6 @@ import {
|
|
|
64
83
|
dismissesUiSurface,
|
|
65
84
|
revealsUiSurface,
|
|
66
85
|
} from "./activity-label.js";
|
|
67
|
-
import {
|
|
68
|
-
createVoiceFrontDecider,
|
|
69
|
-
type VoiceFrontDecider,
|
|
70
|
-
type VoiceProgressTextInput,
|
|
71
|
-
} from "./front-decision.js";
|
|
72
86
|
import { LiveActivityReporter } from "./live-activity-reporter.js";
|
|
73
87
|
import type {
|
|
74
88
|
LiveVoiceAudioArchiveResult,
|
|
@@ -80,10 +94,11 @@ import {
|
|
|
80
94
|
type LiveVoiceMetricsClock,
|
|
81
95
|
LiveVoiceMetricsCollector,
|
|
82
96
|
type LiveVoiceMetricsEvent,
|
|
83
|
-
type LiveVoiceSpokenAckKind,
|
|
84
97
|
type LiveVoiceTurnSeedMarks,
|
|
85
98
|
type VoiceEndpointAction,
|
|
99
|
+
type VoiceEndpointSource,
|
|
86
100
|
} from "./live-voice-metrics.js";
|
|
101
|
+
import { persistLiveVoicePhoto } from "./live-voice-photo.js";
|
|
87
102
|
import {
|
|
88
103
|
type LiveVoiceSession as LiveVoiceSessionContract,
|
|
89
104
|
type LiveVoiceSessionCloseReason,
|
|
@@ -95,8 +110,19 @@ import type {
|
|
|
95
110
|
LiveVoiceTtsOptions,
|
|
96
111
|
LiveVoiceTtsResult,
|
|
97
112
|
} from "./live-voice-tts.js";
|
|
98
|
-
import { pickProgressPhrase } from "./progress-phrases.js";
|
|
99
113
|
import {
|
|
114
|
+
createVoiceProgressNarrator,
|
|
115
|
+
type VoiceProgressNarrator,
|
|
116
|
+
type VoiceProgressTextInput,
|
|
117
|
+
} from "./progress-narration.js";
|
|
118
|
+
import {
|
|
119
|
+
APPROVAL_PENDING_PHRASE_BY_LANGUAGE,
|
|
120
|
+
approvalPendingPhraseFor,
|
|
121
|
+
pickProgressPhrase,
|
|
122
|
+
PROGRESS_FALLBACK_PHRASES_BY_LANGUAGE,
|
|
123
|
+
} from "./progress-phrases.js";
|
|
124
|
+
import {
|
|
125
|
+
type LiveVoiceClientAttachImageFrame,
|
|
100
126
|
type LiveVoiceClientFrame,
|
|
101
127
|
type LiveVoiceClientUpdateConfigFrame,
|
|
102
128
|
LiveVoiceProtocolErrorCode,
|
|
@@ -112,6 +138,13 @@ type LiveVoiceSessionState =
|
|
|
112
138
|
| "failed"
|
|
113
139
|
| "closed";
|
|
114
140
|
|
|
141
|
+
type VadEnergyClassification = "speech" | "silence" | "echo";
|
|
142
|
+
|
|
143
|
+
interface VadClassifiedChunk {
|
|
144
|
+
readonly chunk: Buffer;
|
|
145
|
+
readonly classification: VadEnergyClassification;
|
|
146
|
+
}
|
|
147
|
+
|
|
115
148
|
// Cap on audio buffered while a server-VAD utterance waits for its
|
|
116
149
|
// transcriber (PCM16 mono seconds; oldest chunks are dropped past the cap).
|
|
117
150
|
const SERVER_VAD_PENDING_AUDIO_MAX_SECONDS = 10;
|
|
@@ -130,7 +163,34 @@ const FINALIZE_GRACE_MS = 1_000;
|
|
|
130
163
|
// cancellation cannot kill a reply mid-sentence. Mirrors the
|
|
131
164
|
// liveVoice.vad.bargeInMinSpeechMs schema default; 0 disables the guard for
|
|
132
165
|
// instant barge-in.
|
|
166
|
+
//
|
|
167
|
+
// Local energy detection owns barge-in in every mode, including when a
|
|
168
|
+
// turn-detecting provider owns the turn boundary: an interrupt during TTS has
|
|
169
|
+
// to feel instant, and no provider roundtrip beats a local gate on the audio
|
|
170
|
+
// already in hand. The echo-adaptive part of this guard also has to stay
|
|
171
|
+
// upstream of the provider, which hears only the microphone: it cannot tell
|
|
172
|
+
// our own TTS bleeding through imperfect echo cancellation from the caller,
|
|
173
|
+
// and would report a user turn nobody took.
|
|
133
174
|
const DEFAULT_BARGE_IN_MIN_SPEECH_MS = 250;
|
|
175
|
+
// The playback echo gate learns microphone energy while assistant audio is
|
|
176
|
+
// expected at the speaker. Input must rise above the learned level by this
|
|
177
|
+
// margin to count as user speech.
|
|
178
|
+
const DEFAULT_ECHO_BARGE_IN_MARGIN = 1.5;
|
|
179
|
+
const DEFAULT_ECHO_EMA_HALF_LIFE_MS = 400;
|
|
180
|
+
// Before learning a microphone power baseline, compare a short input window
|
|
181
|
+
// with the PCM sent to the speaker. This keeps a user's first interruption
|
|
182
|
+
// from becoming its own echo threshold.
|
|
183
|
+
const ECHO_CORRELATION_PROBE_MS = 100;
|
|
184
|
+
const ECHO_CORRELATION_MIN_MS = 50;
|
|
185
|
+
const ECHO_CORRELATION_THRESHOLD = 0.65;
|
|
186
|
+
const ECHO_REFERENCE_MAX_MS = 10_000;
|
|
187
|
+
// Echo should reach the microphone near playback onset. If no signal arrives
|
|
188
|
+
// within this much input audio, the gate returns to the fixed base threshold.
|
|
189
|
+
// The same interval expires a learned reference after a real silent gap.
|
|
190
|
+
const ECHO_ONSET_ELIGIBILITY_MS = 300;
|
|
191
|
+
// Client buffering makes audible playback trail the server's send-time
|
|
192
|
+
// estimate. Keep the echo window open briefly past that estimate.
|
|
193
|
+
const DEFAULT_ECHO_DRAIN_SLACK_MS = 300;
|
|
134
194
|
// Mirrors MediaTurnDetector's DEFAULT_SILENCE_THRESHOLD_MS: the session
|
|
135
195
|
// tracks the effective trailing-silence threshold (the detector keeps its own
|
|
136
196
|
// copy private) so the endpoint decider can report the pause length.
|
|
@@ -153,6 +213,12 @@ const BARGE_IN_GAP_TOLERANCE_MS = 200;
|
|
|
153
213
|
// (1 / (1 + ratio) ≈ 20%): once the run is mostly silence it resets, so genuine
|
|
154
214
|
// choppy speech still lands but periodic noise cannot accumulate into one.
|
|
155
215
|
const BARGE_IN_MAX_TOLERATED_SILENCE_RATIO = 4;
|
|
216
|
+
// Slack added to the configured end-of-turn timeout before the session stops
|
|
217
|
+
// waiting for an event that is not coming and falls the utterance back onto
|
|
218
|
+
// the silence-boundary path. A turn-detecting provider force-ends its own turn
|
|
219
|
+
// at that timeout, so anything past it plus a frame's network and parse hop
|
|
220
|
+
// means the stream is gone, not slow.
|
|
221
|
+
const PROVIDER_TURN_END_FALLBACK_MARGIN_MS = 1_000;
|
|
156
222
|
// At most this many TTS segment jobs are open (provider stream started,
|
|
157
223
|
// frames not yet fully emitted) per turn: the emitting job plus one
|
|
158
224
|
// prefetching job. The prefetch buffers its chunks in memory until promoted;
|
|
@@ -261,25 +327,41 @@ export interface LiveVoiceSessionOptions {
|
|
|
261
327
|
* defaults to `DEFAULT_BARGE_IN_MIN_SPEECH_MS`.
|
|
262
328
|
*/
|
|
263
329
|
bargeInMinSpeechMs?: number;
|
|
330
|
+
/**
|
|
331
|
+
* Multiplier over the learned playback echo level that input must exceed
|
|
332
|
+
* to count as speech while assistant audio is playing. Values at or below
|
|
333
|
+
* 1 disable adaptation for internal fixed-gate callers; workspace config
|
|
334
|
+
* requires a value greater than 1.
|
|
335
|
+
*/
|
|
336
|
+
echoBargeInMargin?: number;
|
|
337
|
+
/** Half-life in milliseconds for the learned playback echo level. */
|
|
338
|
+
echoEmaHalfLifeMs?: number;
|
|
339
|
+
/** Extra time after the playback estimate during which echo is expected. */
|
|
340
|
+
echoDrainSlackMs?: number;
|
|
264
341
|
/**
|
|
265
342
|
* Overrides the bounded wait for the shared transcriber's finalize
|
|
266
343
|
* flush in persistent mode (test hook). Defaults to `FINALIZE_GRACE_MS`.
|
|
267
344
|
*/
|
|
268
345
|
finalizeGraceMs?: number;
|
|
269
346
|
/**
|
|
270
|
-
*
|
|
347
|
+
* Voice front-door endpointing and progress narration tuning. The
|
|
271
348
|
* production factory seeds this from `liveVoice.frontModel` config when
|
|
272
349
|
* unset; absent fields fall back to the schema defaults (the constructor
|
|
273
350
|
* schema-parses the partial into a complete config).
|
|
274
351
|
*/
|
|
275
352
|
frontModelConfig?: Partial<LiveVoiceFrontModelConfig>;
|
|
276
353
|
/**
|
|
277
|
-
*
|
|
278
|
-
*
|
|
279
|
-
*
|
|
280
|
-
|
|
354
|
+
* Deepgram Flux turn-detection tuning. The production factory seeds this
|
|
355
|
+
* from `liveVoice.flux` config when unset; absent fields fall back to the
|
|
356
|
+
* schema defaults, which leave `turnEnd.enabled` false.
|
|
357
|
+
*/
|
|
358
|
+
fluxConfig?: Partial<LiveVoiceFluxConfig>;
|
|
359
|
+
/**
|
|
360
|
+
* Progress narration service. The production factory constructs it from
|
|
361
|
+
* `liveVoice.frontModel.progress` config when unset. `null` disables
|
|
362
|
+
* generated narration, so progress updates use their static fallback.
|
|
281
363
|
*/
|
|
282
|
-
|
|
364
|
+
progressNarrator?: VoiceProgressNarrator | null;
|
|
283
365
|
/**
|
|
284
366
|
* Spawns the background continuation for a barged-in turn. The factory wires
|
|
285
367
|
* the real SubagentManager-backed implementation; tests inject a stub.
|
|
@@ -357,11 +439,44 @@ interface UtteranceCycle {
|
|
|
357
439
|
// Consecutive semantic-endpointing "hold" extensions this utterance has
|
|
358
440
|
// consumed, bounded by `endpointMaxExtensions`.
|
|
359
441
|
endpointExtensionCount: number;
|
|
442
|
+
// The provider's end-of-turn never arrived for this cycle, so it fell back
|
|
443
|
+
// to the silence-boundary path and stays there: a late event must not
|
|
444
|
+
// commit a boundary the fallback already owns.
|
|
445
|
+
providerTurnEndTimedOut: boolean;
|
|
446
|
+
// `vadSpeechGeneration` as of the local silence boundary that handed this
|
|
447
|
+
// cycle to the provider, or null while no boundary has fired yet (a turn
|
|
448
|
+
// model routinely beats the trailing-silence countdown, and that fast commit
|
|
449
|
+
// is the point).
|
|
450
|
+
// The staleness signal of last resort: it stands in for the provider's own
|
|
451
|
+
// turn numbering when the provider sends none (see isStaleProviderTurnEnd).
|
|
452
|
+
turnBoundaryGeneration: number | null;
|
|
453
|
+
// Index of the newest provider turn opened in this cycle, from the turn
|
|
454
|
+
// model's own numbering, or null when the provider does not number its
|
|
455
|
+
// turns. An end-of-turn closing an older index describes a turn the
|
|
456
|
+
// provider has already superseded (see isStaleProviderTurnEnd).
|
|
457
|
+
openProviderTurnIndex: number | null;
|
|
360
458
|
// The transcript the most recent hold verdict judged (unified front-door).
|
|
361
459
|
// A final segment arriving during the extension window that extends this
|
|
362
460
|
// text replays the boundary immediately — the hold was judged on stale
|
|
363
461
|
// text, so waiting out the extension only adds silence.
|
|
364
462
|
heldSpeculativeContent: string | null;
|
|
463
|
+
// Count per detected-language base subtag (see voteDominantLanguage)
|
|
464
|
+
// across this cycle's final transcript events. Resolves the turn's spoken
|
|
465
|
+
// language (see turnLanguageFor); empty when the provider tags nothing.
|
|
466
|
+
languageTally: Map<string, number>;
|
|
467
|
+
// Detected languages of the most recent partial that carried any, already
|
|
468
|
+
// normalized, dominance order. Speculative turns dispatch from partials
|
|
469
|
+
// before the first tagged final lands, so turnLanguageFor falls back to
|
|
470
|
+
// this when the final tally is still empty. Never cleared: the tally
|
|
471
|
+
// outranks it once finals arrive, and a revising partial without tags
|
|
472
|
+
// must not wipe an earlier partial's detection.
|
|
473
|
+
latestPartialLanguages: readonly string[] | null;
|
|
474
|
+
// The provider that actually transcribed this cycle, recorded when its
|
|
475
|
+
// transcriber is assigned and kept after teardown nulls `transcriber`.
|
|
476
|
+
// The resolver can silently dial managed vellum when a BYOK provider has
|
|
477
|
+
// no credential, so the language-pin gate in turnLanguageFor must follow
|
|
478
|
+
// this, not the configured provider.
|
|
479
|
+
dialedSttProvider: SttProviderId | null;
|
|
365
480
|
turnId: string | null;
|
|
366
481
|
userMessageId: string | null;
|
|
367
482
|
userAudioChunks: Buffer[];
|
|
@@ -392,6 +507,11 @@ type UtteranceStartResult =
|
|
|
392
507
|
// client in job-list order.
|
|
393
508
|
interface TtsSegmentJob {
|
|
394
509
|
readonly text: string;
|
|
510
|
+
// Per-segment language-hint override, preferred over the turn's language.
|
|
511
|
+
// Set on fixed phrases whose localized table lacks the turn's language:
|
|
512
|
+
// the English fallback text carries "en" so an enforcing provider never
|
|
513
|
+
// renders English words as ar/ko/ta. Undefined means the turn language.
|
|
514
|
+
readonly language: string | undefined;
|
|
395
515
|
// The provider stream was started (the job holds an open-job slot).
|
|
396
516
|
started: boolean;
|
|
397
517
|
// Emission finished; the slot is free for the next queued segment.
|
|
@@ -476,6 +596,13 @@ interface ActiveAssistantTurn {
|
|
|
476
596
|
token: symbol;
|
|
477
597
|
turnId: string;
|
|
478
598
|
utterance: UtteranceCycle;
|
|
599
|
+
// The caller's spoken language for this turn as a lowercase base subtag
|
|
600
|
+
// (see turnLanguageFor): the dominant STT-detected language, else a
|
|
601
|
+
// monolingual services.stt.language pin. Undefined when unknown, which
|
|
602
|
+
// disables every language-aware path (prompt note, TTS hint, localized
|
|
603
|
+
// fallbacks). Re-resolved when a speculative turn commits, since finals
|
|
604
|
+
// can land between dispatch and verdict.
|
|
605
|
+
language: string | undefined;
|
|
479
606
|
abortController: AbortController;
|
|
480
607
|
handle: VoiceTurnHandle | null;
|
|
481
608
|
// When the turn launched, for narration's turnElapsedMs.
|
|
@@ -578,27 +705,11 @@ interface ActiveAssistantTurn {
|
|
|
578
705
|
// context. A bump since then means a stop/interrupt/supersede deliberately
|
|
579
706
|
// dropped the continuation, so a rollback must not resurrect it.
|
|
580
707
|
pendingContextStopGeneration: number;
|
|
581
|
-
// The agent run started a definitive tool use this turn — tool use implies
|
|
582
|
-
// a guaranteed-slow turn, so acknowledgment logic can key off this.
|
|
583
|
-
toolUseStarted: boolean;
|
|
584
|
-
// The model's first streamed delta reached this session — the spoken-ack
|
|
585
|
-
// timer is moot once real output is flowing.
|
|
586
|
-
firstDeltaSeen: boolean;
|
|
587
708
|
// Counts assistant text deltas seen this turn. A narration generation
|
|
588
709
|
// captures it at launch and discards its result if it moved: text the model
|
|
589
710
|
// produced mid-generation makes the narration stale, and proves the model
|
|
590
711
|
// is alive — which is exactly what narration exists to paper over.
|
|
591
712
|
deltaEpoch: number;
|
|
592
|
-
// A spoken ack was enqueued this turn. Every ack trigger shares this
|
|
593
|
-
// one-per-turn budget so a slow turn never stacks fillers.
|
|
594
|
-
ackSpoken: boolean;
|
|
595
|
-
// An ack generation is awaiting the decider. While true, the ack has not
|
|
596
|
-
// yet stamped `lastFloorHolderAtMs`, so narration must treat it as a
|
|
597
|
-
// floor-holder-in-waiting and stand down —
|
|
598
|
-
// otherwise a progress phrase could land back-to-back with the ack.
|
|
599
|
-
ackGenerationPending: boolean;
|
|
600
|
-
// Pending slow-first-delta ack timer; null once cleared or fired.
|
|
601
|
-
ackTimer: ReturnType<typeof setTimeout> | null;
|
|
602
713
|
// Triage-and-escalate (Voice Mode): the front-door leg gave the escalate
|
|
603
714
|
// verdict and the strong "escalated" leg has taken over this same turn.
|
|
604
715
|
// Guards the
|
|
@@ -674,14 +785,8 @@ function createControlMarkerHoldback(
|
|
|
674
785
|
// barge-in, the interruption merge note is appended to it (see
|
|
675
786
|
// buildInterruptionMergeNote) so the model reconciles the interrupted request
|
|
676
787
|
// with the new utterance.
|
|
677
|
-
// Spoken once when a turn starts waiting on the user's decision. Fixed rather
|
|
678
|
-
// than generated: this is a statement about the system's state, not about the
|
|
679
|
-
// work, and it has to be true every time. Kept in the shape of the progress
|
|
680
|
-
// phrases it displaces (short, neutral, no claim about tools).
|
|
681
|
-
const APPROVAL_PENDING_PHRASE = "I need your okay for that one. Take a look.";
|
|
682
|
-
|
|
683
788
|
const LIVE_VOICE_CONTROL_PROMPT_BASE =
|
|
684
|
-
"You are speaking in a local live voice session. Keep replies brief and conversational. Speech is the main channel: say the answer, and do not narrate a surface instead of answering. You can also put something on screen when it genuinely helps (a form, a list to pick from, a progress card for long work); the call overlay minimizes by itself once you finish speaking, so the user sees it without doing anything. Never tell the user you cannot show them something. ";
|
|
789
|
+
"You are speaking in a local live voice session. Keep replies brief and conversational. Speech is the main channel: say the answer, and do not narrate a surface instead of answering. You can also put something on screen when it genuinely helps (a form, a list to pick from, a progress card for long work); the call overlay minimizes by itself once you finish speaking, so the user sees it without doing anything. Never tell the user you cannot show them something. Reply in the language the caller is speaking; if they switch languages, switch with them. ";
|
|
685
790
|
|
|
686
791
|
// Appended for the legs that can actually put something on screen: the main
|
|
687
792
|
// leg and the escalated leg. The front-door (fast) leg never receives it, for
|
|
@@ -699,6 +804,20 @@ const LIVE_VOICE_CONTROL_PROMPT_BASE =
|
|
|
699
804
|
const LIVE_VOICE_SCREEN_REVEAL_TEACHING =
|
|
700
805
|
"The call renders as a full-screen overlay covering the app. Whenever you put something on screen, the overlay minimizes by itself as soon as you finish speaking, and the user is looking at what you made. So speak as if you are showing it to them right now (for example, close with something like: take a look), and never say you cannot show it, that this is a voice call, or that they should check it later. Never emit bracketed markers of any kind. ";
|
|
701
806
|
|
|
807
|
+
// The setup-flow case, spelled out because it is the one the model gets wrong
|
|
808
|
+
// on its own: connecting an account reads as something a call cannot do, so it
|
|
809
|
+
// declines and offers to do it later in text, which is the exact thing the base
|
|
810
|
+
// prompt above forbids.
|
|
811
|
+
//
|
|
812
|
+
// It can do it. The connect card is a ui surface like any other, so showing it
|
|
813
|
+
// minimizes the room on the same latch (`revealsUiSurface`), and the user is at
|
|
814
|
+
// the screen the browser window opens on. Appended alongside the screen-reveal
|
|
815
|
+
// teaching, to the same legs, for the same reason: the front-door leg is
|
|
816
|
+
// toolless, so a setup flow is not its to run, and its capability digest
|
|
817
|
+
// already tells it to escalate anything needing a tool rather than refuse.
|
|
818
|
+
const LIVE_VOICE_SETUP_FLOW_TEACHING =
|
|
819
|
+
"This includes connecting accounts. If a task needs an account connected or a sign-in completed, put the connection up on screen and let the user do it now. Do not decline it, and do not defer it to text chat. Say what you are connecting and that it is in front of them. ";
|
|
820
|
+
|
|
702
821
|
// System-level guidance appended to a barge-in turn's control prompt so the
|
|
703
822
|
// model treats the new utterance as a continuation of the request it was cut
|
|
704
823
|
// off answering, rather than a fresh follow-up. Reaches the model only; it is
|
|
@@ -755,21 +874,24 @@ function buildLiveDeliveryNote(request: string, answer: string): string {
|
|
|
755
874
|
return `The user interrupted you earlier, and in the background you finished ${what}. The call is still live and nobody is speaking, so tell them briefly that it is done and give them the result. Do not re-run any tool calls; the work is already complete, and do not repeat it verbatim if it no longer fits. What you produced was:\n\n${answer}`;
|
|
756
875
|
}
|
|
757
876
|
|
|
758
|
-
// Assemble a leg's model-facing control prompt: the base live-voice rules,
|
|
759
|
-
//
|
|
760
|
-
// LIVE_VOICE_SCREEN_REVEAL_TEACHING),
|
|
761
|
-
//
|
|
762
|
-
//
|
|
763
|
-
//
|
|
764
|
-
// never render as user bubbles.
|
|
877
|
+
// Assemble a leg's model-facing control prompt: the base live-voice rules, the
|
|
878
|
+
// screen-reveal and setup-flow teaching (both withheld from the front-door leg,
|
|
879
|
+
// see LIVE_VOICE_SCREEN_REVEAL_TEACHING), plus any pending barge-in merge
|
|
880
|
+
// context, completed-continuation context, and/or the announcement instruction.
|
|
881
|
+
// A turn can carry several (a barge-in follow-up that also has a continuation
|
|
882
|
+
// result waiting); the notes are model-only and never render as user bubbles.
|
|
765
883
|
function buildVoiceControlPrompt(
|
|
766
884
|
turn: ActiveAssistantTurn,
|
|
767
885
|
leg: { frontDoor?: boolean },
|
|
768
886
|
): string {
|
|
769
887
|
let prompt =
|
|
770
888
|
LIVE_VOICE_CONTROL_PROMPT_BASE +
|
|
771
|
-
(leg.frontDoor === true
|
|
772
|
-
|
|
889
|
+
(leg.frontDoor === true
|
|
890
|
+
? ""
|
|
891
|
+
: LIVE_VOICE_SCREEN_REVEAL_TEACHING + LIVE_VOICE_SETUP_FLOW_TEACHING);
|
|
892
|
+
if (turn.language !== undefined) {
|
|
893
|
+
prompt = `${prompt}\n\nThe caller has been speaking the language with code "${turn.language}" this turn. Reply in that language unless they clearly switch to another.`;
|
|
894
|
+
}
|
|
773
895
|
if (turn.interruptedRequest) {
|
|
774
896
|
prompt = `${prompt}\n\n${buildInterruptionMergeNote(turn.interruptedRequest)}`;
|
|
775
897
|
}
|
|
@@ -817,7 +939,13 @@ function createUtteranceCycle(): UtteranceCycle {
|
|
|
817
939
|
finalTranscriptSegments: [],
|
|
818
940
|
latestPartialText: null,
|
|
819
941
|
endpointExtensionCount: 0,
|
|
942
|
+
providerTurnEndTimedOut: false,
|
|
943
|
+
turnBoundaryGeneration: null,
|
|
944
|
+
openProviderTurnIndex: null,
|
|
820
945
|
heldSpeculativeContent: null,
|
|
946
|
+
languageTally: new Map(),
|
|
947
|
+
latestPartialLanguages: null,
|
|
948
|
+
dialedSttProvider: null,
|
|
821
949
|
turnId: null,
|
|
822
950
|
userMessageId: null,
|
|
823
951
|
userAudioChunks: [],
|
|
@@ -972,11 +1100,41 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
972
1100
|
private outboundFrames: Promise<void> = Promise.resolve();
|
|
973
1101
|
private activeAssistantTurn: ActiveAssistantTurn | null = null;
|
|
974
1102
|
private sessionEndMetricsEmitted = false;
|
|
1103
|
+
/**
|
|
1104
|
+
* Protocol error code of the failure that killed the session, latched by
|
|
1105
|
+
* {@link sendFrame} when an error frame goes out on an already-`failed`
|
|
1106
|
+
* session. Both fatal paths (`failStartup` before `ready`, a failed
|
|
1107
|
+
* utterance arm after it) set `state = "failed"` before sending, and no
|
|
1108
|
+
* other error frame does, so this catches exactly the session-ending
|
|
1109
|
+
* failures and ignores the recoverable mid-session ones. `null` on a
|
|
1110
|
+
* session that never failed.
|
|
1111
|
+
*/
|
|
1112
|
+
private failureCode: LiveVoiceProtocolErrorCode | null = null;
|
|
975
1113
|
// Non-null iff the start frame requested turnDetection "server_vad".
|
|
976
1114
|
private readonly turnDetector: MediaTurnDetector | null;
|
|
977
|
-
//
|
|
978
|
-
//
|
|
1115
|
+
// Base energy gate for server-VAD speech classification. During estimated
|
|
1116
|
+
// playback, classifyVadEnergy raises this above the learned echo level.
|
|
979
1117
|
private readonly speechEnergyThreshold: number | undefined;
|
|
1118
|
+
private readonly echoBargeInMargin: number;
|
|
1119
|
+
private readonly echoEmaHalfLifeMs: number;
|
|
1120
|
+
private readonly echoDrainSlackMs: number;
|
|
1121
|
+
// Learned microphone energy attributable to assistant playback.
|
|
1122
|
+
private echoEnergyEma = 0;
|
|
1123
|
+
// Signal-bearing microphone audio held until it can be compared with the
|
|
1124
|
+
// assistant PCM. A nonmatch is replayed through VAD in original order.
|
|
1125
|
+
private echoProbeChunks: Buffer[] = [];
|
|
1126
|
+
// Recent raw assistant PCM from the current playback burst.
|
|
1127
|
+
private echoReferenceAudio = Buffer.alloc(0);
|
|
1128
|
+
private echoWindowTotalAudioMs = 0;
|
|
1129
|
+
// Consecutive sub-base input expires a reference that can no longer
|
|
1130
|
+
// describe audible playback.
|
|
1131
|
+
private echoSubBaseRunMs = 0;
|
|
1132
|
+
// Once onset eligibility lapses, later user speech cannot seed a new echo
|
|
1133
|
+
// reference in the same playback window.
|
|
1134
|
+
private echoOnsetLapsed = false;
|
|
1135
|
+
// A live speech run that predates playback belongs to the user and bypasses
|
|
1136
|
+
// echo warm-up until that run genuinely resets.
|
|
1137
|
+
private echoWindowGuardCarryover = false;
|
|
980
1138
|
// Mutable so a mid-session `update_config` frame can retune "interrupt
|
|
981
1139
|
// sensitivity" live (see applyConfigUpdate).
|
|
982
1140
|
private bargeInMinSpeechMs: number;
|
|
@@ -1080,13 +1238,38 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1080
1238
|
private readonly finalizeGraceMs: number;
|
|
1081
1239
|
// Rotates through the progress fallback phrases across the session's turns.
|
|
1082
1240
|
private progressPhraseCounter = 0;
|
|
1083
|
-
//
|
|
1084
|
-
//
|
|
1085
|
-
private readonly
|
|
1241
|
+
// Progress-only phrasing service. It never makes routing decisions and
|
|
1242
|
+
// never emits an answer.
|
|
1243
|
+
private readonly progressNarrator: VoiceProgressNarrator | null;
|
|
1086
1244
|
// Complete front-model tunables: the constructor schema-parses the partial
|
|
1087
1245
|
// option once, so every field carries its `liveVoice.frontModel` schema
|
|
1088
1246
|
// default when unset.
|
|
1089
1247
|
private readonly frontModelConfig: LiveVoiceFrontModelConfig;
|
|
1248
|
+
// Complete Flux tunables (the constructor schema-parses the partial option
|
|
1249
|
+
// once, so every field carries its `liveVoice.flux` schema default).
|
|
1250
|
+
private readonly fluxConfig: LiveVoiceFluxConfig;
|
|
1251
|
+
/**
|
|
1252
|
+
* Per-session latch: the provider's committed end-of-turn owns the turn
|
|
1253
|
+
* boundary instead of the silence boundary's front-door hold verdict. Set
|
|
1254
|
+
* when the config flag is on AND the resolved streaming provider declares
|
|
1255
|
+
* `turnDetection: "provider"` in the STT catalog AND the session runs server
|
|
1256
|
+
* VAD. Push-to-talk is excluded deliberately: there the client's release IS
|
|
1257
|
+
* the boundary, and answering while the caller still holds the button is not
|
|
1258
|
+
* turn detection, it is a bug. False leaves every other code path exactly as
|
|
1259
|
+
* it is, with no provider turn detection in the picture.
|
|
1260
|
+
*/
|
|
1261
|
+
private providerTurnEndActive = false;
|
|
1262
|
+
// Wall-clock of the newest above-gate audio chunk, tracked in every
|
|
1263
|
+
// server-VAD session. It is the local VAD's speech-stop mark: the one anchor
|
|
1264
|
+
// the reported end-of-turn latency is measured from whichever decider
|
|
1265
|
+
// commits the turn, and the anchor for the provider fallback deadline. Null
|
|
1266
|
+
// in push-to-talk, where no local VAD runs.
|
|
1267
|
+
private localSpeechStopAtMs: number | null = null;
|
|
1268
|
+
// Fail-open deadline for a provider end-of-turn that never arrives. On
|
|
1269
|
+
// expiry the utterance falls back to the silence-boundary path, so a dead
|
|
1270
|
+
// provider stream degrades to the silence-path behavior instead of a hung
|
|
1271
|
+
// turn.
|
|
1272
|
+
private providerTurnEndTimer: ReturnType<typeof setTimeout> | null = null;
|
|
1090
1273
|
// Effective trailing-silence threshold, mirroring the detector's private
|
|
1091
1274
|
// copy (constructor seed + update_config), reported to the endpoint decider.
|
|
1092
1275
|
private silenceThresholdMs: number;
|
|
@@ -1149,11 +1332,18 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1149
1332
|
context.startFrame.bargeInMinSpeechMs ??
|
|
1150
1333
|
options.bargeInMinSpeechMs ??
|
|
1151
1334
|
DEFAULT_BARGE_IN_MIN_SPEECH_MS;
|
|
1335
|
+
this.echoBargeInMargin =
|
|
1336
|
+
options.echoBargeInMargin ?? DEFAULT_ECHO_BARGE_IN_MARGIN;
|
|
1337
|
+
this.echoEmaHalfLifeMs =
|
|
1338
|
+
options.echoEmaHalfLifeMs ?? DEFAULT_ECHO_EMA_HALF_LIFE_MS;
|
|
1339
|
+
this.echoDrainSlackMs =
|
|
1340
|
+
options.echoDrainSlackMs ?? DEFAULT_ECHO_DRAIN_SLACK_MS;
|
|
1152
1341
|
this.finalizeGraceMs = options.finalizeGraceMs ?? FINALIZE_GRACE_MS;
|
|
1153
|
-
this.
|
|
1342
|
+
this.progressNarrator = options.progressNarrator ?? null;
|
|
1154
1343
|
this.frontModelConfig = LiveVoiceFrontModelConfigSchema.parse(
|
|
1155
1344
|
options.frontModelConfig ?? {},
|
|
1156
1345
|
);
|
|
1346
|
+
this.fluxConfig = LiveVoiceFluxConfigSchema.parse(options.fluxConfig ?? {});
|
|
1157
1347
|
const turnDetectorConfig: TurnDetectorConfig = {
|
|
1158
1348
|
...(options.turnDetectorConfig ?? {}),
|
|
1159
1349
|
...(context.startFrame.silenceThresholdMs !== undefined
|
|
@@ -1184,6 +1374,12 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1184
1374
|
return;
|
|
1185
1375
|
}
|
|
1186
1376
|
|
|
1377
|
+
// Before the preflight, not after: a session rejected for missing
|
|
1378
|
+
// credentials is precisely the one the failure rate needs to count, and
|
|
1379
|
+
// recording the start only once `ready` goes out would hide every such
|
|
1380
|
+
// session from both the numerator and the denominator.
|
|
1381
|
+
recordLiveVoiceSessionStarted(this.context.sessionId);
|
|
1382
|
+
|
|
1187
1383
|
if (this.resolveCredentialReadiness) {
|
|
1188
1384
|
const readiness = await this.resolveCredentialReadiness();
|
|
1189
1385
|
if (readiness.status === "not-ready") {
|
|
@@ -1237,9 +1433,44 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1237
1433
|
case "update_config":
|
|
1238
1434
|
this.applyConfigUpdate(frame);
|
|
1239
1435
|
return;
|
|
1436
|
+
case "attach_image":
|
|
1437
|
+
this.persistPhoto(frame);
|
|
1438
|
+
return;
|
|
1240
1439
|
}
|
|
1241
1440
|
}
|
|
1242
1441
|
|
|
1442
|
+
/**
|
|
1443
|
+
* Persist a photo taken mid-call into the conversation, running no turn.
|
|
1444
|
+
*
|
|
1445
|
+
* Fire-and-forget on purpose: the persist waits out any in-flight turn, and
|
|
1446
|
+
* the socket must keep pumping audio meanwhile. The client already showed a
|
|
1447
|
+
* thumbnail from the local frame, so nothing on screen is waiting on this.
|
|
1448
|
+
*
|
|
1449
|
+
* The photo becomes its own user message rather than riding the next spoken
|
|
1450
|
+
* turn, which is what makes shutter-then-speak and speak-then-shutter
|
|
1451
|
+
* behave the same: either way the model's history has the image by the time
|
|
1452
|
+
* it answers. See `live-voice-photo.ts` for the full reasoning.
|
|
1453
|
+
*/
|
|
1454
|
+
private persistPhoto(frame: LiveVoiceClientAttachImageFrame): void {
|
|
1455
|
+
void persistLiveVoicePhoto(this.conversationId, frame.attachmentId).then(
|
|
1456
|
+
(result) => {
|
|
1457
|
+
if (!result.ok && !this.isClosed) {
|
|
1458
|
+
void this.sendFrame({
|
|
1459
|
+
type: "error",
|
|
1460
|
+
code: LiveVoiceProtocolErrorCode.InvalidFrame,
|
|
1461
|
+
message: "Could not attach that photo to the conversation.",
|
|
1462
|
+
// Names the photo as the casualty so the client can retract the
|
|
1463
|
+
// thumbnail it already showed, rather than filing this with the
|
|
1464
|
+
// transient transcriber and TTS blips that share `recoverable`.
|
|
1465
|
+
frameType: "attach_image",
|
|
1466
|
+
// The session is fine; only this photo failed.
|
|
1467
|
+
recoverable: true,
|
|
1468
|
+
});
|
|
1469
|
+
}
|
|
1470
|
+
},
|
|
1471
|
+
);
|
|
1472
|
+
}
|
|
1473
|
+
|
|
1243
1474
|
/**
|
|
1244
1475
|
* Apply a mid-session `update_config` frame: retune the live turn detector's
|
|
1245
1476
|
* pause ("pause before reply") and/or the barge-in guard ("interrupt
|
|
@@ -1261,11 +1492,24 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1261
1492
|
await this.handleAudio(Buffer.from(chunk));
|
|
1262
1493
|
}
|
|
1263
1494
|
|
|
1264
|
-
async close(
|
|
1495
|
+
async close(reason: LiveVoiceSessionCloseReason): Promise<void> {
|
|
1265
1496
|
if (this.isClosed) {
|
|
1266
1497
|
return;
|
|
1267
1498
|
}
|
|
1268
1499
|
|
|
1500
|
+
// Recorded first, and independently of `shouldEmitSessionEndMetrics`
|
|
1501
|
+
// below: that flag governs the client-facing `metrics` frame, and a
|
|
1502
|
+
// failed session (which suppresses the frame) is the one whose end
|
|
1503
|
+
// telemetry matters most. Session duration downstream is this row's
|
|
1504
|
+
// `recorded_at` minus the started row's, so it must be written before the
|
|
1505
|
+
// teardown below, which awaits a pending continuation and can run long.
|
|
1506
|
+
const failed = this.state === "failed";
|
|
1507
|
+
recordLiveVoiceSessionEnded({
|
|
1508
|
+
sessionId: this.context.sessionId,
|
|
1509
|
+
screen: liveVoiceEndScreen(reason, failed ? this.failureCode : null),
|
|
1510
|
+
outcome: failed ? "failed" : "completed",
|
|
1511
|
+
});
|
|
1512
|
+
|
|
1269
1513
|
const shouldEmitSessionEndMetrics = this.state !== "failed";
|
|
1270
1514
|
this.state = "closed";
|
|
1271
1515
|
// Retire the island before the teardown below starts awaiting things. A
|
|
@@ -1275,6 +1519,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1275
1519
|
this.liveActivityReporter.end();
|
|
1276
1520
|
this.turnDetector?.dispose();
|
|
1277
1521
|
this.clearEndpointExtensionTimer();
|
|
1522
|
+
this.clearProviderTurnEndTimer();
|
|
1278
1523
|
// There is no longer anyone on the call to speak to, so a queued
|
|
1279
1524
|
// announcement cannot be spoken — but the answer behind it is finished
|
|
1280
1525
|
// work the user asked for, so it takes the same conversation route a
|
|
@@ -1316,6 +1561,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1316
1561
|
// Persistent re-arm: the shared stream is already open, so the cycle
|
|
1317
1562
|
// goes straight to streaming with no resolve/start round-trip.
|
|
1318
1563
|
utterance.transcriber = shared;
|
|
1564
|
+
utterance.dialedSttProvider = shared.providerId;
|
|
1319
1565
|
return await this.activateUtterance(utterance, replayTurnEnd);
|
|
1320
1566
|
}
|
|
1321
1567
|
// The shared stream is pinned to the old language, so retire it and
|
|
@@ -1325,11 +1571,28 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1325
1571
|
}
|
|
1326
1572
|
|
|
1327
1573
|
try {
|
|
1328
|
-
// One
|
|
1329
|
-
//
|
|
1330
|
-
// language and the recorded sharedTranscriberLanguage
|
|
1331
|
-
// when config changes while the resolver awaits
|
|
1332
|
-
|
|
1574
|
+
// One config snapshot serves the dial, the re-arm comparison and the
|
|
1575
|
+
// latch seed below: passing the language to the resolver keeps the
|
|
1576
|
+
// stream's actual language and the recorded sharedTranscriberLanguage
|
|
1577
|
+
// identical even when config changes while the resolver awaits
|
|
1578
|
+
// credentials.
|
|
1579
|
+
const stt = getConfig().services.stt;
|
|
1580
|
+
const sttLanguage = stt.language;
|
|
1581
|
+
// Seed the latch from the CONFIGURED provider before the dial, not
|
|
1582
|
+
// only from the resolved one after it. `start()` sends `ready` without
|
|
1583
|
+
// waiting on this resolve, so the caller can speak and close an entire
|
|
1584
|
+
// silence boundary while the provider handshake is still in flight. A
|
|
1585
|
+
// latch still on its default `false` at that boundary hands the first
|
|
1586
|
+
// turn of the session to the silence path and then ignores the
|
|
1587
|
+
// provider end-of-turn that follows, which is invisible from the
|
|
1588
|
+
// outside: the session looks like a turn-detecting session while its
|
|
1589
|
+
// opening turn is not one. The resolved provider reconciles this guess
|
|
1590
|
+
// below.
|
|
1591
|
+
this.setProviderTurnEndActive(
|
|
1592
|
+
this.fluxConfig.turnEnd.enabled &&
|
|
1593
|
+
supportsProviderTurnDetection(stt.provider as SttProviderId) &&
|
|
1594
|
+
this.turnDetector !== null,
|
|
1595
|
+
);
|
|
1333
1596
|
const transcriber = await this.resolveTranscriber({
|
|
1334
1597
|
sampleRate: this.context.startFrame.audio.sampleRate,
|
|
1335
1598
|
...(sttLanguage ? { language: sttLanguage } : {}),
|
|
@@ -1341,6 +1604,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1341
1604
|
}
|
|
1342
1605
|
|
|
1343
1606
|
if (!transcriber) {
|
|
1607
|
+
// No stream answered, so no end-of-turn ever will: the guess above
|
|
1608
|
+
// must not outlive the dial that disproved it.
|
|
1609
|
+
this.setProviderTurnEndActive(false);
|
|
1344
1610
|
return {
|
|
1345
1611
|
status: "unavailable",
|
|
1346
1612
|
message: unavailableTranscriberMessage(),
|
|
@@ -1348,6 +1614,17 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1348
1614
|
}
|
|
1349
1615
|
|
|
1350
1616
|
utterance.transcriber = transcriber;
|
|
1617
|
+
utterance.dialedSttProvider = transcriber.providerId;
|
|
1618
|
+
// Reconcile the pre-dial guess with the provider that actually
|
|
1619
|
+
// answered. The resolver reads the provider from config, which cannot
|
|
1620
|
+
// change under a live session, so this normally confirms the guess; it
|
|
1621
|
+
// clears it when the dial fell back to another provider or resolved
|
|
1622
|
+
// one the config did not name.
|
|
1623
|
+
this.setProviderTurnEndActive(
|
|
1624
|
+
this.fluxConfig.turnEnd.enabled &&
|
|
1625
|
+
supportsProviderTurnDetection(transcriber.providerId) &&
|
|
1626
|
+
this.turnDetector !== null,
|
|
1627
|
+
);
|
|
1351
1628
|
if (
|
|
1352
1629
|
this.turnDetector &&
|
|
1353
1630
|
typeof transcriber.finalizeUtterance === "function"
|
|
@@ -1381,6 +1658,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1381
1658
|
if (this.isUtteranceStale(utterance)) {
|
|
1382
1659
|
return { status: "stale" };
|
|
1383
1660
|
}
|
|
1661
|
+
// The dial threw, so the pre-dial guess is disproved the same way the
|
|
1662
|
+
// unavailable case disproves it: nothing will send an end-of-turn.
|
|
1663
|
+
this.setProviderTurnEndActive(false);
|
|
1384
1664
|
return {
|
|
1385
1665
|
status: "error",
|
|
1386
1666
|
message: `Live voice transcription could not be started: ${errorMessage(
|
|
@@ -1508,6 +1788,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1508
1788
|
|
|
1509
1789
|
this.state = "failed";
|
|
1510
1790
|
this.clearEndpointExtensionTimer();
|
|
1791
|
+
this.clearProviderTurnEndTimer();
|
|
1511
1792
|
await this.sendFrame({
|
|
1512
1793
|
type: "error",
|
|
1513
1794
|
code:
|
|
@@ -1568,12 +1849,29 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1568
1849
|
return;
|
|
1569
1850
|
}
|
|
1570
1851
|
|
|
1571
|
-
const
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
|
|
1852
|
+
for (const classified of this.classifyVadEnergy(chunk)) {
|
|
1853
|
+
await this.handleClassifiedVadAudio(detector, classified);
|
|
1854
|
+
}
|
|
1855
|
+
}
|
|
1856
|
+
|
|
1857
|
+
private async handleClassifiedVadAudio(
|
|
1858
|
+
detector: MediaTurnDetector,
|
|
1859
|
+
classified: VadClassifiedChunk,
|
|
1860
|
+
): Promise<void> {
|
|
1861
|
+
const { chunk, classification: energyClassification } = classified;
|
|
1862
|
+
const hasSpeech = energyClassification === "speech";
|
|
1575
1863
|
detector.onMediaChunk(hasSpeech);
|
|
1576
|
-
this.trackBargeInGuard(
|
|
1864
|
+
this.trackBargeInGuard(energyClassification, chunk);
|
|
1865
|
+
if (hasSpeech) {
|
|
1866
|
+
this.localSpeechStopAtMs = Date.now();
|
|
1867
|
+
}
|
|
1868
|
+
|
|
1869
|
+
// Playback echo is neither user audio nor useful pre-roll. Dropping it
|
|
1870
|
+
// prevents the assistant's reply from reaching transcription as a ghost
|
|
1871
|
+
// follow-up turn.
|
|
1872
|
+
if (energyClassification === "echo") {
|
|
1873
|
+
return;
|
|
1874
|
+
}
|
|
1577
1875
|
|
|
1578
1876
|
// Idle mic: hold silent chunks in the bounded pre-roll instead of
|
|
1579
1877
|
// collecting or streaming them; flushed on speech onset so the
|
|
@@ -1651,6 +1949,193 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1651
1949
|
await this.routeVadAudio(utterance, chunk);
|
|
1652
1950
|
}
|
|
1653
1951
|
|
|
1952
|
+
/**
|
|
1953
|
+
* Classify microphone energy while keeping assistant playback echo out of
|
|
1954
|
+
* barge-in, turn detection, pre-roll, and transcription.
|
|
1955
|
+
*
|
|
1956
|
+
* A short onset probe must correlate with PCM sent to the speaker before its
|
|
1957
|
+
* microphone power can seed the adaptive threshold. Nonmatching probe audio
|
|
1958
|
+
* is replayed through VAD in original order, so a user who talks at playback
|
|
1959
|
+
* onset is neither learned as echo nor lost. Once seeded, the EMA follows
|
|
1960
|
+
* confirmed echo while speech above the learned margin remains frozen out.
|
|
1961
|
+
*/
|
|
1962
|
+
private classifyVadEnergy(chunk: Buffer): VadClassifiedChunk[] {
|
|
1963
|
+
const baseThreshold =
|
|
1964
|
+
this.speechEnergyThreshold ?? DEFAULT_SPEECH_ENERGY_THRESHOLD;
|
|
1965
|
+
const meanAmplitude = pcm16MeanAmplitude(chunk);
|
|
1966
|
+
if (
|
|
1967
|
+
this.echoBargeInMargin <= 1 ||
|
|
1968
|
+
!this.isAssistantPlaybackEchoPossible()
|
|
1969
|
+
) {
|
|
1970
|
+
this.resetEchoReference();
|
|
1971
|
+
return [
|
|
1972
|
+
this.classifyAtFixedThreshold(chunk, baseThreshold, meanAmplitude),
|
|
1973
|
+
];
|
|
1974
|
+
}
|
|
1975
|
+
|
|
1976
|
+
if (this.echoWindowTotalAudioMs === 0) {
|
|
1977
|
+
this.echoWindowGuardCarryover =
|
|
1978
|
+
this.pendingBargeIn !== null && this.pendingBargeIn.speechMs > 0;
|
|
1979
|
+
} else if (this.pendingBargeIn === null) {
|
|
1980
|
+
this.echoWindowGuardCarryover = false;
|
|
1981
|
+
}
|
|
1982
|
+
|
|
1983
|
+
const chunkMs = pcm16DurationMs(
|
|
1984
|
+
chunk.byteLength,
|
|
1985
|
+
this.context.startFrame.audio.sampleRate,
|
|
1986
|
+
);
|
|
1987
|
+
const onsetWasEligible =
|
|
1988
|
+
!this.echoOnsetLapsed &&
|
|
1989
|
+
this.echoWindowTotalAudioMs < ECHO_ONSET_ELIGIBILITY_MS;
|
|
1990
|
+
this.echoWindowTotalAudioMs += chunkMs;
|
|
1991
|
+
|
|
1992
|
+
if (this.echoProbeChunks.length > 0) {
|
|
1993
|
+
this.echoProbeChunks.push(Buffer.from(chunk));
|
|
1994
|
+
return this.resolveEchoProbe(baseThreshold);
|
|
1995
|
+
}
|
|
1996
|
+
|
|
1997
|
+
if (meanAmplitude <= baseThreshold) {
|
|
1998
|
+
this.echoSubBaseRunMs += chunkMs;
|
|
1999
|
+
if (this.echoSubBaseRunMs >= ECHO_ONSET_ELIGIBILITY_MS) {
|
|
2000
|
+
this.echoEnergyEma = 0;
|
|
2001
|
+
this.echoOnsetLapsed = true;
|
|
2002
|
+
}
|
|
2003
|
+
return [{ chunk, classification: "silence" }];
|
|
2004
|
+
}
|
|
2005
|
+
|
|
2006
|
+
this.echoSubBaseRunMs = 0;
|
|
2007
|
+
if (
|
|
2008
|
+
this.echoEnergyEma === 0 &&
|
|
2009
|
+
onsetWasEligible &&
|
|
2010
|
+
!this.echoWindowGuardCarryover
|
|
2011
|
+
) {
|
|
2012
|
+
this.echoProbeChunks.push(Buffer.from(chunk));
|
|
2013
|
+
return this.resolveEchoProbe(baseThreshold);
|
|
2014
|
+
}
|
|
2015
|
+
|
|
2016
|
+
if (this.echoEnergyEma === 0) {
|
|
2017
|
+
this.echoOnsetLapsed = true;
|
|
2018
|
+
return [{ chunk, classification: "speech" }];
|
|
2019
|
+
}
|
|
2020
|
+
|
|
2021
|
+
const speechThreshold = Math.max(
|
|
2022
|
+
baseThreshold,
|
|
2023
|
+
this.echoBargeInMargin * this.echoEnergyEma,
|
|
2024
|
+
);
|
|
2025
|
+
if (meanAmplitude > speechThreshold) {
|
|
2026
|
+
const guardHasSpeech =
|
|
2027
|
+
this.pendingBargeIn !== null && this.pendingBargeIn.speechMs > 0;
|
|
2028
|
+
if (!guardHasSpeech && this.echoMatchesAssistant(chunk)) {
|
|
2029
|
+
this.updateEchoEnergy(meanAmplitude, chunkMs);
|
|
2030
|
+
return [{ chunk, classification: "echo" }];
|
|
2031
|
+
}
|
|
2032
|
+
return [{ chunk, classification: "speech" }];
|
|
2033
|
+
}
|
|
2034
|
+
|
|
2035
|
+
this.updateEchoEnergy(meanAmplitude, chunkMs);
|
|
2036
|
+
return [{ chunk, classification: "echo" }];
|
|
2037
|
+
}
|
|
2038
|
+
|
|
2039
|
+
private resolveEchoProbe(baseThreshold: number): VadClassifiedChunk[] {
|
|
2040
|
+
const probe = Buffer.concat(this.echoProbeChunks);
|
|
2041
|
+
const probeAudioMs = pcm16DurationMs(
|
|
2042
|
+
probe.byteLength,
|
|
2043
|
+
this.context.startFrame.audio.sampleRate,
|
|
2044
|
+
);
|
|
2045
|
+
if (
|
|
2046
|
+
probeAudioMs >= ECHO_CORRELATION_MIN_MS &&
|
|
2047
|
+
this.echoMatchesAssistant(probe)
|
|
2048
|
+
) {
|
|
2049
|
+
this.echoEnergyEma = Math.max(baseThreshold, pcm16MeanAmplitude(probe));
|
|
2050
|
+
const chunks = this.echoProbeChunks.splice(0);
|
|
2051
|
+
return chunks.map((chunk) => ({ chunk, classification: "echo" }));
|
|
2052
|
+
}
|
|
2053
|
+
if (probeAudioMs < ECHO_CORRELATION_PROBE_MS) {
|
|
2054
|
+
return [];
|
|
2055
|
+
}
|
|
2056
|
+
|
|
2057
|
+
this.echoOnsetLapsed = true;
|
|
2058
|
+
const chunks = this.echoProbeChunks.splice(0);
|
|
2059
|
+
return chunks.map((chunk) =>
|
|
2060
|
+
this.classifyAtFixedThreshold(chunk, baseThreshold),
|
|
2061
|
+
);
|
|
2062
|
+
}
|
|
2063
|
+
|
|
2064
|
+
private echoMatchesAssistant(chunk: Buffer): boolean {
|
|
2065
|
+
const sampleRate = this.context.startFrame.audio.sampleRate;
|
|
2066
|
+
const minimumBytes = Math.ceil(
|
|
2067
|
+
(sampleRate * ECHO_CORRELATION_MIN_MS * 2) / 1_000,
|
|
2068
|
+
);
|
|
2069
|
+
if (
|
|
2070
|
+
chunk.byteLength < minimumBytes ||
|
|
2071
|
+
this.echoReferenceAudio.byteLength < minimumBytes
|
|
2072
|
+
) {
|
|
2073
|
+
return false;
|
|
2074
|
+
}
|
|
2075
|
+
const probeByteLength = Math.min(
|
|
2076
|
+
chunk.byteLength,
|
|
2077
|
+
Math.ceil((sampleRate * ECHO_CORRELATION_PROBE_MS * 2) / 1_000),
|
|
2078
|
+
);
|
|
2079
|
+
return (
|
|
2080
|
+
pcm16MaxNormalizedCorrelation(
|
|
2081
|
+
chunk.subarray(0, probeByteLength),
|
|
2082
|
+
this.echoReferenceAudio,
|
|
2083
|
+
) >= ECHO_CORRELATION_THRESHOLD
|
|
2084
|
+
);
|
|
2085
|
+
}
|
|
2086
|
+
|
|
2087
|
+
private updateEchoEnergy(meanAmplitude: number, chunkMs: number): void {
|
|
2088
|
+
const alpha = 1 - 0.5 ** (chunkMs / this.echoEmaHalfLifeMs);
|
|
2089
|
+
this.echoEnergyEma =
|
|
2090
|
+
alpha * meanAmplitude + (1 - alpha) * this.echoEnergyEma;
|
|
2091
|
+
}
|
|
2092
|
+
|
|
2093
|
+
private classifyAtFixedThreshold(
|
|
2094
|
+
chunk: Buffer,
|
|
2095
|
+
baseThreshold: number,
|
|
2096
|
+
meanAmplitude = pcm16MeanAmplitude(chunk),
|
|
2097
|
+
): VadClassifiedChunk {
|
|
2098
|
+
return {
|
|
2099
|
+
chunk,
|
|
2100
|
+
classification: meanAmplitude > baseThreshold ? "speech" : "silence",
|
|
2101
|
+
};
|
|
2102
|
+
}
|
|
2103
|
+
|
|
2104
|
+
private isAssistantPlaybackEchoPossible(): boolean {
|
|
2105
|
+
return (
|
|
2106
|
+
Date.now() < this.assistantPlaybackTailUntilMs + this.echoDrainSlackMs
|
|
2107
|
+
);
|
|
2108
|
+
}
|
|
2109
|
+
|
|
2110
|
+
private resetEchoReference(): void {
|
|
2111
|
+
this.echoEnergyEma = 0;
|
|
2112
|
+
this.echoProbeChunks = [];
|
|
2113
|
+
this.echoReferenceAudio = Buffer.alloc(0);
|
|
2114
|
+
this.echoWindowTotalAudioMs = 0;
|
|
2115
|
+
this.echoSubBaseRunMs = 0;
|
|
2116
|
+
this.echoOnsetLapsed = false;
|
|
2117
|
+
this.echoWindowGuardCarryover = false;
|
|
2118
|
+
}
|
|
2119
|
+
|
|
2120
|
+
private appendEchoReference(chunk: LiveVoiceTtsAudioChunk): void {
|
|
2121
|
+
if (
|
|
2122
|
+
chunk.contentType.split(";", 1)[0]?.trim().toLowerCase() !==
|
|
2123
|
+
"audio/pcm" ||
|
|
2124
|
+
chunk.sampleRate !== this.context.startFrame.audio.sampleRate
|
|
2125
|
+
) {
|
|
2126
|
+
return;
|
|
2127
|
+
}
|
|
2128
|
+
const audio = Buffer.from(chunk.dataBase64, "base64");
|
|
2129
|
+
const maxBytes = Math.ceil(
|
|
2130
|
+
(chunk.sampleRate * ECHO_REFERENCE_MAX_MS * 2) / 1_000,
|
|
2131
|
+
);
|
|
2132
|
+
const combined = Buffer.concat([this.echoReferenceAudio, audio]);
|
|
2133
|
+
this.echoReferenceAudio =
|
|
2134
|
+
combined.byteLength > maxBytes
|
|
2135
|
+
? combined.subarray(combined.byteLength - maxBytes)
|
|
2136
|
+
: combined;
|
|
2137
|
+
}
|
|
2138
|
+
|
|
1654
2139
|
private async routeVadAudio(
|
|
1655
2140
|
utterance: UtteranceCycle,
|
|
1656
2141
|
chunk: Buffer,
|
|
@@ -1780,6 +2265,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1780
2265
|
// utterance keeps accumulating and the detector fires a fresh turn-end.
|
|
1781
2266
|
this.vadSpeechGeneration += 1;
|
|
1782
2267
|
this.clearEndpointExtensionTimer();
|
|
2268
|
+
// ...and so is a fallback deadline armed for the boundary the caller just
|
|
2269
|
+
// spoke through. The next silence boundary arms a fresh one.
|
|
2270
|
+
this.clearProviderTurnEndTimer();
|
|
1783
2271
|
|
|
1784
2272
|
// Speech resumed while a speculative leg was awaiting its verdict: the
|
|
1785
2273
|
// pause was mid-thought after all. Discard silently (no frames were ever
|
|
@@ -1798,7 +2286,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1798
2286
|
// The client can still be draining audible playback after tts_done
|
|
1799
2287
|
// (the turn is already cleared server-side) — that tail deserves the
|
|
1800
2288
|
// same guard, or a noise blip clips the reply's last words.
|
|
1801
|
-
const drainingPlayback =
|
|
2289
|
+
const drainingPlayback = this.isAssistantPlaybackEchoPossible();
|
|
1802
2290
|
|
|
1803
2291
|
if ((bargeableTurn || drainingPlayback) && this.bargeInMinSpeechMs > 0) {
|
|
1804
2292
|
// Onset audio keeps flowing into the cycle/pre-roll while the guard
|
|
@@ -1820,13 +2308,14 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1820
2308
|
}
|
|
1821
2309
|
}
|
|
1822
2310
|
|
|
1823
|
-
//
|
|
1824
|
-
//
|
|
1825
|
-
//
|
|
1826
|
-
//
|
|
1827
|
-
|
|
1828
|
-
|
|
1829
|
-
|
|
2311
|
+
// Advance the sustained-speech barge-in guard by one server-VAD chunk.
|
|
2312
|
+
// Speech accumulates toward bargeInMinSpeechMs, short true-silence gaps are
|
|
2313
|
+
// tolerated, and classified playback echo resets the run immediately.
|
|
2314
|
+
// Longer or mostly silent runs reset through the existing gap limits.
|
|
2315
|
+
private trackBargeInGuard(
|
|
2316
|
+
classification: VadEnergyClassification,
|
|
2317
|
+
chunk: Buffer,
|
|
2318
|
+
): void {
|
|
1830
2319
|
const guard = this.pendingBargeIn;
|
|
1831
2320
|
if (!guard) {
|
|
1832
2321
|
return;
|
|
@@ -1835,7 +2324,11 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1835
2324
|
chunk.byteLength,
|
|
1836
2325
|
this.context.startFrame.audio.sampleRate,
|
|
1837
2326
|
);
|
|
1838
|
-
if (
|
|
2327
|
+
if (classification === "echo") {
|
|
2328
|
+
this.resetBargeInGuardRun();
|
|
2329
|
+
return;
|
|
2330
|
+
}
|
|
2331
|
+
if (classification === "silence") {
|
|
1839
2332
|
guard.silenceMs += chunkMs;
|
|
1840
2333
|
guard.toleratedSilenceMs += chunkMs;
|
|
1841
2334
|
// Strictly greater on the per-gap check: a gap of exactly
|
|
@@ -1849,9 +2342,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1849
2342
|
guard.toleratedSilenceMs >
|
|
1850
2343
|
this.bargeInMinSpeechMs * BARGE_IN_MAX_TOLERATED_SILENCE_RATIO
|
|
1851
2344
|
) {
|
|
1852
|
-
|
|
1853
|
-
guard.silenceMs = 0;
|
|
1854
|
-
guard.toleratedSilenceMs = 0;
|
|
2345
|
+
this.resetBargeInGuardRun();
|
|
1855
2346
|
}
|
|
1856
2347
|
return;
|
|
1857
2348
|
}
|
|
@@ -1869,6 +2360,21 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1869
2360
|
}
|
|
1870
2361
|
}
|
|
1871
2362
|
|
|
2363
|
+
private resetBargeInGuardRun(): void {
|
|
2364
|
+
const guard = this.pendingBargeIn;
|
|
2365
|
+
if (!guard) {
|
|
2366
|
+
return;
|
|
2367
|
+
}
|
|
2368
|
+
guard.speechMs = 0;
|
|
2369
|
+
guard.silenceMs = 0;
|
|
2370
|
+
guard.toleratedSilenceMs = 0;
|
|
2371
|
+
if (this.echoWindowGuardCarryover) {
|
|
2372
|
+
this.echoWindowGuardCarryover = false;
|
|
2373
|
+
this.echoEnergyEma = 0;
|
|
2374
|
+
this.echoProbeChunks = [];
|
|
2375
|
+
}
|
|
2376
|
+
}
|
|
2377
|
+
|
|
1872
2378
|
private bargeIn(turn: ActiveAssistantTurn): void {
|
|
1873
2379
|
// Abort synchronously so no tts_audio frame can follow turn_cancelled,
|
|
1874
2380
|
// and settle the cancelled turn's metrics so the next utterance's marks
|
|
@@ -1918,7 +2424,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1918
2424
|
// Tagged reason: provider catch-sites classify untagged caller aborts as
|
|
1919
2425
|
// retryable transport failures (ERROR log + futile retry against the
|
|
1920
2426
|
// aborted signal). This signal reaches the brain leg and any in-flight
|
|
1921
|
-
//
|
|
2427
|
+
// progress narration.
|
|
1922
2428
|
turn.abortController.abort(
|
|
1923
2429
|
createAbortReason("voice_session_aborted", "live-voice-barge-in"),
|
|
1924
2430
|
);
|
|
@@ -2456,8 +2962,13 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2456
2962
|
// Spoken, because opening the room is only a cue for someone looking at
|
|
2457
2963
|
// the screen, and the case this exists for is a phone the user has put
|
|
2458
2964
|
// down. One line, not narration: the turn is not working, it is waiting,
|
|
2459
|
-
// and it says which
|
|
2460
|
-
|
|
2965
|
+
// and it says which, in the turn's spoken language, like every other
|
|
2966
|
+
// filler phrase.
|
|
2967
|
+
this.enqueueFillerPhrase(
|
|
2968
|
+
turn,
|
|
2969
|
+
approvalPendingPhraseFor(turn.language),
|
|
2970
|
+
this.fixedPhraseLanguage(turn, APPROVAL_PENDING_PHRASE_BY_LANGUAGE),
|
|
2971
|
+
);
|
|
2461
2972
|
}
|
|
2462
2973
|
|
|
2463
2974
|
/** Clear the wait once a decision lands, so the turn narrates normally again. */
|
|
@@ -2666,8 +3177,10 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2666
3177
|
this.pendingBargeIn = null;
|
|
2667
3178
|
if (reason === "max-duration") {
|
|
2668
3179
|
// A max-duration boundary always releases: drop any pending hold
|
|
2669
|
-
// replay so it cannot re-fire a
|
|
3180
|
+
// replay (or provider fallback deadline) so it cannot re-fire a
|
|
3181
|
+
// boundary this one already owns.
|
|
2670
3182
|
this.clearEndpointExtensionTimer();
|
|
3183
|
+
this.clearProviderTurnEndTimer();
|
|
2671
3184
|
}
|
|
2672
3185
|
const utterance = this.currentUtterance;
|
|
2673
3186
|
if (!utterance || utterance.released || utterance.completed) {
|
|
@@ -2689,6 +3202,20 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2689
3202
|
// release (ptt_release forced the boundary) is the user saying "answer
|
|
2690
3203
|
// now" — never second-guess it.
|
|
2691
3204
|
if (reason === "silence" && !manualRelease) {
|
|
3205
|
+
// The provider owns this boundary: its end-of-turn commits the
|
|
3206
|
+
// utterance, so the silence timer only arms the fail-open deadline. An
|
|
3207
|
+
// utterance whose deadline already elapsed falls through to the hold
|
|
3208
|
+
// path below, which is the whole point of the fallback.
|
|
3209
|
+
if (this.providerTurnEndActive && !utterance.providerTurnEndTimedOut) {
|
|
3210
|
+
// Stamp the speech run this boundary closed. This is the staleness
|
|
3211
|
+
// signal of last resort, used only for a provider that does not
|
|
3212
|
+
// number its turns: where the provider's own turn index is on the
|
|
3213
|
+
// wire it answers the question directly (see
|
|
3214
|
+
// isStaleProviderTurnEnd).
|
|
3215
|
+
utterance.turnBoundaryGeneration = this.vadSpeechGeneration;
|
|
3216
|
+
this.armProviderTurnEndFallbackTimer(utterance);
|
|
3217
|
+
return;
|
|
3218
|
+
}
|
|
2692
3219
|
if (await this.launchSpeculativeAssistantTurn(utterance)) {
|
|
2693
3220
|
return;
|
|
2694
3221
|
}
|
|
@@ -2719,6 +3246,217 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2719
3246
|
}, this.frontModelConfig.endpointExtensionMs);
|
|
2720
3247
|
}
|
|
2721
3248
|
|
|
3249
|
+
// Arms the fail-open deadline for a provider end-of-turn. A turn-detecting
|
|
3250
|
+
// provider force-ends its own turn after `eotTimeoutMs` of silence, so once
|
|
3251
|
+
// that budget plus a margin has passed since the caller stopped speaking, no
|
|
3252
|
+
// end-of-turn is coming and the utterance replays this boundary on the
|
|
3253
|
+
// silence path.
|
|
3254
|
+
// `waitMsOverride` collapses that wait when the caller already knows no
|
|
3255
|
+
// end-of-turn is coming (see setProviderTurnEndActive).
|
|
3256
|
+
private armProviderTurnEndFallbackTimer(
|
|
3257
|
+
utterance: UtteranceCycle,
|
|
3258
|
+
waitMsOverride?: number,
|
|
3259
|
+
): void {
|
|
3260
|
+
this.clearProviderTurnEndTimer();
|
|
3261
|
+
const budgetMs =
|
|
3262
|
+
this.fluxConfig.eotTimeoutMs + PROVIDER_TURN_END_FALLBACK_MARGIN_MS;
|
|
3263
|
+
const waitMs =
|
|
3264
|
+
waitMsOverride ?? Math.max(0, budgetMs - this.msSinceLocalSpeechStop());
|
|
3265
|
+
this.providerTurnEndTimer = setTimeout(() => {
|
|
3266
|
+
this.providerTurnEndTimer = null;
|
|
3267
|
+
if (
|
|
3268
|
+
this.currentUtterance !== utterance ||
|
|
3269
|
+
utterance.released ||
|
|
3270
|
+
utterance.completed ||
|
|
3271
|
+
utterance.assistantTurnStarted ||
|
|
3272
|
+
// Speech resumed (the detector owns the next boundary). Onset also
|
|
3273
|
+
// clears this timer, so this is a belt to that suspender.
|
|
3274
|
+
this.turnDetector?.isActive
|
|
3275
|
+
) {
|
|
3276
|
+
return;
|
|
3277
|
+
}
|
|
3278
|
+
utterance.providerTurnEndTimedOut = true;
|
|
3279
|
+
log.warn(
|
|
3280
|
+
{ budgetMs, waitMs },
|
|
3281
|
+
"No provider end-of-turn is coming; falling back to the silence boundary for this utterance",
|
|
3282
|
+
);
|
|
3283
|
+
this.handleVadUtteranceEnd("silence");
|
|
3284
|
+
}, waitMs);
|
|
3285
|
+
}
|
|
3286
|
+
|
|
3287
|
+
/**
|
|
3288
|
+
* Flips the provider end-of-turn latch, unwinding an optimistic arm.
|
|
3289
|
+
*
|
|
3290
|
+
* `beginUtterance` arms the latch from the configured provider before the
|
|
3291
|
+
* dial resolves, so a silence boundary can already have deferred to the
|
|
3292
|
+
* provider by the time the resolved one says otherwise. That deferred
|
|
3293
|
+
* boundary is parked on the fail-open deadline, a whole end-of-turn budget
|
|
3294
|
+
* away, waiting for an event that will now never arrive. Collapse the wait to
|
|
3295
|
+
* zero rather than burn the budget: the deadline body re-checks the cycle
|
|
3296
|
+
* and replays the silence boundary, which with the latch down takes the
|
|
3297
|
+
* ordinary hold path. Replaying through the deadline instead of calling
|
|
3298
|
+
* `handleVadUtteranceEnd` directly keeps the release off the arming
|
|
3299
|
+
* caller's stack, which is still mid-dial.
|
|
3300
|
+
*
|
|
3301
|
+
* A cleared latch with no deadline armed needs no unwind: either no
|
|
3302
|
+
* boundary ever deferred, or the caller resumed speaking and
|
|
3303
|
+
* `handleVadSpeechStart` already dropped the deadline, leaving the detector
|
|
3304
|
+
* owning the next boundary. The cycle's `turnBoundaryGeneration` stamp is
|
|
3305
|
+
* left as it is: `isStaleProviderTurnEnd` is only ever consulted from
|
|
3306
|
+
* `handleProviderTurnEnd`, which returns immediately once the latch is down.
|
|
3307
|
+
*/
|
|
3308
|
+
private setProviderTurnEndActive(active: boolean): void {
|
|
3309
|
+
if (active === this.providerTurnEndActive) {
|
|
3310
|
+
return;
|
|
3311
|
+
}
|
|
3312
|
+
this.providerTurnEndActive = active;
|
|
3313
|
+
if (active || this.providerTurnEndTimer === null) {
|
|
3314
|
+
return;
|
|
3315
|
+
}
|
|
3316
|
+
const utterance = this.currentUtterance;
|
|
3317
|
+
this.clearProviderTurnEndTimer();
|
|
3318
|
+
if (utterance && !utterance.released && !utterance.completed) {
|
|
3319
|
+
this.armProviderTurnEndFallbackTimer(utterance, 0);
|
|
3320
|
+
}
|
|
3321
|
+
}
|
|
3322
|
+
|
|
3323
|
+
/**
|
|
3324
|
+
* Record the provider turn a cycle is currently inside. Providers without
|
|
3325
|
+
* turn numbering send no index, which leaves the cycle on the local speech
|
|
3326
|
+
* generation as its only staleness signal (see isStaleProviderTurnEnd).
|
|
3327
|
+
*/
|
|
3328
|
+
private recordProviderTurnStart(
|
|
3329
|
+
utterance: UtteranceCycle,
|
|
3330
|
+
turnIndex: number | undefined,
|
|
3331
|
+
): void {
|
|
3332
|
+
if (turnIndex === undefined) {
|
|
3333
|
+
return;
|
|
3334
|
+
}
|
|
3335
|
+
utterance.openProviderTurnIndex = turnIndex;
|
|
3336
|
+
}
|
|
3337
|
+
|
|
3338
|
+
/**
|
|
3339
|
+
* A provider end-of-turn that lost its race with the caller's next breath:
|
|
3340
|
+
* it describes speech the caller has already spoken past. Acting on one
|
|
3341
|
+
* would send utterance_end and stop the transcriber while local VAD is
|
|
3342
|
+
* still routing the resumed speech into this same cycle, cutting off the
|
|
3343
|
+
* next words or folding them into the wrong turn.
|
|
3344
|
+
*
|
|
3345
|
+
* The provider's own turn numbering answers this directly and is preferred
|
|
3346
|
+
* wherever it is available: the cycle records the newest turn the provider
|
|
3347
|
+
* has opened, so an end-of-turn for an older index closes a turn the
|
|
3348
|
+
* provider itself has already superseded. An end-of-turn for the turn still
|
|
3349
|
+
* in progress is never stale, however many times the caller drew breath
|
|
3350
|
+
* inside it: the mid-thought pause is exactly the case a provider's turn
|
|
3351
|
+
* model exists to judge, and its verdict covers the resumed speech too.
|
|
3352
|
+
*
|
|
3353
|
+
* With no turn numbering the local speech generation is the only signal
|
|
3354
|
+
* left. The silence boundary stamps the cycle with the generation it
|
|
3355
|
+
* closed and `handleVadSpeechStart` bumps that generation when the caller
|
|
3356
|
+
* resumes, so a turn-end arriving on a newer generation is treated as
|
|
3357
|
+
* stale. That is conservative: it also drops the fast end-of-turn this
|
|
3358
|
+
* feature exists for, which the turn index would have committed. Before
|
|
3359
|
+
* any boundary has fired the stamp is null and nothing is stale.
|
|
3360
|
+
*/
|
|
3361
|
+
private isStaleProviderTurnEnd(
|
|
3362
|
+
utterance: UtteranceCycle,
|
|
3363
|
+
turnIndex: number | undefined,
|
|
3364
|
+
): boolean {
|
|
3365
|
+
if (turnIndex !== undefined && utterance.openProviderTurnIndex !== null) {
|
|
3366
|
+
return turnIndex < utterance.openProviderTurnIndex;
|
|
3367
|
+
}
|
|
3368
|
+
return (
|
|
3369
|
+
utterance.turnBoundaryGeneration !== null &&
|
|
3370
|
+
utterance.turnBoundaryGeneration !== this.vadSpeechGeneration
|
|
3371
|
+
);
|
|
3372
|
+
}
|
|
3373
|
+
|
|
3374
|
+
private clearProviderTurnEndTimer(): void {
|
|
3375
|
+
if (this.providerTurnEndTimer !== null) {
|
|
3376
|
+
clearTimeout(this.providerTurnEndTimer);
|
|
3377
|
+
this.providerTurnEndTimer = null;
|
|
3378
|
+
}
|
|
3379
|
+
}
|
|
3380
|
+
|
|
3381
|
+
// Time since the local VAD last heard above-gate audio: the speech-stop
|
|
3382
|
+
// mark both the fallback deadline and the reported end-of-turn latency are
|
|
3383
|
+
// measured from. Zero when no speech has been heard at all.
|
|
3384
|
+
private msSinceLocalSpeechStop(): number {
|
|
3385
|
+
if (this.localSpeechStopAtMs === null) {
|
|
3386
|
+
return 0;
|
|
3387
|
+
}
|
|
3388
|
+
return Math.max(0, Date.now() - this.localSpeechStopAtMs);
|
|
3389
|
+
}
|
|
3390
|
+
|
|
3391
|
+
/**
|
|
3392
|
+
* The provider committed an end of turn: the caller has finished, so the utterance
|
|
3393
|
+
* commits now instead of waiting out the trailing-silence timer and asking
|
|
3394
|
+
* the front door whether the pause was mid-thought. This runs the ordinary
|
|
3395
|
+
* post-release path (utterance_end, then release), so the front-door leg it
|
|
3396
|
+
* dispatches is non-speculative: `speculativeHoldAllowed` is false, its
|
|
3397
|
+
* decision rule is built with `includeHold: false`, and the model is told
|
|
3398
|
+
* the caller has finished. Escalation and progress narration are untouched;
|
|
3399
|
+
* only the hold verdict is bypassed.
|
|
3400
|
+
*/
|
|
3401
|
+
private async handleProviderTurnEnd(
|
|
3402
|
+
utterance: UtteranceCycle,
|
|
3403
|
+
turnIndex: number | undefined,
|
|
3404
|
+
): Promise<void> {
|
|
3405
|
+
if (
|
|
3406
|
+
!this.providerTurnEndActive ||
|
|
3407
|
+
this.currentUtterance !== utterance ||
|
|
3408
|
+
utterance.released ||
|
|
3409
|
+
utterance.completed ||
|
|
3410
|
+
utterance.assistantTurnStarted ||
|
|
3411
|
+
// The fallback already took this utterance back to the silence path.
|
|
3412
|
+
utterance.providerTurnEndTimedOut
|
|
3413
|
+
) {
|
|
3414
|
+
return;
|
|
3415
|
+
}
|
|
3416
|
+
if (this.isStaleProviderTurnEnd(utterance, turnIndex)) {
|
|
3417
|
+
// At `info`, not `debug`: a drop is rare and is the only signal of the
|
|
3418
|
+
// one known latency-outlier mode, so an operator measuring the two
|
|
3419
|
+
// paths has to be able to see it at the default log level.
|
|
3420
|
+
log.info(
|
|
3421
|
+
{
|
|
3422
|
+
turnIndex,
|
|
3423
|
+
openTurnIndex: utterance.openProviderTurnIndex,
|
|
3424
|
+
boundaryGeneration: utterance.turnBoundaryGeneration,
|
|
3425
|
+
speechGeneration: this.vadSpeechGeneration,
|
|
3426
|
+
},
|
|
3427
|
+
"Dropping a stale provider end-of-turn: the caller resumed speaking past the boundary it closed",
|
|
3428
|
+
);
|
|
3429
|
+
// The cycle stays open and local VAD keeps routing the resumed speech
|
|
3430
|
+
// into it. The detector's next silence boundary re-stamps the
|
|
3431
|
+
// generation and re-arms the fail-open deadline, so the turn still
|
|
3432
|
+
// commits: on the end-of-turn for the resumed speech, or on that
|
|
3433
|
+
// deadline. Re-arm here only if the detector has somehow already gone
|
|
3434
|
+
// idle, so the deadline handleVadSpeechStart cleared can never strand
|
|
3435
|
+
// the cycle with nothing left to close it.
|
|
3436
|
+
if (this.turnDetector?.isActive !== true) {
|
|
3437
|
+
utterance.turnBoundaryGeneration = this.vadSpeechGeneration;
|
|
3438
|
+
this.armProviderTurnEndFallbackTimer(utterance);
|
|
3439
|
+
}
|
|
3440
|
+
return;
|
|
3441
|
+
}
|
|
3442
|
+
this.clearProviderTurnEndTimer();
|
|
3443
|
+
this.markEndpointDecision(
|
|
3444
|
+
utterance,
|
|
3445
|
+
"release",
|
|
3446
|
+
this.msSinceLocalSpeechStop(),
|
|
3447
|
+
"provider",
|
|
3448
|
+
);
|
|
3449
|
+
await this.sendFrame({ type: "utterance_end", reason: "silence" });
|
|
3450
|
+
await this.releaseUtterance();
|
|
3451
|
+
// Leave the local detector idle, which is where every other commit path
|
|
3452
|
+
// leaves it. A provider can close a turn while the trailing-silence
|
|
3453
|
+
// countdown is still running, and barge-in fires from the detector's speech
|
|
3454
|
+
// ONSET: a detector left mid-turn reports no onset, so the caller could
|
|
3455
|
+
// not interrupt the reply they just triggered. The forced boundary
|
|
3456
|
+
// reaches an already-released utterance and returns.
|
|
3457
|
+
this.turnDetector?.forceEnd();
|
|
3458
|
+
}
|
|
3459
|
+
|
|
2722
3460
|
/**
|
|
2723
3461
|
* Unified front-door: dispatch the assistant turn speculatively at the
|
|
2724
3462
|
* silence boundary, judging the transcript accumulated so far — the same
|
|
@@ -2780,6 +3518,13 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2780
3518
|
// are skipped; the thinking frame and timers still apply.
|
|
2781
3519
|
const alreadyReleased = utterance.released;
|
|
2782
3520
|
turn.speculativePending = false;
|
|
3521
|
+
// Finals can land between the speculative dispatch and this verdict.
|
|
3522
|
+
// Fill the language only when dispatch had none: the model request was
|
|
3523
|
+
// already issued with the dispatch language, so overwriting here would
|
|
3524
|
+
// hint TTS (and any voice override) in a different language than the
|
|
3525
|
+
// text it speaks. The tally still carries the corrected detection into
|
|
3526
|
+
// the next turn.
|
|
3527
|
+
turn.language ??= this.turnLanguageFor(utterance);
|
|
2783
3528
|
if (turn.verdictDeadlineTimer !== null) {
|
|
2784
3529
|
clearTimeout(turn.verdictDeadlineTimer);
|
|
2785
3530
|
turn.verdictDeadlineTimer = null;
|
|
@@ -2791,13 +3536,10 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2791
3536
|
if (!alreadyReleased) {
|
|
2792
3537
|
void this.releaseUtterance();
|
|
2793
3538
|
}
|
|
2794
|
-
if (this.streamTtsAudio) {
|
|
2795
|
-
this.armAckTimer(turn);
|
|
2796
|
-
}
|
|
2797
3539
|
if (
|
|
2798
3540
|
this.frontModelConfig.progress.enabled &&
|
|
2799
3541
|
this.streamTtsAudio &&
|
|
2800
|
-
this.
|
|
3542
|
+
this.progressNarrator
|
|
2801
3543
|
) {
|
|
2802
3544
|
this.armProgressIdleTimer(turn);
|
|
2803
3545
|
}
|
|
@@ -2864,8 +3606,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2864
3606
|
): void {
|
|
2865
3607
|
turn.speculativePending = false;
|
|
2866
3608
|
// The dispatch is being unwound, so the barge-in merge note, the finished
|
|
2867
|
-
// continuation's answer,
|
|
2868
|
-
//
|
|
3609
|
+
// continuation's answer, its queued announcement, and any photos it claimed
|
|
3610
|
+
// all go back to the session. The turn that would have delivered them is
|
|
3611
|
+
// gone, and the utterance they belong to is about to be sent by another.
|
|
2869
3612
|
this.restorePendingTurnContext(turn);
|
|
2870
3613
|
// Latched before the handle check: when the discard beats the bridge
|
|
2871
3614
|
// handle's resolution (startVoiceTurn still persisting), the handle's
|
|
@@ -2930,6 +3673,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2930
3673
|
!utterance.completed
|
|
2931
3674
|
) {
|
|
2932
3675
|
this.clearEndpointExtensionTimer();
|
|
3676
|
+
this.clearProviderTurnEndTimer();
|
|
2933
3677
|
await this.sendFrame({ type: "utterance_end", reason: "silence" });
|
|
2934
3678
|
}
|
|
2935
3679
|
await this.releaseUtterance();
|
|
@@ -2944,6 +3688,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2944
3688
|
if (utterance.phase === "transcriber_closed") {
|
|
2945
3689
|
utterance.released = true;
|
|
2946
3690
|
this.markUtteranceReleased(utterance);
|
|
3691
|
+
this.markEndpointCommit(utterance);
|
|
2947
3692
|
await this.startAssistantTurnIfReady();
|
|
2948
3693
|
await this.drainOutboundFrames();
|
|
2949
3694
|
return;
|
|
@@ -2955,6 +3700,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2955
3700
|
|
|
2956
3701
|
utterance.released = true;
|
|
2957
3702
|
this.markUtteranceReleased(utterance);
|
|
3703
|
+
this.markEndpointCommit(utterance);
|
|
2958
3704
|
|
|
2959
3705
|
if (utterance.phase === "pending") {
|
|
2960
3706
|
// The transcriber is still starting; beginUtterance completes the release.
|
|
@@ -3117,16 +3863,39 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3117
3863
|
switch (event.type) {
|
|
3118
3864
|
case "partial":
|
|
3119
3865
|
utterance.latestPartialText = event.text;
|
|
3866
|
+
this.capturePartialLanguages(utterance, event.languages);
|
|
3120
3867
|
this.markFirstPartial(utterance);
|
|
3121
3868
|
await this.sendFrame({ type: "stt_partial", text: event.text });
|
|
3122
3869
|
return;
|
|
3123
3870
|
case "final":
|
|
3124
|
-
await this.recordFinalTranscript(
|
|
3871
|
+
await this.recordFinalTranscript(
|
|
3872
|
+
utterance,
|
|
3873
|
+
event.text,
|
|
3874
|
+
event.languages,
|
|
3875
|
+
);
|
|
3125
3876
|
return;
|
|
3126
3877
|
case "finalized":
|
|
3127
3878
|
// Per-cycle transcribers are torn down with stop(); the finalize
|
|
3128
3879
|
// completion signal has no cycle to advance here.
|
|
3129
3880
|
return;
|
|
3881
|
+
case "turn-start":
|
|
3882
|
+
// Barge-in is deliberately untouched: local VAD still owns it,
|
|
3883
|
+
// because a provider roundtrip cannot beat a local energy gate on an
|
|
3884
|
+
// interrupt during playback (see DEFAULT_BARGE_IN_MIN_SPEECH_MS). The
|
|
3885
|
+
// index is recorded so a later end-of-turn can be told apart from one
|
|
3886
|
+
// this turn superseded (see isStaleProviderTurnEnd).
|
|
3887
|
+
this.recordProviderTurnStart(utterance, event.turnIndex);
|
|
3888
|
+
return;
|
|
3889
|
+
case "eager-turn-end":
|
|
3890
|
+
case "turn-resumed":
|
|
3891
|
+
// Speculative end-of-turn stays off (the config leaves
|
|
3892
|
+
// `eagerEotThreshold` unset, so Deepgram never emits these). Wiring
|
|
3893
|
+
// them onto the speculative dispatch machinery is a deferred
|
|
3894
|
+
// follow-up.
|
|
3895
|
+
return;
|
|
3896
|
+
case "turn-end":
|
|
3897
|
+
await this.handleProviderTurnEnd(utterance, event.turnIndex);
|
|
3898
|
+
return;
|
|
3130
3899
|
case "error":
|
|
3131
3900
|
await this.sendTranscriberErrorFrame(event);
|
|
3132
3901
|
return;
|
|
@@ -3148,6 +3917,10 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3148
3917
|
}
|
|
3149
3918
|
await this.startAssistantTurnIfReady();
|
|
3150
3919
|
return;
|
|
3920
|
+
default: {
|
|
3921
|
+
const _exhaustive: never = event;
|
|
3922
|
+
return;
|
|
3923
|
+
}
|
|
3151
3924
|
}
|
|
3152
3925
|
}
|
|
3153
3926
|
|
|
@@ -3190,6 +3963,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3190
3963
|
return;
|
|
3191
3964
|
}
|
|
3192
3965
|
target.latestPartialText = event.text;
|
|
3966
|
+
this.capturePartialLanguages(target, event.languages);
|
|
3193
3967
|
this.markFirstPartial(target);
|
|
3194
3968
|
await this.sendFrame({ type: "stt_partial", text: event.text });
|
|
3195
3969
|
return;
|
|
@@ -3205,7 +3979,11 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3205
3979
|
// newer cycle.
|
|
3206
3980
|
const owner = this.finalizeQueue[0];
|
|
3207
3981
|
if (owner && !owner.assistantTurnStarted && !owner.completed) {
|
|
3208
|
-
await this.recordFinalTranscript(
|
|
3982
|
+
await this.recordFinalTranscript(
|
|
3983
|
+
owner,
|
|
3984
|
+
event.text,
|
|
3985
|
+
event.languages,
|
|
3986
|
+
);
|
|
3209
3987
|
} else {
|
|
3210
3988
|
log.warn(
|
|
3211
3989
|
"Dropping a late finalize flush segment: its assistant turn already dispatched",
|
|
@@ -3222,7 +4000,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3222
4000
|
);
|
|
3223
4001
|
return;
|
|
3224
4002
|
}
|
|
3225
|
-
await this.recordFinalTranscript(target, event.text);
|
|
4003
|
+
await this.recordFinalTranscript(target, event.text, event.languages);
|
|
3226
4004
|
return;
|
|
3227
4005
|
}
|
|
3228
4006
|
case "finalized": {
|
|
@@ -3246,6 +4024,34 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3246
4024
|
await this.startAssistantTurnIfReady();
|
|
3247
4025
|
return;
|
|
3248
4026
|
}
|
|
4027
|
+
case "turn-start": {
|
|
4028
|
+
// Barge-in is deliberately untouched: local VAD still owns it,
|
|
4029
|
+
// because a provider roundtrip cannot beat a local energy gate on an
|
|
4030
|
+
// interrupt during playback (see DEFAULT_BARGE_IN_MIN_SPEECH_MS). The
|
|
4031
|
+
// index is recorded so a later end-of-turn can be told apart from one
|
|
4032
|
+
// this turn superseded (see isStaleProviderTurnEnd).
|
|
4033
|
+
const target = this.pendingTranscriptCycle();
|
|
4034
|
+
if (target) {
|
|
4035
|
+
this.recordProviderTurnStart(target, event.turnIndex);
|
|
4036
|
+
}
|
|
4037
|
+
return;
|
|
4038
|
+
}
|
|
4039
|
+
case "eager-turn-end":
|
|
4040
|
+
case "turn-resumed":
|
|
4041
|
+
// Speculative end-of-turn stays off (the config leaves
|
|
4042
|
+
// `eagerEotThreshold` unset, so Deepgram never emits these). Wiring
|
|
4043
|
+
// them onto the speculative dispatch machinery is a deferred
|
|
4044
|
+
// follow-up.
|
|
4045
|
+
return;
|
|
4046
|
+
case "turn-end": {
|
|
4047
|
+
// Same routing as transcript events: the cycle awaiting its
|
|
4048
|
+
// transcript owns the turn the provider just closed.
|
|
4049
|
+
const target = this.pendingTranscriptCycle();
|
|
4050
|
+
if (target) {
|
|
4051
|
+
await this.handleProviderTurnEnd(target, event.turnIndex);
|
|
4052
|
+
}
|
|
4053
|
+
return;
|
|
4054
|
+
}
|
|
3249
4055
|
case "error":
|
|
3250
4056
|
await this.sendTranscriberErrorFrame(event);
|
|
3251
4057
|
return;
|
|
@@ -3285,16 +4091,26 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3285
4091
|
await this.startAssistantTurnIfReady();
|
|
3286
4092
|
return;
|
|
3287
4093
|
}
|
|
4094
|
+
default: {
|
|
4095
|
+
const _exhaustive: never = event;
|
|
4096
|
+
return;
|
|
4097
|
+
}
|
|
3288
4098
|
}
|
|
3289
4099
|
}
|
|
3290
4100
|
|
|
3291
4101
|
private async recordFinalTranscript(
|
|
3292
4102
|
utterance: UtteranceCycle,
|
|
3293
4103
|
text: string,
|
|
4104
|
+
languages?: readonly string[],
|
|
3294
4105
|
): Promise<void> {
|
|
3295
4106
|
const transcript = text.trim();
|
|
3296
4107
|
if (transcript.length > 0) {
|
|
3297
4108
|
utterance.finalTranscriptSegments.push(transcript);
|
|
4109
|
+
// Tally only finals that committed transcript: empty silence frames
|
|
4110
|
+
// can still carry container-level language tags describing no emitted
|
|
4111
|
+
// words, and counting those would let silence outvote real speech
|
|
4112
|
+
// (same choice as the adapter's boundary-final aggregation).
|
|
4113
|
+
voteDominantLanguage(utterance.languageTally, languages);
|
|
3298
4114
|
}
|
|
3299
4115
|
// The final commits (and supersedes) whatever partial was trailing it.
|
|
3300
4116
|
utterance.latestPartialText = null;
|
|
@@ -3333,6 +4149,59 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3333
4149
|
await this.startAssistantTurnIfReady();
|
|
3334
4150
|
}
|
|
3335
4151
|
|
|
4152
|
+
// Record a partial event's detected languages so speculative dispatch
|
|
4153
|
+
// has a detection before the first tagged final. The event contract
|
|
4154
|
+
// (stt/types.ts) guarantees the tags arrive as normalized base subtags
|
|
4155
|
+
// in dominance order, so they are stored as-is. Partials revise each
|
|
4156
|
+
// other, so this overwrites rather than tallies, and a tag-less partial
|
|
4157
|
+
// keeps the previous value.
|
|
4158
|
+
private capturePartialLanguages(
|
|
4159
|
+
utterance: UtteranceCycle,
|
|
4160
|
+
languages: readonly string[] | undefined,
|
|
4161
|
+
): void {
|
|
4162
|
+
if (!languages || languages.length === 0) {
|
|
4163
|
+
return;
|
|
4164
|
+
}
|
|
4165
|
+
utterance.latestPartialLanguages = languages;
|
|
4166
|
+
}
|
|
4167
|
+
|
|
4168
|
+
/**
|
|
4169
|
+
* The caller's spoken language for a turn on this utterance, as a
|
|
4170
|
+
* lowercase base subtag: the dominant tallied STT-detected language
|
|
4171
|
+
* (most final-event counts, ties by first appearance), else the latest
|
|
4172
|
+
* tagged partial's dominant language (speculative turns dispatch from
|
|
4173
|
+
* partials), else a monolingual `services.stt.language` pin (a pinned
|
|
4174
|
+
* language IS the spoken language), else undefined ("multi" with no tags,
|
|
4175
|
+
* non-tagging providers, silence).
|
|
4176
|
+
*/
|
|
4177
|
+
private turnLanguageFor(utterance: UtteranceCycle): string | undefined {
|
|
4178
|
+
const dominant = dominantLanguageTag(utterance.languageTally);
|
|
4179
|
+
if (dominant !== undefined) {
|
|
4180
|
+
return dominant;
|
|
4181
|
+
}
|
|
4182
|
+
// No tagged final yet (speculative turns dispatch from partials): the
|
|
4183
|
+
// latest tagged partial is the best detection available and outranks a
|
|
4184
|
+
// static pin for the same reason the tally does.
|
|
4185
|
+
const partialDominant = utterance.latestPartialLanguages?.[0];
|
|
4186
|
+
if (partialDominant !== undefined) {
|
|
4187
|
+
return partialDominant;
|
|
4188
|
+
}
|
|
4189
|
+
// A persisted pin only counts when the provider that actually
|
|
4190
|
+
// transcribed honors manual language selection (the shared
|
|
4191
|
+
// pinnedListeningLanguage gate). The DIALED transcriber's providerId
|
|
4192
|
+
// is authoritative, because the resolver silently falls back to
|
|
4193
|
+
// managed vellum (which honors the pin) when a BYOK provider has no
|
|
4194
|
+
// credential; the configured provider is only the last resort when no
|
|
4195
|
+
// transcriber reference survives.
|
|
4196
|
+
const { language: configured, provider: sttProvider } =
|
|
4197
|
+
getConfig().services.stt;
|
|
4198
|
+
const dialedProvider =
|
|
4199
|
+
utterance.dialedSttProvider ??
|
|
4200
|
+
this.sharedTranscriber?.providerId ??
|
|
4201
|
+
(sttProvider as SttProviderId);
|
|
4202
|
+
return pinnedListeningLanguage(dialedProvider, configured);
|
|
4203
|
+
}
|
|
4204
|
+
|
|
3336
4205
|
// Providers emit `error` mid-stream and may keep streaming; `closed` /
|
|
3337
4206
|
// `final` still drive turn lifecycle. Only transient categories are
|
|
3338
4207
|
// recoverable — auth/rate-limit/invalid-audio will not self-heal, so
|
|
@@ -3361,8 +4230,10 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3361
4230
|
this.assistantPlaybackTailUntilMs = 0;
|
|
3362
4231
|
this.takeVadPreRoll();
|
|
3363
4232
|
this.vadPendingTurnEnd = null;
|
|
3364
|
-
// ...and abandons any semantic-endpointing hold still awaiting replay
|
|
4233
|
+
// ...and abandons any semantic-endpointing hold still awaiting replay,
|
|
4234
|
+
// along with any provider end-of-turn the session was still waiting on.
|
|
3365
4235
|
this.clearEndpointExtensionTimer();
|
|
4236
|
+
this.clearProviderTurnEndTimer();
|
|
3366
4237
|
// A client interrupt is a hard reset: any barge-in merge context waiting for
|
|
3367
4238
|
// the next turn is now stale (the interrupted utterance may be discarded
|
|
3368
4239
|
// without ever reaching finalizePendingUtterance).
|
|
@@ -3588,6 +4459,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3588
4459
|
token,
|
|
3589
4460
|
turnId,
|
|
3590
4461
|
utterance,
|
|
4462
|
+
language: this.turnLanguageFor(utterance),
|
|
3591
4463
|
abortController,
|
|
3592
4464
|
handle: null,
|
|
3593
4465
|
launchedAtMs: Date.now(),
|
|
@@ -3628,12 +4500,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3628
4500
|
consumedAnnouncement: pending?.announcement ?? null,
|
|
3629
4501
|
pendingContextStopGeneration: this.detachStopGeneration,
|
|
3630
4502
|
continuationDelivery: opts?.continuationDelivery ?? null,
|
|
3631
|
-
toolUseStarted: false,
|
|
3632
|
-
firstDeltaSeen: false,
|
|
3633
4503
|
deltaEpoch: 0,
|
|
3634
|
-
ackSpoken: false,
|
|
3635
|
-
ackGenerationPending: false,
|
|
3636
|
-
ackTimer: null,
|
|
3637
4504
|
escalationHandedOff: false,
|
|
3638
4505
|
ttsBuffer: "",
|
|
3639
4506
|
ttsSegmentEnqueued: false,
|
|
@@ -3658,8 +4525,8 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3658
4525
|
// Verdict-deadline fail-open: the deferred-everything window is only
|
|
3659
4526
|
// safe while the verdict is fast. If the leg produces no verdict
|
|
3660
4527
|
// within the endpoint budget (provider TTFT tail), commit anyway so
|
|
3661
|
-
// the thinking frame shows and
|
|
3662
|
-
//
|
|
4528
|
+
// the thinking frame shows and progress narration can begin, bounding
|
|
4529
|
+
// the structural silence. A verdict that arrives
|
|
3663
4530
|
// after the commit still works: escalate hands off normally, and a
|
|
3664
4531
|
// late hold token is stripped like any stray token (the utterance is
|
|
3665
4532
|
// already released, so holding is no longer possible).
|
|
@@ -3682,22 +4549,14 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3682
4549
|
}, this.frontModelConfig.endpointDecisionTimeoutMs);
|
|
3683
4550
|
}
|
|
3684
4551
|
|
|
3685
|
-
// Slow-first-delta spoken ack: if the model produces no delta within the
|
|
3686
|
-
// budget, a short front-model-phrased sentence holds the floor. Only
|
|
3687
|
-
// meaningful on TTS sessions — a text-only session has no audio gap to
|
|
3688
|
-
// bridge.
|
|
3689
|
-
if (this.streamTtsAudio && !opts?.speculative) {
|
|
3690
|
-
this.armAckTimer(activeTurn);
|
|
3691
|
-
}
|
|
3692
|
-
|
|
3693
4552
|
// Progress narration speaks into the turn's audible dead air on a
|
|
3694
|
-
// cadence, wherever in the turn it occurs; without TTS or a
|
|
4553
|
+
// cadence, wherever in the turn it occurs; without TTS or a narrator there
|
|
3695
4554
|
// is nothing to speak (the idle trigger's static fallback still needs a
|
|
3696
4555
|
// generation attempt to fall back from).
|
|
3697
4556
|
if (
|
|
3698
4557
|
this.frontModelConfig.progress.enabled &&
|
|
3699
4558
|
this.streamTtsAudio &&
|
|
3700
|
-
this.
|
|
4559
|
+
this.progressNarrator &&
|
|
3701
4560
|
!opts?.speculative
|
|
3702
4561
|
) {
|
|
3703
4562
|
this.armProgressIdleTimer(activeTurn);
|
|
@@ -3770,7 +4629,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3770
4629
|
return;
|
|
3771
4630
|
}
|
|
3772
4631
|
this.markFirstAssistantDelta(utterance, turnId);
|
|
3773
|
-
this.
|
|
4632
|
+
this.markAssistantDelta(activeTurn);
|
|
3774
4633
|
// Send-time abort gate: a delta queued behind a backed-up outbound
|
|
3775
4634
|
// frame must not be written once barge-in aborts the turn, or the
|
|
3776
4635
|
// cancelled reply's text leaks ahead of turn_cancelled. Key off this
|
|
@@ -3807,8 +4666,20 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3807
4666
|
voiceSessionId: this.context.sessionId,
|
|
3808
4667
|
userMessageChannel: "vellum",
|
|
3809
4668
|
assistantMessageChannel: "vellum",
|
|
4669
|
+
// Fixed, and NOT the originating client: this pair resolves the turn's
|
|
4670
|
+
// channel capabilities, where `macos` is what grants a live-voice turn
|
|
4671
|
+
// desktop UI and dynamic surfaces. Reporting the true client here would
|
|
4672
|
+
// strip `supportsDynamicUi` from every iOS session: a behavior change
|
|
4673
|
+
// wearing an attribution fix's clothes. The originating client travels
|
|
4674
|
+
// as telemetry instead, on `voiceTelemetry` below.
|
|
3810
4675
|
userMessageInterface: "macos",
|
|
3811
4676
|
assistantMessageInterface: "macos",
|
|
4677
|
+
voiceTelemetry: {
|
|
4678
|
+
sessionId: this.context.sessionId,
|
|
4679
|
+
...(this.context.startFrame.client
|
|
4680
|
+
? { client: this.context.startFrame.client }
|
|
4681
|
+
: {}),
|
|
4682
|
+
},
|
|
3812
4683
|
voiceControlPrompt: buildVoiceControlPrompt(activeTurn, {
|
|
3813
4684
|
...(leg.frontDoor !== undefined ? { frontDoor: leg.frontDoor } : {}),
|
|
3814
4685
|
}),
|
|
@@ -3996,7 +4867,6 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3996
4867
|
if (current?.token !== token) {
|
|
3997
4868
|
return;
|
|
3998
4869
|
}
|
|
3999
|
-
current.toolUseStarted = true;
|
|
4000
4870
|
// Foreground wins the workspace: the continuation runs with full
|
|
4001
4871
|
// subagent abilities (it can write files, run commands), so the
|
|
4002
4872
|
// moment a live turn starts a consequential tool the two could
|
|
@@ -4045,21 +4915,6 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
4045
4915
|
current.progress.stateEpoch += 1;
|
|
4046
4916
|
this.refreshActivity(current);
|
|
4047
4917
|
log.debug({ turnId, toolName }, "Live voice turn started tool use");
|
|
4048
|
-
// Definitive tool use means the turn is guaranteed slow: speak
|
|
4049
|
-
// the floor-holding ack now instead of waiting out the
|
|
4050
|
-
// first-delta timer (same one-ack-per-turn `ackSpoken` budget).
|
|
4051
|
-
// Post-handoff the escalation bridge phrase holds the floor and
|
|
4052
|
-
// front-door deltas never set firstDeltaSeen, so the escalated
|
|
4053
|
-
// leg's tool starts must not stack another filler on the bridge.
|
|
4054
|
-
if (
|
|
4055
|
-
this.streamTtsAudio &&
|
|
4056
|
-
!current.firstDeltaSeen &&
|
|
4057
|
-
!current.ackSpoken &&
|
|
4058
|
-
!current.escalationHandedOff
|
|
4059
|
-
) {
|
|
4060
|
-
this.clearAckTimer(current);
|
|
4061
|
-
this.speakAck(current, "tool_use", toolName);
|
|
4062
|
-
}
|
|
4063
4918
|
this.maybeNarrateProgress(current, "ops");
|
|
4064
4919
|
},
|
|
4065
4920
|
tool_result: (event) => {
|
|
@@ -4213,9 +5068,8 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
4213
5068
|
return;
|
|
4214
5069
|
}
|
|
4215
5070
|
activeTurn.escalationHandedOff = true;
|
|
4216
|
-
// The escalation bridge below holds the floor, so
|
|
4217
|
-
//
|
|
4218
|
-
// filler on top of it.
|
|
5071
|
+
// The escalation bridge below holds the floor, so pending progress
|
|
5072
|
+
// narration would only stack a second filler on top of it.
|
|
4219
5073
|
this.clearFillerTimers(activeTurn);
|
|
4220
5074
|
// Abort the front-door leg so a model that keeps generating past the
|
|
4221
5075
|
// bridge cap adds no latency before the escalated leg starts.
|
|
@@ -4228,21 +5082,34 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
4228
5082
|
// deleted row for a bridge the model never produced).
|
|
4229
5083
|
const usesFallbackBridge = cappedBridge.length < MIN_SPOKEN_BRIDGE_CHARS;
|
|
4230
5084
|
const spokenBridge = usesFallbackBridge
|
|
4231
|
-
?
|
|
5085
|
+
? fallbackEscalationBridgeFor(activeTurn.language)
|
|
4232
5086
|
: cappedBridge;
|
|
4233
5087
|
if (!usesFallbackBridge) {
|
|
4234
5088
|
this.markFirstAssistantDelta(activeTurn.utterance, activeTurn.turnId);
|
|
4235
|
-
this.
|
|
5089
|
+
this.markAssistantDelta(activeTurn);
|
|
4236
5090
|
void this.sendFrame(
|
|
4237
5091
|
{ type: "assistant_text_delta", text: spokenBridge },
|
|
4238
5092
|
() => !activeTurn.abortController.signal.aborted && !this.isClosed,
|
|
4239
5093
|
);
|
|
5094
|
+
this.bufferAssistantTextForTts(activeTurn.token, `${spokenBridge} `);
|
|
5095
|
+
// Force-flush now: on the TTS path an unpunctuated bridge would
|
|
5096
|
+
// otherwise sit buffered until a sentence boundary and leave the
|
|
5097
|
+
// caller in silence during the escalated model's call.
|
|
5098
|
+
this.flushTtsBuffer(activeTurn.token, true);
|
|
5099
|
+
} else {
|
|
5100
|
+
// The canned bridge is a fixed localized-table phrase, enqueued
|
|
5101
|
+
// directly (it is already one complete sentence) so the segment can
|
|
5102
|
+
// carry the "en" override when the table lacks the turn's language.
|
|
5103
|
+
const speakable = sanitizeForTts(spokenBridge).trim();
|
|
5104
|
+
if (speakable.length > 0) {
|
|
5105
|
+
this.enqueueTtsSegment(activeTurn.token, speakable, {
|
|
5106
|
+
language: this.fixedPhraseLanguage(
|
|
5107
|
+
activeTurn,
|
|
5108
|
+
FALLBACK_ESCALATION_BRIDGE_BY_LANGUAGE,
|
|
5109
|
+
),
|
|
5110
|
+
});
|
|
5111
|
+
}
|
|
4240
5112
|
}
|
|
4241
|
-
this.bufferAssistantTextForTts(activeTurn.token, `${spokenBridge} `);
|
|
4242
|
-
// Force-flush now: on the TTS path an unpunctuated bridge would otherwise
|
|
4243
|
-
// sit buffered until a sentence boundary and leave the caller in silence
|
|
4244
|
-
// during the escalated model's call.
|
|
4245
|
-
this.flushTtsBuffer(activeTurn.token, true);
|
|
4246
5113
|
|
|
4247
5114
|
// No overrideProfile: the escalated leg runs on the call-site default —
|
|
4248
5115
|
// the exact profile an un-routed voice turn would use (see
|
|
@@ -4265,7 +5132,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
4265
5132
|
if (
|
|
4266
5133
|
this.frontModelConfig.progress.enabled &&
|
|
4267
5134
|
this.streamTtsAudio &&
|
|
4268
|
-
this.
|
|
5135
|
+
this.progressNarrator
|
|
4269
5136
|
) {
|
|
4270
5137
|
this.armProgressIdleTimer(activeTurn);
|
|
4271
5138
|
}
|
|
@@ -4337,42 +5204,11 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
4337
5204
|
);
|
|
4338
5205
|
}
|
|
4339
5206
|
|
|
4340
|
-
//
|
|
4341
|
-
//
|
|
4342
|
-
//
|
|
4343
|
-
private
|
|
4344
|
-
const { token } = activeTurn;
|
|
4345
|
-
activeTurn.ackTimer = setTimeout(() => {
|
|
4346
|
-
activeTurn.ackTimer = null;
|
|
4347
|
-
if (
|
|
4348
|
-
!this.isActiveAssistantTurn(token) ||
|
|
4349
|
-
activeTurn.firstDeltaSeen ||
|
|
4350
|
-
activeTurn.ackSpoken
|
|
4351
|
-
) {
|
|
4352
|
-
return;
|
|
4353
|
-
}
|
|
4354
|
-
this.speakAck(activeTurn, "first_delta");
|
|
4355
|
-
}, this.frontModelConfig.ackFirstDeltaTimeoutMs);
|
|
4356
|
-
}
|
|
4357
|
-
|
|
4358
|
-
private clearAckTimer(turn: ActiveAssistantTurn): void {
|
|
4359
|
-
if (turn.ackTimer !== null) {
|
|
4360
|
-
clearTimeout(turn.ackTimer);
|
|
4361
|
-
turn.ackTimer = null;
|
|
4362
|
-
}
|
|
4363
|
-
}
|
|
4364
|
-
|
|
4365
|
-
// The model produced real output for this turn: the pending slow-first-delta
|
|
4366
|
-
// ack is moot, and any in-flight narration generation is stale (it captures
|
|
4367
|
-
// deltaEpoch to notice). The narration idle timer stays armed — the dead air
|
|
4368
|
-
// narration exists to fill is almost always MID-turn (between tool phases,
|
|
4369
|
-
// during long silent LLM calls after the model's opening words), so it must
|
|
4370
|
-
// keep watching after the model speaks and re-trigger once the audio drains
|
|
4371
|
-
// and silence stretches again.
|
|
4372
|
-
private markFirstDeltaForAck(turn: ActiveAssistantTurn): void {
|
|
4373
|
-
turn.firstDeltaSeen = true;
|
|
5207
|
+
// A real assistant delta makes any in-flight progress narration stale. The
|
|
5208
|
+
// idle timer stays armed so it can detect a later stretch of mid-turn
|
|
5209
|
+
// silence.
|
|
5210
|
+
private markAssistantDelta(turn: ActiveAssistantTurn): void {
|
|
4374
5211
|
turn.deltaEpoch += 1;
|
|
4375
|
-
this.clearAckTimer(turn);
|
|
4376
5212
|
}
|
|
4377
5213
|
|
|
4378
5214
|
// Arms (or re-arms) the dead-air narration timer. The countdown measures
|
|
@@ -4448,14 +5284,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
4448
5284
|
}
|
|
4449
5285
|
}
|
|
4450
5286
|
|
|
4451
|
-
// Clears
|
|
4452
|
-
//
|
|
4453
|
-
// lifecycle: escalation hand-off (which then re-arms narration for the
|
|
4454
|
-
// escalated leg), barge-in, cancellation, tts-completion, finalize. Real
|
|
4455
|
-
// output moots only the ack (markFirstDeltaForAck) — the narration timer
|
|
4456
|
-
// keeps watching for mid-turn dead air.
|
|
5287
|
+
// Clears progress narration and verdict timers for events that end the
|
|
5288
|
+
// current filler lifecycle.
|
|
4457
5289
|
private clearFillerTimers(turn: ActiveAssistantTurn): void {
|
|
4458
|
-
this.clearAckTimer(turn);
|
|
4459
5290
|
this.clearProgressIdleTimer(turn);
|
|
4460
5291
|
if (turn.verdictDeadlineTimer !== null) {
|
|
4461
5292
|
clearTimeout(turn.verdictDeadlineTimer);
|
|
@@ -4463,26 +5294,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
4463
5294
|
}
|
|
4464
5295
|
}
|
|
4465
5296
|
|
|
4466
|
-
//
|
|
4467
|
-
// Once the turn is stale, the brain's first delta has arrived, an
|
|
4468
|
-
// escalation hand-off has enqueued its bridge phrase (which holds the floor
|
|
4469
|
-
// itself), or the turn has completed outright — a tool-only/no-text turn
|
|
4470
|
-
// completes with firstDeltaSeen still false — an ack is moot and must not
|
|
4471
|
-
// speak over real output, stack on the escalation bridge, or voice a
|
|
4472
|
-
// finished turn.
|
|
4473
|
-
private turnCanSpeakFiller(turn: ActiveAssistantTurn): boolean {
|
|
4474
|
-
return (
|
|
4475
|
-
this.isActiveAssistantTurn(turn.token) &&
|
|
4476
|
-
!turn.firstDeltaSeen &&
|
|
4477
|
-
!turn.escalationHandedOff &&
|
|
4478
|
-
!turn.assistantCompleted
|
|
4479
|
-
);
|
|
4480
|
-
}
|
|
4481
|
-
|
|
4482
|
-
// Narration, unlike the ack, is not confined to the pre-first-delta window:
|
|
4483
|
-
// the dead air it exists to fill is almost always mid-turn, after the
|
|
4484
|
-
// model's opening words. It may speak whenever the live turn is audibly
|
|
4485
|
-
// silent right now — nothing streaming, queued, or still playing — and has
|
|
5297
|
+
// Narration may speak whenever the live turn is audibly silent and has
|
|
4486
5298
|
// not completed. Escalated turns qualify too: the bridge phrase holds the
|
|
4487
5299
|
// floor only while its audio plays (covered by the audible-idle check),
|
|
4488
5300
|
// and the strong leg's tool loops are the longest dead air in the system.
|
|
@@ -4519,31 +5331,20 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
4519
5331
|
return false;
|
|
4520
5332
|
}
|
|
4521
5333
|
|
|
4522
|
-
// Gatekeeper for
|
|
4523
|
-
//
|
|
4524
|
-
// narration), one generation at a time, and — per trigger — only once the
|
|
4525
|
-
// ops trigger has `opsThreshold` ops accumulated or the idle trigger has
|
|
4526
|
-
// something new to report. No per-turn count cap: the cadence guards bound
|
|
4527
|
-
// the rate, and going quiet deep into a long turn is the failure mode
|
|
4528
|
-
// narration exists to prevent. Every failing guard short-circuits silently;
|
|
4529
|
-
// a skipped ops trigger keeps its accumulated count, so the next tool event
|
|
4530
|
-
// or idle tick retries.
|
|
5334
|
+
// Gatekeeper for progress narration. It speaks only while the turn is
|
|
5335
|
+
// audibly silent, with one generation at a time and the configured spacing.
|
|
4531
5336
|
private maybeNarrateProgress(
|
|
4532
5337
|
turn: ActiveAssistantTurn,
|
|
4533
5338
|
trigger: "ops" | "idle" | "op_complete",
|
|
4534
5339
|
): void {
|
|
4535
5340
|
const cfg = this.frontModelConfig.progress;
|
|
4536
5341
|
const { progress } = turn;
|
|
4537
|
-
const
|
|
5342
|
+
const progressNarrator = this.progressNarrator;
|
|
4538
5343
|
if (
|
|
4539
5344
|
!cfg.enabled ||
|
|
4540
5345
|
!this.streamTtsAudio ||
|
|
4541
|
-
!
|
|
5346
|
+
!progressNarrator ||
|
|
4542
5347
|
!this.turnCanNarrateProgress(turn) ||
|
|
4543
|
-
// A pending ack generation is a floor-holder-in-waiting: starting a
|
|
4544
|
-
// narration generation now would only be discarded by the post-await
|
|
4545
|
-
// re-check once the ack enqueues — a guaranteed wasted provider call.
|
|
4546
|
-
turn.ackGenerationPending ||
|
|
4547
5348
|
progress.narrationInFlight ||
|
|
4548
5349
|
(progress.lastFloorHolderAtMs !== null &&
|
|
4549
5350
|
Date.now() - progress.lastFloorHolderAtMs < cfg.minGapMs) ||
|
|
@@ -4552,25 +5353,20 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
4552
5353
|
) {
|
|
4553
5354
|
return;
|
|
4554
5355
|
}
|
|
4555
|
-
void this.speakProgressUpdate(turn,
|
|
5356
|
+
void this.speakProgressUpdate(turn, progressNarrator, trigger);
|
|
4556
5357
|
}
|
|
4557
5358
|
|
|
4558
|
-
// Generate and speak one progress narration.
|
|
4559
|
-
//
|
|
4560
|
-
//
|
|
4561
|
-
// by `progress.generationTimeoutMs` and resolves null on every failure
|
|
4562
|
-
// mode; on null the idle trigger falls back to a static phrase (silence is
|
|
4563
|
-
// actively harmful there) while the tool-activity triggers stay silent — a
|
|
4564
|
-
// generic filler is not worth it when narration was merely opportunistic.
|
|
5359
|
+
// Generate and speak one audio-only progress narration. On a null result,
|
|
5360
|
+
// idle narration falls back to a static phrase while activity-triggered
|
|
5361
|
+
// narration stays silent.
|
|
4565
5362
|
private async speakProgressUpdate(
|
|
4566
5363
|
turn: ActiveAssistantTurn,
|
|
4567
|
-
|
|
5364
|
+
progressNarrator: VoiceProgressNarrator,
|
|
4568
5365
|
trigger: "ops" | "idle" | "op_complete",
|
|
4569
5366
|
): Promise<void> {
|
|
4570
5367
|
const { progress } = turn;
|
|
4571
5368
|
progress.narrationInFlight = true;
|
|
4572
|
-
// Any delta that lands while
|
|
4573
|
-
// generated text stale — and proves the model is speaking again.
|
|
5369
|
+
// Any delta that lands while generation is in flight makes the text stale.
|
|
4574
5370
|
const deltaEpochAtLaunch = turn.deltaEpoch;
|
|
4575
5371
|
// The activity this update describes. Tool events that land mid-generation
|
|
4576
5372
|
// are news the generated text cannot carry, so they must leave the idle
|
|
@@ -4588,7 +5384,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
4588
5384
|
(op): op is TurnProgressOp & { completedAtMs: number } =>
|
|
4589
5385
|
op.completedAtMs !== undefined,
|
|
4590
5386
|
)
|
|
4591
|
-
// `ops` is in start order; the
|
|
5387
|
+
// `ops` is in start order; the narrator wants completion
|
|
4592
5388
|
// order, which differs when parallel tools finish out of order.
|
|
4593
5389
|
.sort((a, b) => a.completedAtMs - b.completedAtMs)
|
|
4594
5390
|
.map((op) => ({
|
|
@@ -4606,27 +5402,16 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
4606
5402
|
: null,
|
|
4607
5403
|
turnElapsedMs: now - turn.launchedAtMs,
|
|
4608
5404
|
updateIndex: progress.updatesSpoken + 1,
|
|
5405
|
+
...(turn.language !== undefined ? { languageHint: turn.language } : {}),
|
|
4609
5406
|
};
|
|
4610
|
-
const generated = await
|
|
5407
|
+
const generated = await progressNarrator
|
|
4611
5408
|
.generateProgressText(input, turn.abortController.signal)
|
|
4612
|
-
// The
|
|
5409
|
+
// The narrator contract never rejects. Keep test stubs fail-soft too.
|
|
4613
5410
|
.catch(() => null);
|
|
4614
|
-
//
|
|
4615
|
-
// condition may have flipped while generating, and a delta that landed
|
|
4616
|
-
// mid-generation (deltaEpoch moved) makes the text stale even when the
|
|
4617
|
-
// resulting audio has already drained by now. The ack-generation and
|
|
4618
|
-
// floor re-checks close the race with a generated ack that
|
|
4619
|
-
// STARTED after this generation did — the entry guard rejects a
|
|
4620
|
-
// narration while an ack generation is already pending, but cannot see
|
|
4621
|
-
// one that begins mid-generation: while that ack is pending it has not
|
|
4622
|
-
// yet stamped `lastFloorHolderAtMs`, and once it enqueues the minGapMs
|
|
4623
|
-
// spacing must be re-applied from that stamp — either way, enqueueing
|
|
4624
|
-
// now would stack back-to-back filler. A bail here is silent and keeps
|
|
4625
|
-
// the update budget, exactly like the stale-turn bail.
|
|
5411
|
+
// Re-check liveness and staleness after the provider call.
|
|
4626
5412
|
if (
|
|
4627
5413
|
!this.turnCanNarrateProgress(turn) ||
|
|
4628
5414
|
turn.deltaEpoch !== deltaEpochAtLaunch ||
|
|
4629
|
-
turn.ackGenerationPending ||
|
|
4630
5415
|
(progress.lastFloorHolderAtMs !== null &&
|
|
4631
5416
|
Date.now() - progress.lastFloorHolderAtMs <
|
|
4632
5417
|
this.frontModelConfig.progress.minGapMs)
|
|
@@ -4634,20 +5419,26 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
4634
5419
|
return;
|
|
4635
5420
|
}
|
|
4636
5421
|
let raw = generated;
|
|
5422
|
+
// Generated text is in the turn's language; only the static
|
|
5423
|
+
// fallback comes from a localized table and may need the "en" override.
|
|
5424
|
+
let fillerLanguage: string | undefined;
|
|
4637
5425
|
if (raw === null) {
|
|
4638
5426
|
if (trigger !== "idle") {
|
|
4639
5427
|
return;
|
|
4640
5428
|
}
|
|
4641
|
-
raw = pickProgressPhrase(this.progressPhraseCounter
|
|
5429
|
+
raw = pickProgressPhrase(this.progressPhraseCounter++, turn.language);
|
|
5430
|
+
fillerLanguage = this.fixedPhraseLanguage(
|
|
5431
|
+
turn,
|
|
5432
|
+
PROGRESS_FALLBACK_PHRASES_BY_LANGUAGE,
|
|
5433
|
+
);
|
|
4642
5434
|
}
|
|
4643
|
-
if (!this.enqueueFillerPhrase(turn, raw)) {
|
|
5435
|
+
if (!this.enqueueFillerPhrase(turn, raw, fillerLanguage)) {
|
|
4644
5436
|
return;
|
|
4645
5437
|
}
|
|
4646
5438
|
progress.opsSinceNarration = 0;
|
|
4647
5439
|
progress.narratedEpoch = stateEpochAtLaunch;
|
|
4648
5440
|
progress.updatesSpoken += 1;
|
|
4649
|
-
//
|
|
4650
|
-
// enqueued (decider text or static fallback alike).
|
|
5441
|
+
// Record only when narration audio actually enqueued.
|
|
4651
5442
|
this.markProgressSpoken(turn);
|
|
4652
5443
|
// Restart the dead-air countdown from this narration.
|
|
4653
5444
|
this.armProgressIdleTimer(turn);
|
|
@@ -4656,101 +5447,40 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
4656
5447
|
}
|
|
4657
5448
|
}
|
|
4658
5449
|
|
|
4659
|
-
//
|
|
4660
|
-
|
|
4661
|
-
|
|
4662
|
-
// own output. It enqueues as a complete standalone sentence on the turn's
|
|
4663
|
-
// ordered TTS queue, so the model's audio always follows it cleanly; it
|
|
4664
|
-
// never consumes the eager first-clause flush reserved for the model's real
|
|
4665
|
-
// first segment.
|
|
4666
|
-
private speakAck(
|
|
4667
|
-
activeTurn: ActiveAssistantTurn,
|
|
4668
|
-
kind: LiveVoiceSpokenAckKind,
|
|
4669
|
-
toolName?: string,
|
|
4670
|
-
): void {
|
|
4671
|
-
if (!this.frontDecider) {
|
|
4672
|
-
// No decider, no ack: a canned line is worse than the silence, which
|
|
4673
|
-
// progress narration already covers once the turn runs long.
|
|
4674
|
-
return;
|
|
4675
|
-
}
|
|
4676
|
-
activeTurn.ackSpoken = true;
|
|
4677
|
-
// Optimistic ackSpoken above keeps the one-per-turn budget honest for
|
|
4678
|
-
// any other ack trigger while the generation is in flight; the async
|
|
4679
|
-
// path undoes it if the ack turns out moot.
|
|
4680
|
-
void this.speakGeneratedAck(activeTurn, this.frontDecider, kind, toolName);
|
|
4681
|
-
}
|
|
4682
|
-
|
|
4683
|
-
// Ask the front model for one short contextual sentence. The decider
|
|
4684
|
-
// internally bounds the call by `ackGenerationTimeoutMs` and resolves null
|
|
4685
|
-
// on every failure mode, so the enqueue never waits beyond that budget; a
|
|
4686
|
-
// null generation speaks nothing.
|
|
4687
|
-
private async speakGeneratedAck(
|
|
4688
|
-
activeTurn: ActiveAssistantTurn,
|
|
4689
|
-
frontDecider: VoiceFrontDecider,
|
|
4690
|
-
kind: LiveVoiceSpokenAckKind,
|
|
4691
|
-
toolName?: string,
|
|
4692
|
-
): Promise<void> {
|
|
4693
|
-
activeTurn.ackGenerationPending = true;
|
|
4694
|
-
try {
|
|
4695
|
-
const transcript = activeTurn.utterance.finalTranscriptSegments
|
|
4696
|
-
.join(" ")
|
|
4697
|
-
.trim();
|
|
4698
|
-
const generated = await frontDecider
|
|
4699
|
-
.generateAckText(
|
|
4700
|
-
{
|
|
4701
|
-
transcriptSoFar: transcript,
|
|
4702
|
-
toolName,
|
|
4703
|
-
},
|
|
4704
|
-
activeTurn.abortController.signal,
|
|
4705
|
-
)
|
|
4706
|
-
// The decider contract never rejects; belt-and-braces for a stub.
|
|
4707
|
-
.catch(() => null);
|
|
4708
|
-
// Liveness re-check after the await: any turnCanSpeakFiller condition
|
|
4709
|
-
// may have flipped while generating, mooting the ack. A null generation
|
|
4710
|
-
// (no provider, timeout, unusable output) speaks nothing. Either way,
|
|
4711
|
-
// undo the optimistic ackSpoken so the flag reflects reality and a
|
|
4712
|
-
// later trigger this turn can still hold the floor.
|
|
4713
|
-
if (generated === null || !this.turnCanSpeakFiller(activeTurn)) {
|
|
4714
|
-
activeTurn.ackSpoken = false;
|
|
4715
|
-
return;
|
|
4716
|
-
}
|
|
4717
|
-
this.enqueueAckPhrase(activeTurn, generated, kind);
|
|
4718
|
-
} finally {
|
|
4719
|
-
activeTurn.ackGenerationPending = false;
|
|
4720
|
-
}
|
|
4721
|
-
}
|
|
4722
|
-
|
|
4723
|
-
// Records the ack-spoken metric only when a phrase actually enqueues; a
|
|
4724
|
-
// phrase that sanitizes away releases the turn's ack budget again.
|
|
4725
|
-
private enqueueAckPhrase(
|
|
4726
|
-
activeTurn: ActiveAssistantTurn,
|
|
5450
|
+
// Sanitize and enqueue one progress sentence on the ordered TTS queue.
|
|
5451
|
+
private enqueueFillerPhrase(
|
|
5452
|
+
turn: ActiveAssistantTurn,
|
|
4727
5453
|
raw: string,
|
|
4728
|
-
|
|
4729
|
-
):
|
|
4730
|
-
if (this.enqueueFillerPhrase(activeTurn, raw)) {
|
|
4731
|
-
this.markAckSpoken(activeTurn, kind);
|
|
4732
|
-
return;
|
|
4733
|
-
}
|
|
4734
|
-
activeTurn.ackSpoken = false;
|
|
4735
|
-
}
|
|
4736
|
-
|
|
4737
|
-
// Sanitize and enqueue one filler sentence (spoken ack or progress
|
|
4738
|
-
// narration) on the turn's ordered TTS queue — the shared tail of every
|
|
4739
|
-
// filler path. Returns whether a phrase actually enqueued; per-kind metric
|
|
4740
|
-
// marks and bookkeeping are the caller's.
|
|
4741
|
-
private enqueueFillerPhrase(turn: ActiveAssistantTurn, raw: string): boolean {
|
|
5454
|
+
language?: string,
|
|
5455
|
+
): boolean {
|
|
4742
5456
|
const phrase = sanitizeForTts(raw).trim();
|
|
4743
5457
|
if (phrase.length === 0) {
|
|
4744
5458
|
return false;
|
|
4745
5459
|
}
|
|
4746
5460
|
this.enqueueTtsSegment(turn.token, phrase, {
|
|
4747
5461
|
countsAsFirstSegment: false,
|
|
5462
|
+
...(language !== undefined ? { language } : {}),
|
|
4748
5463
|
});
|
|
4749
5464
|
// A spoken filler holds the floor, so narration's minGapMs spaces from it.
|
|
4750
5465
|
turn.progress.lastFloorHolderAtMs = Date.now();
|
|
4751
5466
|
return true;
|
|
4752
5467
|
}
|
|
4753
5468
|
|
|
5469
|
+
// The TTS hint override for a fixed phrase picked from a localized table:
|
|
5470
|
+
// "en" when the turn has a language the table does not cover (the picker
|
|
5471
|
+
// fell back to English text, which must not be synthesized under an
|
|
5472
|
+
// ar/ko/ta hint), undefined otherwise (the segment rides the turn's
|
|
5473
|
+
// language, or no hint at all when the language is unknown).
|
|
5474
|
+
private fixedPhraseLanguage(
|
|
5475
|
+
turn: ActiveAssistantTurn,
|
|
5476
|
+
table: Readonly<Record<string, unknown>>,
|
|
5477
|
+
): string | undefined {
|
|
5478
|
+
return turn.language !== undefined &&
|
|
5479
|
+
!hasLocalizedEntry(table, turn.language)
|
|
5480
|
+
? "en"
|
|
5481
|
+
: undefined;
|
|
5482
|
+
}
|
|
5483
|
+
|
|
4754
5484
|
private bufferAssistantTextForTts(token: symbol, text: string): void {
|
|
4755
5485
|
if (!this.streamTtsAudio || text.length === 0) {
|
|
4756
5486
|
return;
|
|
@@ -4871,20 +5601,21 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
4871
5601
|
private enqueueTtsSegment(
|
|
4872
5602
|
token: symbol,
|
|
4873
5603
|
segment: string,
|
|
4874
|
-
options: { countsAsFirstSegment?: boolean } = {},
|
|
5604
|
+
options: { countsAsFirstSegment?: boolean; language?: string } = {},
|
|
4875
5605
|
): void {
|
|
4876
5606
|
const activeTurn = this.activeAssistantTurn;
|
|
4877
5607
|
if (activeTurn?.token !== token || !this.streamTtsAudio) {
|
|
4878
5608
|
return;
|
|
4879
5609
|
}
|
|
4880
5610
|
|
|
4881
|
-
//
|
|
4882
|
-
//
|
|
5611
|
+
// Progress narration leaves the eager first-clause flush available for
|
|
5612
|
+
// the model's real first segment.
|
|
4883
5613
|
if (options.countsAsFirstSegment ?? true) {
|
|
4884
5614
|
activeTurn.ttsSegmentEnqueued = true;
|
|
4885
5615
|
}
|
|
4886
5616
|
const job: TtsSegmentJob = {
|
|
4887
5617
|
text: segment,
|
|
5618
|
+
language: options.language,
|
|
4888
5619
|
started: false,
|
|
4889
5620
|
settled: false,
|
|
4890
5621
|
emitting: false,
|
|
@@ -4924,10 +5655,14 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
4924
5655
|
return;
|
|
4925
5656
|
}
|
|
4926
5657
|
job.started = true;
|
|
5658
|
+
// The segment's own language override (fixed English fallback text)
|
|
5659
|
+
// wins over the turn's language.
|
|
5660
|
+
const language = job.language ?? activeTurn.language;
|
|
4927
5661
|
let synthesis: Promise<void>;
|
|
4928
5662
|
try {
|
|
4929
5663
|
synthesis = streamTtsAudio({
|
|
4930
5664
|
text: job.text,
|
|
5665
|
+
...(language !== undefined ? { language } : {}),
|
|
4931
5666
|
signal: activeTurn.abortController.signal,
|
|
4932
5667
|
outputFormat: "pcm",
|
|
4933
5668
|
sampleRate: this.context.startFrame.audio.sampleRate,
|
|
@@ -5067,6 +5802,10 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5067
5802
|
chunk.sampleRate,
|
|
5068
5803
|
);
|
|
5069
5804
|
const now = Date.now();
|
|
5805
|
+
if (!this.isAssistantPlaybackEchoPossible()) {
|
|
5806
|
+
this.resetEchoReference();
|
|
5807
|
+
}
|
|
5808
|
+
this.appendEchoReference(chunk);
|
|
5070
5809
|
this.assistantPlaybackTailUntilMs =
|
|
5071
5810
|
Math.max(now, this.assistantPlaybackTailUntilMs) + chunkMs;
|
|
5072
5811
|
const turnAfterSend = this.activeAssistantTurn;
|
|
@@ -5173,23 +5912,42 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5173
5912
|
utterance: UtteranceCycle,
|
|
5174
5913
|
action: VoiceEndpointAction,
|
|
5175
5914
|
latencyMs: number,
|
|
5915
|
+
// Absent means the front door decided, which is what the collector
|
|
5916
|
+
// defaults to; the provider path names itself so one metrics frame can
|
|
5917
|
+
// compare the two.
|
|
5918
|
+
source?: VoiceEndpointSource,
|
|
5176
5919
|
): void {
|
|
5177
5920
|
const turnId = this.ensureTurnId(utterance);
|
|
5178
5921
|
if (!this.startMetricsTurnIfNeeded(utterance, turnId)) {
|
|
5179
5922
|
return;
|
|
5180
5923
|
}
|
|
5181
|
-
this.metrics.markEndpointDecision(turnId, {
|
|
5924
|
+
this.metrics.markEndpointDecision(turnId, {
|
|
5925
|
+
action,
|
|
5926
|
+
latencyMs,
|
|
5927
|
+
...(source ? { source } : {}),
|
|
5928
|
+
});
|
|
5182
5929
|
}
|
|
5183
5930
|
|
|
5184
|
-
|
|
5185
|
-
|
|
5186
|
-
|
|
5187
|
-
|
|
5188
|
-
|
|
5189
|
-
|
|
5931
|
+
/**
|
|
5932
|
+
* The turn's end-of-turn latency, stamped at the moment it commits. Called
|
|
5933
|
+
* from `releaseUtterance`, which every committed turn passes through
|
|
5934
|
+
* whichever decider released it, so the front-door and provider samples span
|
|
5935
|
+
* the same thing from the same anchor over the same population. That is
|
|
5936
|
+
* what `endpointDecisionMaxLatencyMs` cannot offer, and why this exists
|
|
5937
|
+
* alongside it.
|
|
5938
|
+
*
|
|
5939
|
+
* Skipped with no speech-stop mark, which is push-to-talk: there the
|
|
5940
|
+
* client's release is the boundary and no local VAD runs.
|
|
5941
|
+
*/
|
|
5942
|
+
private markEndpointCommit(utterance: UtteranceCycle): void {
|
|
5943
|
+
if (this.localSpeechStopAtMs === null) {
|
|
5944
|
+
return;
|
|
5945
|
+
}
|
|
5946
|
+
const turnId = this.ensureTurnId(utterance);
|
|
5947
|
+
if (!this.startMetricsTurnIfNeeded(utterance, turnId)) {
|
|
5190
5948
|
return;
|
|
5191
5949
|
}
|
|
5192
|
-
this.metrics.
|
|
5950
|
+
this.metrics.markEndpointCommit(turnId, this.msSinceLocalSpeechStop());
|
|
5193
5951
|
}
|
|
5194
5952
|
|
|
5195
5953
|
private markProgressSpoken(activeTurn: ActiveAssistantTurn): void {
|
|
@@ -5506,6 +6264,13 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5506
6264
|
// method, and a per-call-site hook would drift from it on the first frame
|
|
5507
6265
|
// anyone added. Fire-and-forget by contract — see the reporter.
|
|
5508
6266
|
this.liveActivityReporter.report(frame);
|
|
6267
|
+
// Latched here for the same reason: both fatal paths mark the session
|
|
6268
|
+
// `failed` and then send their error frame through this method, so the
|
|
6269
|
+
// code that ended the session is readable at close without either path
|
|
6270
|
+
// having to remember to stash it.
|
|
6271
|
+
if (frame.type === "error" && this.state === "failed") {
|
|
6272
|
+
this.failureCode ??= frame.code;
|
|
6273
|
+
}
|
|
5509
6274
|
let sent = false;
|
|
5510
6275
|
this.outboundFrames = this.outboundFrames
|
|
5511
6276
|
.catch(() => {})
|
|
@@ -5538,14 +6303,16 @@ export function createLiveVoiceSession(
|
|
|
5538
6303
|
options: LiveVoiceSessionOptions = {},
|
|
5539
6304
|
): LiveVoiceSession {
|
|
5540
6305
|
// Workspace-tunable server-VAD thresholds. The `liveVoice.vad` schema
|
|
5541
|
-
// defaults
|
|
5542
|
-
//
|
|
5543
|
-
//
|
|
5544
|
-
//
|
|
5545
|
-
//
|
|
6306
|
+
// defaults are 800 energy / 1200 ms silence / 30 s max turn / 250 ms
|
|
6307
|
+
// barge-in guard. Three of the four match their in-code fallback; the
|
|
6308
|
+
// trailing-silence threshold does not, so a session built with a
|
|
6309
|
+
// `liveVoice` config waits 1200 ms and one built without waits the in-code
|
|
6310
|
+
// DEFAULT_SILENCE_THRESHOLD_MS of 800. Optional-chained because hand-built
|
|
6311
|
+
// test configs may predate the liveVoice namespace; absent config falls
|
|
6312
|
+
// through to the in-code defaults.
|
|
5546
6313
|
const liveVoiceConfig = getConfig().liveVoice;
|
|
5547
6314
|
const vadConfig = liveVoiceConfig?.vad;
|
|
5548
|
-
// Parsed once here into a complete config, shared by
|
|
6315
|
+
// Parsed once here into a complete config, shared by endpointing and the
|
|
5549
6316
|
// session (the constructor's own parse is then a no-op re-validation).
|
|
5550
6317
|
const frontModelConfig = LiveVoiceFrontModelConfigSchema.parse(
|
|
5551
6318
|
options.frontModelConfig ?? liveVoiceConfig?.frontModel ?? {},
|
|
@@ -5564,14 +6331,20 @@ export function createLiveVoiceSession(
|
|
|
5564
6331
|
options.speechEnergyThreshold ?? vadConfig?.speechEnergyThreshold,
|
|
5565
6332
|
bargeInMinSpeechMs:
|
|
5566
6333
|
options.bargeInMinSpeechMs ?? vadConfig?.bargeInMinSpeechMs,
|
|
6334
|
+
echoBargeInMargin:
|
|
6335
|
+
options.echoBargeInMargin ?? vadConfig?.echoBargeInMargin,
|
|
6336
|
+
echoEmaHalfLifeMs:
|
|
6337
|
+
options.echoEmaHalfLifeMs ?? vadConfig?.echoEmaHalfLifeMs,
|
|
6338
|
+
echoDrainSlackMs: options.echoDrainSlackMs ?? vadConfig?.echoDrainSlackMs,
|
|
5567
6339
|
frontModelConfig,
|
|
5568
|
-
//
|
|
5569
|
-
//
|
|
5570
|
-
|
|
5571
|
-
|
|
5572
|
-
|
|
5573
|
-
|
|
5574
|
-
|
|
6340
|
+
// Absent config leaves the schema defaults, which keep provider turn
|
|
6341
|
+
// detection off.
|
|
6342
|
+
fluxConfig: options.fluxConfig ?? liveVoiceConfig?.flux,
|
|
6343
|
+
// An explicit option, including null, always wins for the test seam.
|
|
6344
|
+
progressNarrator:
|
|
6345
|
+
options.progressNarrator !== undefined
|
|
6346
|
+
? options.progressNarrator
|
|
6347
|
+
: createVoiceProgressNarrator({ config: frontModelConfig.progress }),
|
|
5575
6348
|
resolveCredentialReadiness:
|
|
5576
6349
|
options.resolveCredentialReadiness === undefined
|
|
5577
6350
|
? defaultResolveLiveVoiceCredentialReadiness
|
|
@@ -5675,6 +6448,12 @@ export async function defaultSpawnBackgroundContinuation(args: {
|
|
|
5675
6448
|
objective: args.objective,
|
|
5676
6449
|
fork: true,
|
|
5677
6450
|
sendResultToUser: false,
|
|
6451
|
+
// Distinct spawn mode so this unattended continuation is separable in
|
|
6452
|
+
// cost telemetry from a tool-initiated fork. Mechanically both are
|
|
6453
|
+
// forks, but they come from different call sites with different cost
|
|
6454
|
+
// profiles, and lumping them together is what made delegated spend
|
|
6455
|
+
// opaque in the first place.
|
|
6456
|
+
spawnMode: "voice_continuation",
|
|
5678
6457
|
// Full subagent abilities: the continuation runs like any other
|
|
5679
6458
|
// background subagent, so it can genuinely finish build-shaped work
|
|
5680
6459
|
// (JARVIS-1354). Side effects are governed by the standard
|
|
@@ -5735,7 +6514,14 @@ async function defaultStartVoiceTurn(
|
|
|
5735
6514
|
// inside `startVoiceTurn` trips `FOREIGN KEY constraint failed`. Ensure it
|
|
5736
6515
|
// exists (idempotent) before persisting. Lives in the production wiring, not
|
|
5737
6516
|
// the session state machine, so session unit tests stay DB-free.
|
|
5738
|
-
|
|
6517
|
+
// Native: this is the local live-voice session, which adopts a conversation
|
|
6518
|
+
// id the app supplied. Its trust context resolves through
|
|
6519
|
+
// `resolveLocalLiveVoiceTrustContext`, and a phone call reaches the assistant
|
|
6520
|
+
// through the telephony path rather than here.
|
|
6521
|
+
const createdConversation = ensureConversationExists(
|
|
6522
|
+
options.conversationId,
|
|
6523
|
+
"vellum",
|
|
6524
|
+
);
|
|
5739
6525
|
if (createdConversation) {
|
|
5740
6526
|
// The row was created outside the normal send-message route, which is where
|
|
5741
6527
|
// sibling clients/sidebars learn about a new conversation. Emit the same
|