@vellumai/assistant 0.12.1 → 0.12.2-dev.202609162214.308bda0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +2 -0
- package/AGENTS.md +5 -1
- package/ARCHITECTURE.md +20 -21
- package/Dockerfile +7 -7
- package/docs/architecture/integrations.md +7 -0
- package/docs/architecture/memory.md +11 -2
- package/docs/architecture/turn-actor.md +15 -8
- package/docs/credential-execution-service.md +1 -1
- package/docs/desktop-browser-cli.md +35 -0
- package/docs/guardian-request-flow.md +16 -0
- package/node_modules/@vellumai/environments/src/shell.test.ts +21 -0
- package/node_modules/@vellumai/environments/src/shell.ts +24 -0
- package/node_modules/@vellumai/gateway-client/src/inbound-contract.ts +8 -2
- package/node_modules/@vellumai/slack-text/src/index.ts +13 -8
- package/node_modules/@vellumai/slack-text/src/label-resolution-entities.test.ts +95 -0
- package/openapi.yaml +564 -111
- package/package.json +7 -4
- package/scripts/bundled-plugin-packages.ts +154 -0
- package/scripts/generate-bundled-plugin-packages.ts +29 -0
- package/scripts/postinstall.ts +33 -0
- package/scripts/smoke-desktop-browser-cli.ts +280 -0
- package/scripts/smoke-desktop-window-theme.ts +186 -0
- package/scripts/sync-llm-catalog.ts +3 -0
- package/scripts/test.ts +15 -13
- package/scripts/voice-ttft-spike.ts +2 -2
- package/src/__tests__/agent-loop.test.ts +124 -0
- package/src/__tests__/always-loaded-tools-guard.test.ts +18 -10
- package/src/__tests__/anthropic-provider.test.ts +0 -63
- package/src/__tests__/app-compiler.test.ts +6 -0
- package/src/__tests__/app-control-tool-schemas.test.ts +5 -2
- package/src/__tests__/app-dir-path-guard.test.ts +1 -3
- package/src/__tests__/approval-interception-trust-gates.test.ts +40 -0
- package/src/__tests__/assistant-event-hub-machine-name.test.ts +6 -7
- package/src/__tests__/assistant-event-hub.test.ts +44 -0
- package/src/__tests__/attachments-store.test.ts +22 -3
- package/src/__tests__/channel-approval.test.ts +9 -14
- package/src/__tests__/channel-reply-delivery.test.ts +56 -0
- package/src/__tests__/computer-use-screenshot-attachments.test.ts +411 -0
- package/src/__tests__/computer-use-screenshot-selection.test.ts +50 -0
- package/src/__tests__/computer-use-skill-manifest-regression.test.ts +1 -1
- package/src/__tests__/computer-use-skill-proxy-bridge.test.ts +5 -2
- package/src/__tests__/computer-use-tools.test.ts +19 -7
- package/src/__tests__/config-schema-cmd.test.ts +2 -2
- package/src/__tests__/config-schema.test.ts +29 -30
- package/src/__tests__/config-sounds-sync.test.ts +23 -0
- package/src/__tests__/config-watcher.test.ts +23 -8
- package/src/__tests__/connection-model-compat.test.ts +2 -2
- package/src/__tests__/conversation-agent-loop-inference-profile.test.ts +2 -0
- package/src/__tests__/conversation-agent-loop-overflow.test.ts +2 -0
- package/src/__tests__/conversation-agent-loop.test.ts +257 -6
- package/src/__tests__/conversation-attachments.test.ts +106 -0
- package/src/__tests__/conversation-error.test.ts +4 -1
- package/src/__tests__/conversation-fork-crud.test.ts +191 -0
- package/src/__tests__/conversation-key-store-disk-view.test.ts +1 -1
- package/src/__tests__/conversation-pairing.test.ts +26 -0
- package/src/__tests__/conversation-process-app-control-preactivation.test.ts +4 -10
- package/src/__tests__/conversation-queue.test.ts +48 -207
- package/src/__tests__/conversation-retry-discard.test.ts +54 -1
- package/src/__tests__/conversation-runtime-assembly.test.ts +33 -3
- package/src/__tests__/conversation-skill-tools.test.ts +27 -0
- package/src/__tests__/conversation-surfaces-app-control.test.ts +8 -3
- package/src/__tests__/conversation-surfaces-point-at-budget.test.ts +1 -0
- package/src/__tests__/conversation-surfaces-point-at-capability.test.ts +1 -0
- package/src/__tests__/conversation-surfaces-task-progress.test.ts +67 -0
- package/src/__tests__/conversation-title-service.test.ts +14 -10
- package/src/__tests__/conversation-tool-setup-app-refresh.test.ts +3 -5
- package/src/__tests__/conversation-tool-setup-attribution.test.ts +51 -0
- package/src/__tests__/credential-execution-client.test.ts +70 -53
- package/src/__tests__/credential-record-retire.test.ts +171 -1
- package/src/__tests__/credential-routes.test.ts +250 -3
- package/src/__tests__/credential-security-invariants.test.ts +5 -3
- package/src/__tests__/cu-unified-flow.test.ts +14 -4
- package/src/__tests__/db-conversation-tool-surface.test.ts +144 -0
- package/src/__tests__/document-sync-tags.test.ts +25 -0
- package/src/__tests__/drain-kick-guard.test.ts +0 -2
- package/src/__tests__/drain-requeue-on-contention.test.ts +0 -6
- package/src/__tests__/events-tail-route.test.ts +33 -0
- package/src/__tests__/evict-conversations-for-reload.test.ts +1 -1
- package/src/__tests__/external-plugin-loader.test.ts +52 -0
- package/src/__tests__/git-porcelain.test.ts +47 -0
- package/src/__tests__/headless-browser-mode.test.ts +48 -0
- package/src/__tests__/heartbeat-service.test.ts +56 -40
- package/src/__tests__/helpers/set-workspace-mcp.ts +11 -0
- package/src/__tests__/host-bash-proxy.test.ts +26 -45
- package/src/__tests__/host-browser-proxy.test.ts +6 -17
- package/src/__tests__/host-cu-proxy.test.ts +712 -18
- package/src/__tests__/host-file-proxy-targeted.test.ts +23 -0
- package/src/__tests__/host-file-proxy.test.ts +11 -23
- package/src/__tests__/host-transfer-proxy-targeted.test.ts +30 -9
- package/src/__tests__/inference-profile-session-handler.test.ts +26 -0
- package/src/__tests__/list-all-apps.test.ts +40 -8
- package/src/__tests__/list-messages-camera-frame.test.ts +121 -0
- package/src/__tests__/list-messages-tool-merge.test.ts +85 -1
- package/src/__tests__/llm-catalog-parity.test.ts +51 -3
- package/src/__tests__/managed-store.test.ts +121 -0
- package/src/__tests__/mcp-auth-routes.test.ts +104 -8
- package/src/__tests__/mcp-client-auth.test.ts +54 -0
- package/src/__tests__/mcp-config-secret-boundary.test.ts +78 -136
- package/src/__tests__/mcp-health-check.test.ts +40 -43
- package/src/__tests__/mcp-list-plugin-servers.test.ts +85 -42
- package/src/__tests__/mcp-tool-annotations-risk.test.ts +4 -2
- package/src/__tests__/message-deleted-hook-dispatch.test.ts +119 -0
- package/src/__tests__/messages-after-tiebreaker.test.ts +91 -0
- package/src/__tests__/messages-read-boundary-guard.test.ts +1 -1
- package/src/__tests__/messaging-send-tool.test.ts +355 -548
- package/src/__tests__/model-intents.test.ts +3 -0
- package/src/__tests__/notification-vellum-adapter.test.ts +85 -2
- package/src/__tests__/oauth-apps-routes.test.ts +1 -0
- package/src/__tests__/oauth-commands-routes.test.ts +282 -101
- package/src/__tests__/oauth-connect-orchestrator.test.ts +2 -0
- package/src/__tests__/oauth-provider-profiles.test.ts +18 -0
- package/src/__tests__/oauth-provider-serializer.test.ts +23 -0
- package/src/__tests__/oauth-providers-routes.test.ts +3 -0
- package/src/__tests__/openai-provider.test.ts +1 -0
- package/src/__tests__/outlook-messaging-provider.test.ts +6 -6
- package/src/__tests__/persist-media-references.test.ts +50 -0
- package/src/__tests__/platform-callback-registration.test.ts +8 -16
- package/src/__tests__/platform-client-verify-credential.test.ts +2 -0
- package/src/__tests__/plugin-import-boundary-guard.test.ts +13 -0
- package/src/__tests__/qdrant-manager.test.ts +132 -0
- package/src/__tests__/resolve-app.test.ts +135 -0
- package/src/__tests__/run-conversation-turn-persistence.test.ts +138 -1
- package/src/__tests__/scaffold-managed-skill-tool.test.ts +317 -6
- package/src/__tests__/schedule-routes.test.ts +20 -0
- package/src/__tests__/scheduler-result-notification.test.ts +23 -4
- package/src/__tests__/script-proxy-certs.test.ts +35 -0
- package/src/__tests__/secret-routes-acp-guard.test.ts +59 -1
- package/src/__tests__/secret-routes-platform-proxy.test.ts +86 -31
- package/src/__tests__/secret-routes-scrub.test.ts +22 -27
- package/src/__tests__/secure-keys-managed-failover.test.ts +8 -0
- package/src/__tests__/secure-keys.test.ts +7 -3
- package/src/__tests__/service-account-token.test.ts +247 -0
- package/src/__tests__/skill-feature-flags-integration.test.ts +1 -1
- package/src/__tests__/skill-feature-flags.test.ts +2 -2
- package/src/__tests__/skills.test.ts +20 -4
- package/src/__tests__/source-fingerprint.test.ts +184 -0
- package/src/__tests__/subagent-tool-gate-mode.test.ts +239 -3
- package/src/__tests__/subagent-tools.test.ts +3 -2
- package/src/__tests__/telephony-tts-guard.test.ts +38 -3
- package/src/__tests__/terminal-tools.test.ts +8 -0
- package/src/__tests__/test-support/computer-use-skill-harness.ts +3 -2
- package/src/__tests__/tool-result-metadata-plumbing.test.ts +63 -0
- package/src/__tests__/ui-channel-variants.test.ts +1 -56
- package/src/__tests__/unicode.test.ts +49 -0
- package/src/__tests__/user-plugin-loader.test.ts +21 -0
- package/src/__tests__/vellum-self-knowledge-inline-command.test.ts +4 -0
- package/src/__tests__/watch-retro-tool-availability.test.ts +8 -17
- package/src/__tests__/workspace-git-service.test.ts +26 -0
- package/src/__tests__/workspace-migration-154-repair-retired-codex-gpt-5-4-model-ids.test.ts +336 -0
- package/src/acp/__tests__/acp-claude-oauth.test.ts +257 -6
- package/src/acp/__tests__/acp-credentials.test.ts +13 -0
- package/src/acp/__tests__/agent-process.test.ts +258 -4
- package/src/acp/__tests__/auth-required.test.ts +96 -0
- package/src/acp/__tests__/claude-token-refresh.test.ts +257 -0
- package/src/acp/__tests__/client-handler.test.ts +90 -2
- package/src/acp/__tests__/helpers/acp-model-option.ts +51 -0
- package/src/acp/__tests__/prepare-agent-env.test.ts +60 -1
- package/src/acp/__tests__/session-manager-persistence.test.ts +5 -1
- package/src/acp/__tests__/session-manager-resume.test.ts +270 -6
- package/src/acp/__tests__/session-manager-usage.test.ts +4 -2
- package/src/acp/__tests__/session-manager.test.ts +895 -4
- package/src/acp/acp-claude-oauth.ts +328 -14
- package/src/acp/acp-credentials.ts +19 -0
- package/src/acp/agent-process.test.ts +4 -1
- package/src/acp/agent-process.ts +96 -10
- package/src/acp/auth-required.ts +50 -1
- package/src/acp/auto-install.test.ts +9 -7
- package/src/acp/claude-token-refresh.ts +150 -0
- package/src/acp/client-handler.ts +16 -3
- package/src/acp/model-config.test.ts +166 -0
- package/src/acp/model-config.ts +138 -0
- package/src/acp/prepare-agent-env.ts +21 -6
- package/src/acp/resolve-agent.test.ts +220 -6
- package/src/acp/resolve-agent.ts +88 -23
- package/src/acp/session-manager.test.ts +196 -3
- package/src/acp/session-manager.ts +441 -18
- package/src/acp/session-snapshot.ts +260 -0
- package/src/acp/types.ts +50 -5
- package/src/agent/loop.ts +20 -0
- package/src/api/attachment-provenance.test.ts +68 -0
- package/src/api/computer-use-tool.test.ts +55 -0
- package/src/api/computer-use-tool.ts +27 -0
- package/src/api/events/acp-session-model-update.test.ts +88 -0
- package/src/api/events/acp-session-model-update.ts +46 -0
- package/src/api/events/app-files-changed.ts +4 -3
- package/src/api/events/assistant-outbound-attachment.ts +3 -0
- package/src/api/events/desktop-activity-changed.ts +10 -0
- package/src/api/events/notification-intent.ts +2 -0
- package/src/api/events/user-message-echo.ts +7 -1
- package/src/api/index.ts +17 -0
- package/src/api/responses/conversation-message.ts +8 -0
- package/src/approvals/approval-primitive.ts +5 -2
- package/src/approvals/scoped-approval-grants.ts +6 -2
- package/src/apps/app-store.ts +7 -6
- package/src/apps/resolve-app.ts +81 -0
- package/src/apps/source-fingerprint.ts +266 -0
- package/src/avatar/identity-avatar.ts +7 -2
- package/src/browser/operations.ts +2 -1
- package/src/browser/types.ts +7 -0
- package/src/browser/virtual-desktop-target.ts +34 -0
- package/src/bundler/app-compiler.ts +15 -0
- package/src/calls/__tests__/barge-in-guard.test.ts +81 -0
- package/src/{__tests__ → calls/__tests__}/call-controller.test.ts +503 -82
- package/src/{__tests__ → calls/__tests__}/media-stream-audio-transcode.test.ts +1 -1
- package/src/{__tests__ → calls/__tests__}/media-stream-output.test.ts +37 -8
- package/src/{__tests__ → calls/__tests__}/media-stream-parser.test.ts +1 -1
- package/src/{__tests__ → calls/__tests__}/media-stream-server-integration.test.ts +150 -57
- package/src/{__tests__ → calls/__tests__}/media-stream-stt-session.test.ts +5 -5
- package/src/{__tests__ → calls/__tests__}/media-turn-detector.test.ts +1 -1
- package/src/{live-voice → calls}/__tests__/progress-narration.test.ts +27 -4
- package/src/{live-voice → calls}/__tests__/progress-phrases.test.ts +1 -1
- package/src/calls/__tests__/voice-control-protocol.test.ts +151 -27
- package/src/calls/__tests__/voice-leg-coordinator.test.ts +229 -0
- package/src/calls/__tests__/voice-progress-cadence.test.ts +274 -0
- package/src/{__tests__/voice-session-bridge.test.ts → calls/__tests__/voice-session-bridge-turn-flow.test.ts} +20 -12
- package/src/calls/__tests__/voice-session-bridge.test.ts +140 -3
- package/src/calls/__tests__/voice-triage-escalate.test.ts +105 -0
- package/src/calls/barge-in-guard.ts +118 -0
- package/src/calls/call-constants.ts +24 -4
- package/src/calls/call-controller.ts +499 -199
- package/src/calls/call-transport.ts +16 -0
- package/src/calls/media-stream-output.ts +32 -0
- package/src/calls/media-stream-server.ts +82 -30
- package/src/calls/media-stream-stt-session.ts +15 -0
- package/src/{live-voice → calls}/progress-narration.ts +17 -4
- package/src/calls/resolve-call-tts-provider.ts +33 -2
- package/src/calls/tts-call-strategy.ts +9 -5
- package/src/calls/types.ts +2 -0
- package/src/calls/voice-control-protocol.ts +194 -21
- package/src/calls/voice-leg-coordinator.ts +189 -0
- package/src/calls/voice-progress-cadence.ts +423 -0
- package/src/calls/voice-session-bridge.ts +140 -76
- package/src/calls/voice-triage-escalate.ts +178 -59
- package/src/cli/commands/__tests__/apps.test.ts +245 -0
- package/src/cli/commands/__tests__/bash.test.ts +53 -0
- package/src/cli/commands/__tests__/browser.test.ts +84 -3
- package/src/cli/commands/__tests__/cli-test-harness.ts +21 -3
- package/src/cli/commands/__tests__/plugins.test.ts +85 -0
- package/src/cli/commands/__tests__/roadmap.help.test.ts +19 -0
- package/src/cli/commands/__tests__/schedules.test.ts +14 -0
- package/src/cli/commands/apps.help.ts +67 -13
- package/src/cli/commands/apps.ts +183 -0
- package/src/cli/commands/bash.help.ts +9 -13
- package/src/cli/commands/bash.ts +7 -25
- package/src/cli/commands/browser.help.ts +43 -0
- package/src/cli/commands/browser.ts +52 -8
- package/src/cli/commands/channels/__tests__/send.test.ts +218 -0
- package/src/cli/commands/channels/index.help.ts +70 -2
- package/src/cli/commands/channels/index.ts +16 -3
- package/src/cli/commands/channels/send.ts +128 -0
- package/src/cli/commands/credentials.help.ts +9 -5
- package/src/cli/commands/credentials.ts +2 -2
- package/src/cli/commands/mcp.help.ts +8 -8
- package/src/cli/commands/mcp.ts +1 -1
- package/src/cli/commands/oauth/index.help.ts +10 -0
- package/src/cli/commands/oauth/providers.ts +12 -0
- package/src/cli/commands/oauth/request.test.ts +179 -2
- package/src/cli/commands/oauth/request.ts +41 -10
- package/src/cli/commands/oauth/status.ts +5 -0
- package/src/cli/commands/plugins.ts +73 -23
- package/src/cli/commands/roadmap.help.ts +5 -1
- package/src/cli/commands/roadmap.ts +1 -1
- package/src/cli/commands/schedules.ts +2 -0
- package/src/cli/lib/__tests__/inspect-plugin.test.ts +54 -0
- package/src/cli/lib/__tests__/install-from-github.test.ts +219 -0
- package/src/cli/lib/__tests__/install-from-platform.test.ts +77 -2
- package/src/cli/lib/__tests__/list-installed-plugins.test.ts +45 -3
- package/src/cli/lib/__tests__/local-plugin-upgrade.test.ts +251 -0
- package/src/cli/lib/__tests__/plugin-catalog-cache.test.ts +65 -2
- package/src/cli/lib/__tests__/plugin-catalog-platform.test.ts +14 -0
- package/src/cli/lib/__tests__/plugin-catalog-resolve.test.ts +27 -1
- package/src/cli/lib/__tests__/plugin-details.test.ts +35 -2
- package/src/cli/lib/__tests__/plugin-marketplace.test.ts +20 -0
- package/src/cli/lib/__tests__/plugins-install-offline.test.ts +43 -0
- package/src/cli/lib/__tests__/search-plugins.test.ts +31 -0
- package/src/cli/lib/__tests__/uninstall-plugin.test.ts +122 -5
- package/src/cli/lib/bundled-marketplace.json +962 -81
- package/src/cli/lib/bundled-plugin-packages.json +786 -0
- package/src/cli/lib/bundled-plugin-packages.ts +87 -0
- package/src/cli/lib/diff-plugin.ts +1 -1
- package/src/cli/lib/inspect-plugin.ts +87 -12
- package/src/cli/lib/install-from-github.ts +150 -94
- package/src/cli/lib/install-from-platform.ts +14 -9
- package/src/cli/lib/list-installed-plugins.ts +21 -35
- package/src/cli/lib/plugin-catalog-cache.ts +22 -3
- package/src/cli/lib/plugin-catalog-local.ts +13 -3
- package/src/cli/lib/plugin-catalog-platform.ts +6 -1
- package/src/cli/lib/plugin-catalog-resolve.ts +20 -0
- package/src/cli/lib/plugin-details.ts +33 -8
- package/src/cli/lib/plugin-marketplace.ts +121 -21
- package/src/cli/lib/plugin-pin-history.ts +5 -2
- package/src/cli/lib/search-plugins.ts +58 -16
- package/src/cli/lib/uninstall-plugin.ts +59 -1
- package/src/cli/lib/upgrade-plugin.ts +61 -8
- package/src/cli/output.ts +18 -4
- package/src/config/__tests__/default-profile-catalog.test.ts +22 -0
- package/src/config/__tests__/default-provider.test.ts +4 -0
- package/src/config/__tests__/memory-retrospective-schema.test.ts +15 -1
- package/src/config/__tests__/platform-identity.test.ts +255 -0
- package/src/config/__tests__/platform-rehydration.test.ts +99 -9
- package/src/config/__tests__/plugin-resident-skill-discovery.test.ts +21 -0
- package/src/config/acp-agent-write.ts +43 -0
- package/src/config/acp-defaults.test.ts +13 -3
- package/src/config/acp-defaults.ts +23 -6
- package/src/config/acp-schema.test.ts +81 -0
- package/src/config/acp-schema.ts +30 -2
- package/src/config/bundled-skills/acp/SKILL.md +23 -15
- package/src/config/bundled-skills/acp/TOOLS.json +7 -3
- package/src/config/bundled-skills/app-builder/SKILL.md +12 -9
- package/src/config/bundled-skills/app-builder/references/CUSTOM_ROUTES.md +1 -1
- package/src/config/bundled-skills/computer-use/SKILL.md +59 -2
- package/src/config/bundled-skills/computer-use/TOOLS.json +101 -2
- package/src/config/bundled-skills/computer-use/tools/computer-use-sequence.ts +12 -0
- package/src/config/bundled-skills/media-processing/services/preprocess.ts +2 -1
- package/src/config/bundled-skills/media-processing/services/processing-pipeline.ts +2 -1
- package/src/config/bundled-skills/media-processing/tools/generate-clip.ts +2 -1
- package/src/config/bundled-skills/messaging/SKILL.md +3 -3
- package/src/config/bundled-skills/messaging/TOOLS.json +3 -3
- package/src/config/bundled-skills/messaging/tools/messaging-send.ts +67 -139
- package/src/config/bundled-skills/phone-calls/references/TRANSCRIPTS.md +1 -1
- package/src/config/bundled-skills/playbooks/tools/playbook-create.ts +2 -1
- package/src/config/bundled-skills/playbooks/tools/playbook-update.ts +2 -1
- package/src/config/bundled-skills/schedule/SKILL.md +7 -8
- package/src/config/bundled-skills/screen-annotation/SKILL.md +10 -3
- package/src/config/bundled-skills/screen-annotation/tools/screen-annotation.test.ts +4 -1
- package/src/config/bundled-skills/skill-management/SKILL.md +1 -1
- package/src/config/bundled-skills/skill-management/TOOLS.json +13 -9
- package/src/config/bundled-skills/transcribe/tools/transcribe-media.ts +5 -2
- package/src/config/default-profile-catalog.ts +1 -1
- package/src/config/env.ts +2 -2
- package/src/config/feature-flag-registry.json +33 -13
- package/src/config/loader.ts +11 -17
- package/src/config/memory-v3-gate.test.ts +31 -0
- package/src/config/memory-v3-gate.ts +20 -16
- package/src/config/platform-identity.ts +201 -0
- package/src/config/platform-rehydration.ts +11 -27
- package/src/config/profile-text-generation.test.ts +51 -0
- package/src/config/profile-text-generation.ts +51 -0
- package/src/config/schema.ts +2 -2
- package/src/config/schemas/__tests__/live-voice.test.ts +8 -158
- package/src/config/schemas/__tests__/voice.test.ts +148 -0
- package/src/config/schemas/calls.ts +1 -28
- package/src/config/schemas/live-voice.ts +1 -173
- package/src/config/schemas/llm.ts +7 -2
- package/src/config/schemas/mcp.ts +15 -9
- package/src/config/schemas/memory-retrospective.ts +10 -0
- package/src/config/schemas/monitoring.ts +9 -0
- package/src/config/schemas/services.ts +18 -0
- package/src/config/schemas/skills.ts +0 -14
- package/src/config/schemas/voice.ts +150 -0
- package/src/config/skills.ts +13 -21
- package/src/credential-execution/ces-runtime.ts +2 -2
- package/src/credential-execution/executable-discovery.ts +36 -134
- package/src/credential-execution/process-manager.test.ts +37 -42
- package/src/credential-execution/process-manager.ts +24 -19
- package/src/credential-health/credential-health-service.ts +11 -17
- package/src/daemon/__tests__/conversation-tool-setup-exclude.test.ts +47 -0
- package/src/daemon/__tests__/conversation-tool-setup.test.ts +109 -121
- package/src/daemon/__tests__/plugin-mcp-reconcile.test.ts +5 -1
- package/src/daemon/__tests__/skill-memory-refresh-mtime.test.ts +133 -0
- package/src/daemon/__tests__/turn-tail-deleted-conversation.test.ts +36 -0
- package/src/daemon/app-change-notify.ts +45 -0
- package/src/daemon/assistant-attachments.ts +20 -12
- package/src/daemon/config-watcher.ts +13 -194
- package/src/daemon/conversation-agent-loop-handlers.ts +74 -3
- package/src/daemon/conversation-agent-loop.ts +12 -0
- package/src/daemon/conversation-attachments.ts +93 -12
- package/src/daemon/conversation-client-surface.ts +26 -0
- package/src/daemon/conversation-error.ts +3 -2
- package/src/daemon/conversation-media-retry.ts +2 -1
- package/src/daemon/conversation-process.ts +35 -64
- package/src/daemon/conversation-runtime-assembly.ts +21 -8
- package/src/daemon/conversation-skill-tools.ts +13 -31
- package/src/daemon/conversation-store.ts +1 -1
- package/src/daemon/conversation-surfaces.ts +21 -10
- package/src/daemon/conversation-tool-setup.ts +135 -109
- package/src/daemon/conversation-turn-finalize.ts +26 -13
- package/src/daemon/conversation.ts +17 -3
- package/src/daemon/daemon-control.ts +3 -7
- package/src/daemon/dictation-profile-store.ts +2 -1
- package/src/daemon/handlers/shared.ts +2 -1
- package/src/daemon/handlers/skills.ts +6 -3
- package/src/daemon/host-bash-proxy.ts +8 -12
- package/src/daemon/host-browser-proxy.ts +15 -1
- package/src/daemon/host-cu-proxy.ts +275 -45
- package/src/daemon/host-file-proxy.ts +8 -10
- package/src/daemon/host-proxy-preactivation.ts +4 -1
- package/src/daemon/host-transfer-proxy.ts +14 -18
- package/src/daemon/lifecycle.ts +4 -9
- package/src/daemon/mcp-reload-service.ts +13 -6
- package/src/daemon/message-types/shared.ts +1 -0
- package/src/daemon/message-types/sync.ts +2 -0
- package/src/daemon/orphan-reaper.ts +4 -3
- package/src/daemon/persist-media-references.ts +20 -3
- package/src/daemon/providers-setup.ts +5 -1
- package/src/daemon/reaction-history-render.ts +2 -1
- package/src/daemon/shutdown-handlers.ts +1 -9
- package/src/daemon/skill-memory-refresh.ts +64 -0
- package/src/daemon/tool-setup-types.ts +6 -0
- package/src/daemon/tool-side-effects.ts +7 -18
- package/src/daemon/wake-conversation-ops.ts +38 -15
- package/src/desktop/__tests__/fake-desktop.ts +123 -0
- package/src/desktop/desktop-automation-lease.test.ts +351 -0
- package/src/desktop/desktop-automation-lease.ts +306 -0
- package/src/desktop/desktop-browser-client.test.ts +413 -0
- package/src/desktop/desktop-browser-client.ts +430 -0
- package/src/desktop/desktop-browser-cursor.ts +40 -0
- package/src/desktop/desktop-browser-endpoint.test.ts +78 -0
- package/src/desktop/desktop-browser-endpoint.ts +145 -0
- package/src/desktop/desktop-browser-operations.ts +130 -0
- package/src/desktop/desktop-chrome-policy.test.ts +79 -0
- package/src/desktop/desktop-chrome-policy.ts +47 -0
- package/src/desktop/desktop-chrome-session.test.ts +48 -0
- package/src/desktop/desktop-chrome-session.ts +87 -0
- package/src/desktop/desktop-dependencies.test.ts +131 -0
- package/src/desktop/desktop-dependencies.ts +329 -0
- package/src/desktop/desktop-display.ts +9 -0
- package/src/desktop/desktop-panel-config.test.ts +135 -0
- package/src/desktop/desktop-panel-config.ts +162 -0
- package/src/desktop/desktop-session-manager.test.ts +813 -0
- package/src/desktop/desktop-session-manager.ts +970 -0
- package/src/desktop/desktop-stream-bridge.test.ts +395 -0
- package/src/desktop/desktop-stream-bridge.ts +214 -0
- package/src/desktop/desktop-wallpaper.test.ts +191 -0
- package/src/desktop/desktop-wallpaper.ts +113 -0
- package/src/desktop/desktop-window-drag.ts +177 -0
- package/src/desktop/desktop-window-manager-config.test.ts +230 -0
- package/src/desktop/desktop-window-manager-config.ts +238 -0
- package/src/desktop/desktop-window-theme.ts +170 -0
- package/src/desktop/virtual-desktop-feature.ts +18 -0
- package/src/desktop/virtual-desktop-platform.test.ts +61 -0
- package/src/heartbeat/heartbeat-service.ts +21 -3
- package/src/home/__tests__/feed-writer.test.ts +22 -0
- package/src/home/feed-writer.ts +11 -2
- package/src/hooks/types.ts +33 -0
- package/src/i18n/AGENTS.md +22 -0
- package/src/i18n/__tests__/i18n.test.ts +112 -0
- package/src/i18n/index.ts +16 -0
- package/src/i18n/locales.ts +93 -0
- package/src/i18n/messages.ts +70 -0
- package/src/i18n/title.ts +25 -0
- package/src/inbound/platform-callback-registration.ts +4 -5
- package/src/ipc/__tests__/browser-ipc.test.ts +230 -2
- package/src/ipc/__tests__/cancel-on-disconnect.test.ts +54 -0
- package/src/ipc/assistant-server.ts +15 -1
- package/src/ipc/cli-client.ts +13 -5
- package/src/live-voice/__tests__/live-voice-agent-turn.test.ts +143 -0
- package/src/live-voice/__tests__/live-voice-events.test.ts +9 -7
- package/src/live-voice/__tests__/live-voice-look-follow-up.test.ts +374 -0
- package/src/live-voice/__tests__/live-voice-metrics.test.ts +0 -3
- package/src/live-voice/__tests__/live-voice-progress.test.ts +94 -15
- package/src/live-voice/__tests__/live-voice-session-telemetry.test.ts +19 -0
- package/src/live-voice/__tests__/live-voice-sight-frame-echo.test.ts +238 -0
- package/src/live-voice/__tests__/live-voice-vad.test.ts +660 -5
- package/src/live-voice/__tests__/protocol.test.ts +95 -0
- package/src/live-voice/__tests__/runtime-websocket-shell.test.ts +13 -69
- package/src/live-voice/__tests__/session-controls.test.ts +156 -0
- package/src/live-voice/live-voice-metrics.ts +9 -0
- package/src/live-voice/live-voice-photo.ts +25 -6
- package/src/live-voice/live-voice-session.ts +1081 -760
- package/src/live-voice/protocol.ts +89 -2
- package/src/live-voice/session-controls.ts +197 -0
- package/src/mcp/__tests__/credential-target.test.ts +59 -0
- package/src/mcp/__tests__/effective-config.test.ts +13 -2
- package/src/mcp/__tests__/manager-state.test.ts +128 -0
- package/src/mcp/__tests__/manager-tool-caps.test.ts +111 -0
- package/src/mcp/__tests__/mcp-auth-orchestrator.test.ts +20 -6
- package/src/mcp/__tests__/mcp-auth-state-target.test.ts +49 -0
- package/src/mcp/__tests__/plugin-mcp-oauth-cleanup.test.ts +107 -0
- package/src/mcp/__tests__/plugin-mcp-oauth-provider-isolation.test.ts +89 -0
- package/src/mcp/__tests__/plugin-server-credential-isolation.test.ts +63 -11
- package/src/mcp/__tests__/reload-signal-emission.test.ts +39 -2
- package/src/mcp/__tests__/startup.test.ts +30 -7
- package/src/mcp/__tests__/tool-caps.test.ts +107 -0
- package/src/mcp/__tests__/workspace-mcp-config.test.ts +135 -0
- package/src/mcp/client.ts +64 -16
- package/src/mcp/credential-target.ts +105 -0
- package/src/mcp/effective-config.ts +45 -18
- package/src/mcp/manager.ts +201 -81
- package/src/mcp/mcp-auth-orchestrator.ts +17 -4
- package/src/mcp/mcp-auth-state.ts +38 -6
- package/src/mcp/mcp-header-store.ts +29 -45
- package/src/mcp/mcp-oauth-provider.ts +101 -33
- package/src/mcp/spec-schema.ts +111 -0
- package/src/mcp/startup.ts +41 -17
- package/src/mcp/tool-caps.ts +176 -0
- package/src/mcp/workspace-mcp-config.ts +188 -0
- package/src/messaging/provider-message-metadata.ts +3 -3
- package/src/messaging/provider.ts +10 -3
- package/src/messaging/providers/__tests__/proactive-address.test.ts +95 -0
- package/src/messaging/providers/callback-routing.ts +18 -0
- package/src/messaging/providers/channel-transport.ts +56 -0
- package/src/messaging/providers/discord/transport.ts +17 -0
- package/src/messaging/providers/slack/__tests__/adapter-token-routing.test.ts +6 -17
- package/src/messaging/providers/slack/adapter.ts +7 -44
- package/src/messaging/providers/slack/transport.ts +16 -0
- package/src/messaging/providers/telegram-bot/adapter.ts +1 -48
- package/src/messaging/providers/telegram-bot/client.ts +0 -26
- package/src/messaging/providers/telegram-bot/transport.ts +20 -0
- package/src/messaging/providers/whatsapp/adapter.ts +1 -48
- package/src/messaging/providers/whatsapp/transport.ts +18 -0
- package/src/monitoring/__tests__/mount-watch.test.ts +236 -0
- package/src/monitoring/__tests__/workspace-git-heartbeat.test.ts +8 -0
- package/src/monitoring/mount-watch.ts +348 -0
- package/src/monitoring/plugin-auto-update.ts +6 -0
- package/src/monitoring/worker.ts +9 -0
- package/src/notifications/__tests__/assistant-reply-producer.test.ts +55 -0
- package/src/notifications/__tests__/copy-composer.test.ts +70 -0
- package/src/notifications/__tests__/decision-engine.test.ts +175 -0
- package/src/notifications/__tests__/guardian-feed-projection.test.ts +158 -2
- package/src/notifications/__tests__/home-feed-side-effect.test.ts +153 -3
- package/src/notifications/__tests__/schedule-result-producer.test.ts +99 -3
- package/src/notifications/adapters/macos.ts +3 -0
- package/src/notifications/assistant-reply-producer.ts +8 -3
- package/src/notifications/conversation-pairing.ts +36 -16
- package/src/notifications/copy-composer.ts +11 -3
- package/src/notifications/decision-engine.ts +43 -8
- package/src/notifications/delivered-post-record.ts +11 -1
- package/src/notifications/guardian-feed-projection.ts +125 -5
- package/src/notifications/home-feed-side-effect.ts +55 -7
- package/src/notifications/schedule-result-producer.ts +64 -22
- package/src/oauth/AGENTS.md +4 -0
- package/src/oauth/__tests__/identity-verifier.test.ts +36 -1
- package/src/oauth/__tests__/seed-providers-managed.test.ts +149 -0
- package/src/oauth/byo-connection.test.ts +105 -0
- package/src/oauth/byo-connection.ts +36 -6
- package/src/oauth/connection-resolver.test.ts +100 -0
- package/src/oauth/connection-resolver.ts +120 -3
- package/src/oauth/identity-verifier.ts +24 -0
- package/src/oauth/oauth-store.ts +11 -1
- package/src/oauth/provider-serializer.ts +10 -0
- package/src/oauth/scope-utils.ts +24 -0
- package/src/oauth/seed-providers.ts +367 -27
- package/src/onboarding/checkin-event.ts +2 -1
- package/src/outbound-proxy/certs.ts +3 -5
- package/src/permissions/checker.test.ts +11 -6
- package/src/permissions/confirmation-guardian-request.test.ts +16 -0
- package/src/permissions/confirmation-guardian-request.ts +2 -3
- package/src/persistence/attachments-store.ts +84 -44
- package/src/persistence/compaction-log-store-clickhouse.ts +3 -3
- package/src/persistence/conversation-crud.ts +182 -83
- package/src/persistence/conversation-lineage.ts +12 -4
- package/src/persistence/conversation-plugin-facade.ts +13 -0
- package/src/persistence/conversation-queries.ts +2 -2
- package/src/persistence/conversation-title-placeholders.ts +23 -0
- package/src/persistence/conversation-title-service.ts +9 -26
- package/src/persistence/conversation-tool-surface.ts +86 -0
- package/src/persistence/conversation-types.test.ts +32 -0
- package/src/persistence/conversation-types.ts +26 -4
- package/src/persistence/embeddings/qdrant-manager.ts +163 -18
- package/src/persistence/llm-request-log-sink-clickhouse.ts +3 -3
- package/src/persistence/llm-request-log-source-clickhouse.ts +3 -3
- package/src/persistence/message-cursor.ts +78 -0
- package/src/persistence/message-reads.ts +30 -0
- package/src/persistence/migrations/157-invite-contact-id.ts +18 -6
- package/src/persistence/migrations/378-create-conversation-tool-surfaces.test.ts +78 -0
- package/src/persistence/migrations/378-create-conversation-tool-surfaces.ts +29 -0
- package/src/persistence/migrations/379-oauth-providers-response-ok-field.test.ts +92 -0
- package/src/persistence/migrations/379-oauth-providers-response-ok-field.ts +32 -0
- package/src/persistence/migrations/__tests__/157-invite-contact-id.test.ts +149 -0
- package/src/persistence/schema/conversation-tool-surfaces.ts +22 -0
- package/src/persistence/schema/index.ts +1 -0
- package/src/persistence/schema/memory-core.ts +5 -0
- package/src/persistence/schema/oauth.ts +1 -0
- package/src/persistence/steps.ts +15 -3
- package/src/platform/client.test.ts +4 -2
- package/src/platform/client.ts +2 -10
- package/src/plugin-api/constants.ts +2 -0
- package/src/plugin-api/conversation-turn.ts +31 -7
- package/src/plugin-api/index.ts +10 -1
- package/src/plugin-api/plugin-channel-turn-trust.test.ts +133 -0
- package/src/plugin-api/plugin-channel-turn-trust.ts +71 -0
- package/src/plugin-api/types.ts +2 -0
- package/src/plugins/__tests__/installed-plugin-dirs.test.ts +14 -1
- package/src/plugins/__tests__/mcp-servers.test.ts +38 -8
- package/src/plugins/defaults/index.ts +2 -0
- package/src/plugins/defaults/memory/AGENTS.md +14 -2
- package/src/plugins/defaults/memory/__tests__/buffer-file.test.ts +365 -0
- package/src/plugins/defaults/memory/__tests__/buffer-format.test.ts +43 -0
- package/src/plugins/defaults/memory/__tests__/fixtures/buffer-appender.ts +17 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-accounting.test.ts +46 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-cursor-backfill.test.ts +118 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-cursor-preserve.test.ts +83 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-enqueue.test.ts +9 -5
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-job.test.ts +66 -10
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-prompt.test.ts +20 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-state.test.ts +219 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-sweep.test.ts +31 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-trigger-check.test.ts +63 -0
- package/src/plugins/defaults/memory/__tests__/memory-run-evidence.test.ts +161 -0
- package/src/plugins/defaults/memory/__tests__/shutdown-hook.test.ts +4 -0
- package/src/plugins/defaults/memory/buffer-file.ts +354 -0
- package/src/plugins/defaults/memory/buffer-format.ts +40 -0
- package/src/plugins/defaults/memory/context-search/agent-runner.ts +3 -3
- package/src/plugins/defaults/memory/context-search/format.ts +4 -2
- package/src/plugins/defaults/memory/context-search/sources/memory-v2.ts +7 -3
- package/src/plugins/defaults/memory/context-search/sources/workspace.ts +4 -2
- package/src/plugins/defaults/memory/graph/capability-seed.ts +6 -8
- package/src/plugins/defaults/memory/graph/tool-handlers.ts +1 -42
- package/src/plugins/defaults/memory/hooks/init.ts +10 -0
- package/src/plugins/defaults/memory/hooks/message-deleted.ts +21 -0
- package/src/plugins/defaults/memory/hooks/shutdown.ts +2 -0
- package/src/plugins/defaults/memory/injectors.ts +4 -3
- package/src/plugins/defaults/memory/jobs-worker.ts +13 -0
- package/src/plugins/defaults/memory/memory-retrospective-accounting.ts +34 -60
- package/src/plugins/defaults/memory/memory-retrospective-cursor-backfill.ts +126 -0
- package/src/plugins/defaults/memory/memory-retrospective-cursor-preserve.ts +79 -0
- package/src/plugins/defaults/memory/memory-retrospective-cursor.ts +28 -0
- package/src/plugins/defaults/memory/memory-retrospective-enqueue.ts +3 -5
- package/src/plugins/defaults/memory/memory-retrospective-job.ts +67 -184
- package/src/plugins/defaults/memory/memory-retrospective-prompt.ts +1 -1
- package/src/plugins/defaults/memory/memory-retrospective-state.ts +312 -71
- package/src/plugins/defaults/memory/memory-retrospective-sweep.ts +2 -1
- package/src/plugins/defaults/memory/memory-retrospective-trigger-check.ts +5 -3
- package/src/plugins/defaults/memory/memory-run-evidence.ts +213 -0
- package/src/plugins/defaults/memory/skill-md-mtime-poll.ts +9 -0
- package/src/plugins/defaults/memory/src/memory-item-routes.test.ts +1 -1
- package/src/plugins/defaults/memory/startup.ts +28 -2
- package/src/plugins/defaults/memory/substrate/__tests__/consolidation-job.test.ts +407 -99
- package/src/plugins/defaults/memory/substrate/__tests__/consolidation-prompt-flag-gating-guard.test.ts +10 -0
- package/src/plugins/defaults/memory/substrate/__tests__/prompts-consolidation.test.ts +107 -7
- package/src/plugins/defaults/memory/substrate/__tests__/skill-content.test.ts +7 -9
- package/src/plugins/defaults/memory/substrate/consolidation-job.ts +307 -86
- package/src/plugins/defaults/memory/substrate/consolidation-tool-surface.ts +34 -0
- package/src/plugins/defaults/memory/substrate/page-index.ts +7 -1
- package/src/plugins/defaults/memory/substrate/prompts/consolidation.ts +89 -49
- package/src/plugins/defaults/memory/substrate/skill-content.ts +2 -2
- package/src/plugins/defaults/memory/substrate/sweep-job.ts +1 -1
- package/src/plugins/defaults/memory/tools.ts +1 -1
- package/src/plugins/defaults/memory/v1/graph/consolidation.ts +4 -2
- package/src/plugins/defaults/memory/v1/graph/extraction.ts +4 -1
- package/src/plugins/defaults/memory/v1/graph/retriever.ts +4 -1
- package/src/plugins/defaults/memory/v2/__tests__/migration.test.ts +5 -0
- package/src/plugins/defaults/memory/v2/__tests__/reranker.test.ts +5 -2
- package/src/plugins/defaults/memory/v2/reranker.ts +7 -1
- package/src/plugins/defaults/memory/v3/__tests__/card.test.ts +24 -0
- package/src/plugins/defaults/memory/v3/__tests__/injection.test.ts +81 -1
- package/src/plugins/defaults/memory/v3/__tests__/orchestrate.test.ts +87 -0
- package/src/plugins/defaults/memory/v3/__tests__/pool-select.test.ts +125 -0
- package/src/plugins/defaults/memory/v3/__tests__/sections.test.ts +26 -0
- package/src/plugins/defaults/memory/v3/__tests__/shadow-plugin.test.ts +51 -0
- package/src/plugins/defaults/memory/v3/card.ts +4 -2
- package/src/plugins/defaults/memory/v3/injector.ts +212 -178
- package/src/plugins/defaults/memory/v3/orchestrate.ts +86 -22
- package/src/plugins/defaults/memory/v3/pool-select.ts +13 -8
- package/src/plugins/defaults/memory/v3/sections.ts +10 -3
- package/src/plugins/defaults/memory/v3/shadow-plugin.ts +13 -1
- package/src/plugins/defaults/memory/worker.ts +6 -0
- package/src/plugins/defaults/tool-result-truncate/terminal.ts +1 -46
- package/src/plugins/external-plugin-loader.ts +49 -86
- package/src/plugins/installed-plugin-dirs.ts +6 -4
- package/src/plugins/mcp-servers.ts +24 -90
- package/src/plugins/mtime-cache.ts +2 -1
- package/src/plugins/user-loader.ts +2 -1
- package/src/providers/__tests__/connection-model-compat.test.ts +3 -3
- package/src/providers/__tests__/retry-callsite.test.ts +41 -0
- package/src/providers/__tests__/retry-outbound-sanitize.test.ts +176 -0
- package/src/providers/anthropic/client.ts +0 -47
- package/src/providers/gemini/client.ts +9 -23
- package/src/providers/gemini-thought-signature.test.ts +63 -0
- package/src/providers/gemini-thought-signature.ts +44 -0
- package/src/providers/inference/__tests__/endpoint-probe.test.ts +87 -3
- package/src/providers/inference/adapter-factory.ts +6 -0
- package/src/providers/inference/auth.ts +2 -3
- package/src/providers/inference/connection-availability.ts +27 -8
- package/src/providers/inference/endpoint-probe.ts +36 -12
- package/src/providers/inference/resolve-auth.ts +22 -4
- package/src/providers/inference/service-account-token.ts +248 -0
- package/src/providers/jev/client.test.ts +260 -0
- package/src/providers/jev/client.ts +518 -0
- package/src/providers/model-catalog.ts +92 -3
- package/src/providers/model-intents.ts +10 -5
- package/src/providers/openai/__tests__/chat-completions-output-token-limit.test.ts +141 -0
- package/src/providers/openai/__tests__/chat-completions-provider-reasoning.test.ts +146 -18
- package/src/providers/openai/__tests__/chat-completions-provider-thought-signature.test.ts +613 -0
- package/src/providers/openai/__tests__/google-thought-signature.test.ts +235 -0
- package/src/providers/openai/__tests__/tool-choice-mapping.test.ts +83 -1
- package/src/providers/openai/chat-completions-provider.ts +212 -79
- package/src/providers/openai/codex-models.ts +0 -4
- package/src/providers/openai/google-thought-signature.ts +320 -0
- package/src/providers/openai/responses-provider.ts +9 -2
- package/src/providers/opencode/client.test.ts +240 -4
- package/src/providers/opencode/client.ts +129 -3
- package/src/providers/openrouter/client.ts +20 -0
- package/src/providers/outbound-request-sanitize.ts +96 -0
- package/src/providers/retry.ts +30 -19
- package/src/providers/unparseable-tool-args.ts +5 -1
- package/src/runtime/AGENTS.md +9 -3
- package/src/runtime/__tests__/agent-wake.test.ts +86 -1
- package/src/runtime/__tests__/channel-send.test.ts +443 -0
- package/src/runtime/__tests__/desktop-stream-upgrade.test.ts +132 -0
- package/src/runtime/__tests__/host-observe.test.ts +40 -1
- package/src/runtime/__tests__/live-voice-upgrade.test.ts +58 -0
- package/src/runtime/__tests__/runtime-ws-test-utils.ts +72 -0
- package/src/runtime/__tests__/watch-stream-upgrade.test.ts +59 -0
- package/src/runtime/agent-wake.ts +22 -5
- package/src/runtime/assistant-event-hub.ts +21 -3
- package/src/runtime/auth/same-actor.ts +38 -3
- package/src/runtime/channel-send.ts +316 -0
- package/src/runtime/guardian-action-service.ts +2 -17
- package/src/runtime/guardian-reply-router.ts +1 -8
- package/src/runtime/host-observe.ts +11 -1
- package/src/runtime/http-server.ts +160 -151
- package/src/runtime/migrations/__tests__/vbundle-import-policy.test.ts +89 -0
- package/src/runtime/migrations/vbundle-import-policy.ts +33 -11
- package/src/runtime/routes/__tests__/acp-claude-auth-routes.test.ts +16 -5
- package/src/runtime/routes/__tests__/acp-routes.test.ts +178 -4
- package/src/runtime/routes/__tests__/apps-refresh-route.test.ts +275 -0
- package/src/runtime/routes/__tests__/channel-send-routes.test.ts +127 -0
- package/src/runtime/routes/__tests__/conversation-query-routes.test.ts +108 -1
- package/src/runtime/routes/__tests__/default-provider-routes.test.ts +2 -2
- package/src/runtime/routes/__tests__/dictation-cleanup-guard.test.ts +107 -0
- package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +24 -2
- package/src/runtime/routes/__tests__/mcp-add-default-risk.test.ts +16 -12
- package/src/runtime/routes/__tests__/plugins-routes.test.ts +127 -3
- package/src/runtime/routes/acp-claude-auth-routes.ts +11 -3
- package/src/runtime/routes/acp-routes.test.ts +15 -0
- package/src/runtime/routes/acp-routes.ts +84 -165
- package/src/runtime/routes/app-management-routes.ts +90 -2
- package/src/runtime/routes/app-routes.ts +1 -1
- package/src/runtime/routes/browser-context.ts +39 -0
- package/src/runtime/routes/browser-routes.ts +16 -47
- package/src/runtime/routes/browser-tabs-routes.ts +72 -29
- package/src/runtime/routes/channel-route-shared.ts +1 -9
- package/src/runtime/routes/channel-send-routes.ts +85 -0
- package/src/runtime/routes/conversation-list-routes.ts +14 -3
- package/src/runtime/routes/conversation-query-routes.ts +40 -0
- package/src/runtime/routes/conversation-routes.ts +45 -7
- package/src/runtime/routes/credential-routes.ts +20 -14
- package/src/runtime/routes/desktop-setup-routes.test.ts +89 -0
- package/src/runtime/routes/desktop-setup-routes.ts +41 -0
- package/src/runtime/routes/diagnostics-routes.ts +61 -4
- package/src/runtime/routes/events-routes.ts +4 -0
- package/src/runtime/routes/guardian-approval-interception.ts +24 -0
- package/src/runtime/routes/host-cu-routes.ts +6 -0
- package/src/runtime/routes/inbound-message-handler.ts +2 -3
- package/src/runtime/routes/inbound-stages/background-dispatch.test.ts +1 -1
- package/src/runtime/routes/inbound-stages/background-dispatch.ts +7 -4
- package/src/runtime/routes/index.ts +4 -2
- package/src/runtime/routes/inference-profile-session-handler.ts +11 -1
- package/src/runtime/routes/inference-profiles-routes.ts +9 -0
- package/src/runtime/routes/log-export-routes.ts +2 -0
- package/src/runtime/routes/mcp-auth-routes.ts +141 -137
- package/src/runtime/routes/migration-routes.ts +6 -20
- package/src/runtime/routes/oauth-commands-routes.ts +98 -45
- package/src/runtime/routes/oauth-providers.ts +11 -0
- package/src/runtime/routes/oauth-request-hints.test.ts +246 -0
- package/src/runtime/routes/oauth-request-hints.ts +149 -0
- package/src/runtime/routes/platform-routes.ts +18 -15
- package/src/runtime/routes/plugins-routes.ts +83 -50
- package/src/runtime/routes/schedule-routes.ts +2 -0
- package/src/runtime/routes/secret-routes.ts +99 -119
- package/src/runtime/routes/workspace-routes.test.ts +55 -26
- package/src/runtime/routes/workspace-routes.ts +46 -63
- package/src/runtime/routes/workspace-tree-walk.test.ts +244 -0
- package/src/runtime/routes/workspace-tree-walk.ts +240 -0
- package/src/runtime/routes/workspace-utils.ts +17 -0
- package/src/runtime/services/__tests__/conversation-serializer.test.ts +28 -0
- package/src/runtime/services/conversation-serializer.ts +16 -3
- package/src/runtime/sync/resource-sync-events.ts +6 -1
- package/src/schedule/__tests__/worker-mcp-bootstrap.test.ts +107 -0
- package/src/schedule/__tests__/worker-mcp-readiness.test.ts +173 -0
- package/src/schedule/__tests__/worker-mcp-tools.test.ts +13 -47
- package/src/schedule/run-script.ts +4 -3
- package/src/schedule/schedule-store.ts +9 -2
- package/src/schedule/scheduler.ts +47 -11
- package/src/schedule/tool-surface-readiness.ts +65 -0
- package/src/schedule/worker-mcp.ts +85 -0
- package/src/schedule/worker.ts +9 -31
- package/src/security/__tests__/ces-http-record-backend.test.ts +137 -0
- package/src/security/ces-http-record-backend.ts +120 -0
- package/src/security/ces-rpc-record-backend.ts +7 -0
- package/src/security/secure-keys.ts +60 -48
- package/src/security/untrusted-content.ts +3 -1
- package/src/skills/__tests__/workspace-skill-md-signature.test.ts +60 -0
- package/src/skills/managed-store.ts +153 -31
- package/src/skills/workspace-skill-md-signature.ts +50 -0
- package/src/subagent/consult-context.ts +12 -25
- package/src/tools/__tests__/side-effect-abort-guard.test.ts +1 -0
- package/src/tools/acp/list-agents.test.ts +3 -1
- package/src/tools/acp/spawn.test.ts +146 -11
- package/src/tools/acp/spawn.ts +21 -3
- package/src/tools/acp/status.test.ts +349 -0
- package/src/tools/acp/status.ts +112 -9
- package/src/tools/app-control/skill-proxy-bridge.ts +2 -1
- package/src/tools/browser/browser-execution.ts +45 -14
- package/src/tools/browser/cdp-client/__tests__/factory.test.ts +69 -0
- package/src/tools/browser/cdp-client/accessibility-snapshot.ts +9 -3
- package/src/tools/browser/cdp-client/cdp-inspect/__tests__/ws-transport.test.ts +1 -0
- package/src/tools/browser/cdp-client/cdp-inspect/discovery.ts +2 -1
- package/src/tools/browser/cdp-client/cdp-inspect/ws-transport.ts +5 -0
- package/src/tools/browser/cdp-client/extension-cdp-client.ts +3 -2
- package/src/tools/browser/network-recorder.ts +2 -1
- package/src/tools/capability-offer.test.ts +22 -2
- package/src/tools/capability-offer.ts +16 -1
- package/src/tools/computer-use/definitions.ts +123 -5
- package/src/tools/computer-use/skill-proxy-bridge.ts +2 -1
- package/src/tools/credentials/metadata-store.ts +100 -0
- package/src/tools/filesystem/search.ts +2 -1
- package/src/tools/host-filesystem/edit.test.ts +2 -1
- package/src/tools/host-filesystem/edit.ts +2 -2
- package/src/tools/host-filesystem/read.test.ts +2 -1
- package/src/tools/host-filesystem/read.ts +2 -2
- package/src/tools/host-filesystem/transfer.test.ts +2 -1
- package/src/tools/host-filesystem/transfer.ts +2 -2
- package/src/tools/host-filesystem/write.test.ts +2 -1
- package/src/tools/host-filesystem/write.ts +2 -2
- package/src/tools/host-terminal/host-shell.test.ts +16 -0
- package/src/tools/host-terminal/host-shell.ts +16 -26
- package/src/tools/network/web-search.ts +6 -1
- package/src/tools/policy-context.ts +6 -7
- package/src/tools/shared/filesystem/file-ops-service.ts +1 -31
- package/src/tools/shared/shell-output.test.ts +10 -0
- package/src/tools/shared/shell-output.ts +14 -2
- package/src/tools/skills/find-similar-skills.test.ts +214 -2
- package/src/tools/skills/find-similar-skills.ts +62 -9
- package/src/tools/skills/resolve-execute-invocation.ts +24 -0
- package/src/tools/skills/sandbox-runner.ts +13 -2
- package/src/tools/skills/scaffold-managed.ts +78 -16
- package/src/tools/subagent/spawn.ts +5 -8
- package/src/tools/terminal/__tests__/safe-env.test.ts +33 -4
- package/src/tools/terminal/__tests__/sanitized-bash.test.ts +43 -0
- package/src/tools/terminal/safe-env.ts +35 -24
- package/src/tools/terminal/sanitized-bash.ts +118 -0
- package/src/tools/terminal/shell-launch.test.ts +162 -0
- package/src/tools/terminal/shell.test.ts +29 -0
- package/src/tools/terminal/shell.ts +13 -7
- package/src/tools/ui-surface/channel-variants.ts +10 -59
- package/src/tools/watch/watch-retro-report.ts +0 -8
- package/src/types/host-capabilities.ts +5 -3
- package/src/util/__tests__/language-subtag.test.ts +23 -1
- package/src/util/abort-reasons.ts +12 -1
- package/src/util/ca-bundle.ts +13 -0
- package/src/util/host-process.test.ts +17 -1
- package/src/util/host-process.ts +24 -0
- package/src/util/language-subtag.ts +16 -0
- package/src/util/platform.ts +8 -0
- package/src/util/plugin-manifest.ts +203 -0
- package/src/util/unicode.ts +44 -1
- package/src/watch/watch-retro.ts +6 -8
- package/src/watch/watch-timeline.ts +2 -2
- package/src/workflows/run-manager.ts +2 -1
- package/src/workspace/git-porcelain.ts +66 -0
- package/src/workspace/git-service.ts +6 -55
- package/src/workspace/migrations/154-repair-retired-codex-gpt-5-4-model-ids.ts +258 -0
- package/src/workspace/migrations/155-move-front-model-config-to-voice.ts +78 -0
- package/src/workspace/migrations/156-extract-workspace-mcp-json.ts +214 -0
- package/src/workspace/migrations/__tests__/155-move-front-model-config-to-voice.test.ts +102 -0
- package/src/workspace/migrations/__tests__/156-extract-workspace-mcp-json.test.ts +199 -0
- package/src/workspace/migrations/registry.ts +6 -0
- package/src/__tests__/app-source-watcher.test.ts +0 -218
- package/src/__tests__/config-watcher-skill-reseed.test.ts +0 -359
- package/src/__tests__/speaker-identification.test.ts +0 -56
- package/src/calls/speaker-identification.ts +0 -233
- package/src/daemon/app-source-watcher.ts +0 -218
- package/src/messaging/providers/whatsapp/client.ts +0 -29
- package/src/runtime/routes/debug-bash-routes.ts +0 -173
- package/src/util/fs-watcher-error.ts +0 -36
- /package/src/{live-voice → calls}/progress-phrases.ts +0 -0
|
@@ -1,19 +1,41 @@
|
|
|
1
1
|
import { Buffer } from "node:buffer";
|
|
2
2
|
import { randomUUID } from "node:crypto";
|
|
3
3
|
|
|
4
|
+
import {
|
|
5
|
+
type BargeInGuard,
|
|
6
|
+
createBargeInGuard,
|
|
7
|
+
DEFAULT_BARGE_IN_MIN_SPEECH_MS,
|
|
8
|
+
} from "../calls/barge-in-guard.js";
|
|
4
9
|
import {
|
|
5
10
|
MediaTurnDetector,
|
|
6
11
|
type TurnDetectorConfig,
|
|
7
12
|
} from "../calls/media-turn-detector.js";
|
|
13
|
+
import {
|
|
14
|
+
createVoiceProgressNarrator,
|
|
15
|
+
type VoiceProgressNarrator,
|
|
16
|
+
} from "../calls/progress-narration.js";
|
|
17
|
+
import {
|
|
18
|
+
APPROVAL_PENDING_PHRASE_BY_LANGUAGE,
|
|
19
|
+
approvalPendingPhraseFor,
|
|
20
|
+
} from "../calls/progress-phrases.js";
|
|
8
21
|
import {
|
|
9
22
|
SPOKEN_REPLY_LENGTH_RULE,
|
|
10
23
|
SPOKEN_REPLY_PLAIN_TEXT_RULE,
|
|
11
24
|
} from "../calls/spoken-reply-rules.js";
|
|
12
25
|
import { sanitizeForTts } from "../calls/tts-text-sanitizer.js";
|
|
13
26
|
import {
|
|
14
|
-
|
|
15
|
-
|
|
27
|
+
createControlMarkerHoldback,
|
|
28
|
+
TASK_STOP_MARKER,
|
|
16
29
|
} from "../calls/voice-control-protocol.js";
|
|
30
|
+
import {
|
|
31
|
+
createFrontDoorLegCoordinator,
|
|
32
|
+
type FrontDoorLegCoordinator,
|
|
33
|
+
type SpokenEscalationBridge,
|
|
34
|
+
} from "../calls/voice-leg-coordinator.js";
|
|
35
|
+
import {
|
|
36
|
+
createProgressCadence,
|
|
37
|
+
type ProgressCadence,
|
|
38
|
+
} from "../calls/voice-progress-cadence.js";
|
|
17
39
|
import type {
|
|
18
40
|
VoiceTurnHandle,
|
|
19
41
|
VoiceTurnOptions,
|
|
@@ -23,24 +45,16 @@ import {
|
|
|
23
45
|
resolveProcessingWaitMs,
|
|
24
46
|
waitForPriorTurnTeardown,
|
|
25
47
|
} from "../calls/voice-session-bridge.js";
|
|
26
|
-
import {
|
|
27
|
-
capEscalationBridge,
|
|
28
|
-
classifyFrontDoorLeading,
|
|
29
|
-
ESCALATE_VERDICT_TOKEN,
|
|
30
|
-
ESCALATION_CONTINUATION_CONTENT,
|
|
31
|
-
FALLBACK_ESCALATION_BRIDGE_BY_LANGUAGE,
|
|
32
|
-
fallbackEscalationBridgeFor,
|
|
33
|
-
isEscalationBridgeComplete,
|
|
34
|
-
MIN_SPOKEN_BRIDGE_CHARS,
|
|
35
|
-
type VoiceRoutingLeg,
|
|
36
|
-
} from "../calls/voice-triage-escalate.js";
|
|
48
|
+
import { type VoiceRoutingLeg } from "../calls/voice-triage-escalate.js";
|
|
37
49
|
import { getConfig } from "../config/loader.js";
|
|
38
50
|
import {
|
|
39
51
|
type LiveVoiceFluxConfig,
|
|
40
52
|
LiveVoiceFluxConfigSchema,
|
|
41
|
-
type LiveVoiceFrontModelConfig,
|
|
42
|
-
LiveVoiceFrontModelConfigSchema,
|
|
43
53
|
} from "../config/schemas/live-voice.js";
|
|
54
|
+
import {
|
|
55
|
+
type VoiceFrontModelConfig,
|
|
56
|
+
VoiceFrontModelConfigSchema,
|
|
57
|
+
} from "../config/schemas/voice.js";
|
|
44
58
|
import { ABORT_WATCHDOG_MS } from "../daemon/abort-watchdog.js";
|
|
45
59
|
import { isRefusedInReadOnlyPass } from "../daemon/conversation-tool-setup.js";
|
|
46
60
|
import type { TrustContext } from "../daemon/trust-context-types.js";
|
|
@@ -86,14 +100,15 @@ import {
|
|
|
86
100
|
liveVoiceSilenceReason,
|
|
87
101
|
liveVoiceStartScreen,
|
|
88
102
|
} from "../telemetry/live-voice-funnel.js";
|
|
89
|
-
import { getToolOwner } from "../tools/registry.js";
|
|
103
|
+
import { getTool, getToolOwner } from "../tools/registry.js";
|
|
104
|
+
import { resolveSkillExecuteInvocation } from "../tools/skills/resolve-execute-invocation.js";
|
|
90
105
|
import {
|
|
91
106
|
createReasoningTagFilter,
|
|
92
107
|
type ReasoningTagFilter,
|
|
93
108
|
} from "../tts/reasoning-tag-filter.js";
|
|
94
109
|
import { extractSpeakableSegments } from "../tts/speakable-segments.js";
|
|
95
110
|
import { createAbortReason } from "../util/abort-reasons.js";
|
|
96
|
-
import {
|
|
111
|
+
import { fixedPhraseLanguage } from "../util/language-subtag.js";
|
|
97
112
|
import { getLogger } from "../util/logger.js";
|
|
98
113
|
import {
|
|
99
114
|
activityLabelForTool,
|
|
@@ -136,17 +151,6 @@ import type {
|
|
|
136
151
|
LiveVoiceTtsOptions,
|
|
137
152
|
LiveVoiceTtsResult,
|
|
138
153
|
} from "./live-voice-tts.js";
|
|
139
|
-
import {
|
|
140
|
-
createVoiceProgressNarrator,
|
|
141
|
-
type VoiceProgressNarrator,
|
|
142
|
-
type VoiceProgressTextInput,
|
|
143
|
-
} from "./progress-narration.js";
|
|
144
|
-
import {
|
|
145
|
-
APPROVAL_PENDING_PHRASE_BY_LANGUAGE,
|
|
146
|
-
approvalPendingPhraseFor,
|
|
147
|
-
pickProgressPhrase,
|
|
148
|
-
PROGRESS_FALLBACK_PHRASES_BY_LANGUAGE,
|
|
149
|
-
} from "./progress-phrases.js";
|
|
150
154
|
import {
|
|
151
155
|
type LiveVoiceClientAttachFrameFrame,
|
|
152
156
|
type LiveVoiceClientAttachImageFrame,
|
|
@@ -156,7 +160,19 @@ import {
|
|
|
156
160
|
type LiveVoiceClientUpdateConfigFrame,
|
|
157
161
|
LiveVoiceProtocolErrorCode,
|
|
158
162
|
type LiveVoiceServerFramePayload,
|
|
163
|
+
type LiveVoiceSessionControl,
|
|
159
164
|
} from "./protocol.js";
|
|
165
|
+
import {
|
|
166
|
+
type ClientSessionControlRequest,
|
|
167
|
+
isLookSessionControl,
|
|
168
|
+
LOOK_FOLLOW_UP_CONTENT,
|
|
169
|
+
LOOK_FRAME_REASON,
|
|
170
|
+
lookFollowUpNote,
|
|
171
|
+
type LookSessionControl,
|
|
172
|
+
progressConfigForCadence,
|
|
173
|
+
requestedSessionControl,
|
|
174
|
+
sessionControlTeaching,
|
|
175
|
+
} from "./session-controls.js";
|
|
160
176
|
|
|
161
177
|
const log = getLogger("live-voice-session");
|
|
162
178
|
|
|
@@ -200,7 +216,6 @@ const FINALIZE_GRACE_MS = 1_000;
|
|
|
200
216
|
// upstream of the provider, which hears only the microphone: it cannot tell
|
|
201
217
|
// our own TTS bleeding through imperfect echo cancellation from the caller,
|
|
202
218
|
// and would report a user turn nobody took.
|
|
203
|
-
const DEFAULT_BARGE_IN_MIN_SPEECH_MS = 250;
|
|
204
219
|
// The playback echo gate learns microphone energy while assistant audio is
|
|
205
220
|
// expected at the speaker. Input must rise above the learned level by this
|
|
206
221
|
// margin to count as user speech.
|
|
@@ -236,24 +251,6 @@ const NOISE_FLOOR_MAX_BASE_MULTIPLE = 4;
|
|
|
236
251
|
// tracks the effective trailing-silence threshold (the detector keeps its own
|
|
237
252
|
// copy private) so the endpoint decider can report the pause length.
|
|
238
253
|
const DEFAULT_SILENCE_THRESHOLD_MS = 800;
|
|
239
|
-
// Longest continuous sub-threshold gap the sustained-speech barge-in run
|
|
240
|
-
// tolerates without resetting. A gap this short is a syllable boundary, or the
|
|
241
|
-
// choppy energy the browser's half-duplex echo canceller produces while the
|
|
242
|
-
// assistant is still playing (it ducks the user's near-end voice, so post-AEC
|
|
243
|
-
// user speech arrives as intermittent above-gate chunks) — so the run keeps
|
|
244
|
-
// accumulating across it and a barge-in during playback still lands. Only a
|
|
245
|
-
// longer continuous silence (a real end of speech, or an isolated cough) resets
|
|
246
|
-
// the run.
|
|
247
|
-
const BARGE_IN_GAP_TOLERANCE_MS = 200;
|
|
248
|
-
// Ceiling on cumulative sub-threshold time across a whole barge-in run, as a
|
|
249
|
-
// multiple of bargeInMinSpeechMs. Per-gap tolerance alone lets sparse isolated
|
|
250
|
-
// blips (e.g. a 10 ms echo spike every 200 ms) each clear the consecutive-gap
|
|
251
|
-
// timer while retaining prior speech, so they would sum to the guard over
|
|
252
|
-
// several seconds and fire a barge-in with no sustained user speech. Capping the
|
|
253
|
-
// run's total tolerated silence imposes a minimum above-gate duty cycle
|
|
254
|
-
// (1 / (1 + ratio) ≈ 20%): once the run is mostly silence it resets, so genuine
|
|
255
|
-
// choppy speech still lands but periodic noise cannot accumulate into one.
|
|
256
|
-
const BARGE_IN_MAX_TOLERATED_SILENCE_RATIO = 4;
|
|
257
254
|
// Slack added to the configured end-of-turn timeout before the session stops
|
|
258
255
|
// waiting for an event that is not coming and falls the utterance back onto
|
|
259
256
|
// the silence-boundary path. A turn-detecting provider force-ends its own turn
|
|
@@ -277,11 +274,26 @@ const CONTINUATION_ANNOUNCE_SILENCE_MS = 1_500;
|
|
|
277
274
|
// reply's audio landing between two checks. Beyond the cap the announcement
|
|
278
275
|
// falls back to the stash rather than chasing a call that keeps talking.
|
|
279
276
|
const CONTINUATION_ANNOUNCE_MAX_DRAIN_REARMS = 3;
|
|
277
|
+
const FOREGROUND_TASK_RESUME_SILENCE_MS = 1_500;
|
|
278
|
+
const FOREGROUND_TASK_MAX_SUSPENDED_MS = 120_000;
|
|
279
|
+
const FOREGROUND_TASK_MAX_INTERVENING_TURNS = 3;
|
|
280
280
|
// `content` of an announcement turn. The answer never rides here — it goes in
|
|
281
281
|
// the model-facing control prompt (buildLiveDeliveryNote), so the only thing
|
|
282
282
|
// persisted on the user side is this marker, and it persists hidden.
|
|
283
283
|
export const CONTINUATION_DELIVERY_CONTENT =
|
|
284
284
|
"(background work finished — deliver it now)";
|
|
285
|
+
// How long a look control waits for the fresh frame the client takes for it.
|
|
286
|
+
// Long enough for a camera that has to open and warm up before its first keep;
|
|
287
|
+
// past it the look is dropped rather than answered into a later, unrelated
|
|
288
|
+
// silence.
|
|
289
|
+
const LOOK_FRAME_WAIT_MS = 10_000;
|
|
290
|
+
// How long after the look was asked for its answer may still start. Past it
|
|
291
|
+
// the floor has been taken for so long (a user talking through noise that never
|
|
292
|
+
// became a turn, say) that the look no longer answers what is on screen.
|
|
293
|
+
const LOOK_ANSWER_DEADLINE_MS = 15_000;
|
|
294
|
+
// The shortest wait before checking the floor again, for a blocker with no
|
|
295
|
+
// known end time (the look's own turn clearing, an utterance in capture).
|
|
296
|
+
const LOOK_FOLLOW_UP_REARM_MS = 250;
|
|
285
297
|
|
|
286
298
|
export type LiveVoiceStreamingTranscriberResolver = (
|
|
287
299
|
options: ResolveStreamingTranscriberOptions,
|
|
@@ -407,11 +419,11 @@ export interface LiveVoiceSessionOptions {
|
|
|
407
419
|
finalizeGraceMs?: number;
|
|
408
420
|
/**
|
|
409
421
|
* Voice front-door endpointing and progress narration tuning. The
|
|
410
|
-
* production factory seeds this from `
|
|
422
|
+
* production factory seeds this from `voice.frontModel` config when
|
|
411
423
|
* unset; absent fields fall back to the schema defaults (the constructor
|
|
412
424
|
* schema-parses the partial into a complete config).
|
|
413
425
|
*/
|
|
414
|
-
frontModelConfig?: Partial<
|
|
426
|
+
frontModelConfig?: Partial<VoiceFrontModelConfig>;
|
|
415
427
|
/**
|
|
416
428
|
* Deepgram Flux turn-detection tuning. The production factory seeds this
|
|
417
429
|
* from `liveVoice.flux` config when unset; absent fields fall back to the
|
|
@@ -421,7 +433,7 @@ export interface LiveVoiceSessionOptions {
|
|
|
421
433
|
fluxConfig?: Partial<LiveVoiceFluxConfig>;
|
|
422
434
|
/**
|
|
423
435
|
* Progress narration service. The production factory constructs it from
|
|
424
|
-
* `
|
|
436
|
+
* `voice.frontModel.progress` config when unset. `null` disables
|
|
425
437
|
* generated narration, so progress updates use their static fallback.
|
|
426
438
|
*/
|
|
427
439
|
progressNarrator?: VoiceProgressNarrator | null;
|
|
@@ -457,6 +469,12 @@ export interface LiveVoiceSessionOptions {
|
|
|
457
469
|
* `CONTINUATION_ANNOUNCE_SILENCE_MS`.
|
|
458
470
|
*/
|
|
459
471
|
continuationAnnounceSilenceMs?: number;
|
|
472
|
+
/** Overrides the silence before a suspended foreground task resumes (test hook). */
|
|
473
|
+
foregroundTaskResumeSilenceMs?: number;
|
|
474
|
+
/** Overrides the maximum time a foreground task may remain suspended (test hook). */
|
|
475
|
+
foregroundTaskMaxSuspendedMs?: number;
|
|
476
|
+
/** Overrides the committed user-turn limit while suspended (test hook). */
|
|
477
|
+
foregroundTaskMaxInterveningTurns?: number;
|
|
460
478
|
}
|
|
461
479
|
|
|
462
480
|
type LiveVoiceUtterancePhase =
|
|
@@ -604,70 +622,6 @@ interface TtsSegmentJob {
|
|
|
604
622
|
frames: Promise<void>;
|
|
605
623
|
}
|
|
606
624
|
|
|
607
|
-
// One tool operation observed on a turn, fed by the bridge's structured tool
|
|
608
|
-
// callbacks. `completedAtMs`/`isError`/`resultPreview` land with tool_result.
|
|
609
|
-
interface TurnProgressOp {
|
|
610
|
-
toolName: string;
|
|
611
|
-
toolUseId?: string;
|
|
612
|
-
startedAtMs: number;
|
|
613
|
-
completedAtMs?: number;
|
|
614
|
-
isError?: boolean;
|
|
615
|
-
resultPreview?: string;
|
|
616
|
-
}
|
|
617
|
-
|
|
618
|
-
// Per-turn tool-activity log and narration cadence state for spoken progress
|
|
619
|
-
// updates (liveVoice.frontModel.progress).
|
|
620
|
-
interface TurnProgressState {
|
|
621
|
-
// Tool operations this turn, in start order.
|
|
622
|
-
ops: TurnProgressOp[];
|
|
623
|
-
// Ops accumulated toward the next ops-triggered narration. Counted once per
|
|
624
|
-
// op, on start (not completion), so a burst of slow tools still trips the
|
|
625
|
-
// threshold while they run.
|
|
626
|
-
opsSinceNarration: number;
|
|
627
|
-
// Bumped by every observable change to the turn's tool activity — an op
|
|
628
|
-
// starting or finishing. The idle trigger compares it against
|
|
629
|
-
// `narratedEpoch` so a tick with nothing new to report stays silent.
|
|
630
|
-
stateEpoch: number;
|
|
631
|
-
// The `stateEpoch` the last spoken narration described: the activity the
|
|
632
|
-
// user has already been told about.
|
|
633
|
-
narratedEpoch: number;
|
|
634
|
-
// Narrations actually spoken this turn — the metrics count and the
|
|
635
|
-
// decider's 1-based updateIndex. Rate, not count, bounds narration:
|
|
636
|
-
// idleIntervalMs/minGapMs cap the cadence and the session duration cap
|
|
637
|
-
// bounds the turn.
|
|
638
|
-
updatesSpoken: number;
|
|
639
|
-
// When the last spoken floor-holder (ack or narration) enqueued — gates the
|
|
640
|
-
// progress.minGapMs spacing guard. Null until something speaks.
|
|
641
|
-
lastFloorHolderAtMs: number | null;
|
|
642
|
-
// When the turn's last TTS segment finished emitting (turn launch until
|
|
643
|
-
// anything speaks). Together with the session-level playback-tail estimate
|
|
644
|
-
// this anchors the dead-air countdown: idle time is measured from when the
|
|
645
|
-
// user last heard something, not from when the turn started.
|
|
646
|
-
lastAudibleAtMs: number;
|
|
647
|
-
// Self-re-arming dead-air narration timer; null once cleared or fired.
|
|
648
|
-
idleTimer: ReturnType<typeof setTimeout> | null;
|
|
649
|
-
// A narration generation is awaiting the decider; serializes narrations.
|
|
650
|
-
narrationInFlight: boolean;
|
|
651
|
-
}
|
|
652
|
-
|
|
653
|
-
// Newest incomplete op, optionally restricted to a tool name (the
|
|
654
|
-
// tool_result fallback match when no toolUseId correlates).
|
|
655
|
-
function findLastIncompleteOp(
|
|
656
|
-
ops: TurnProgressOp[],
|
|
657
|
-
toolName?: string,
|
|
658
|
-
): TurnProgressOp | undefined {
|
|
659
|
-
for (let i = ops.length - 1; i >= 0; i -= 1) {
|
|
660
|
-
const op = ops[i];
|
|
661
|
-
if (
|
|
662
|
-
op.completedAtMs === undefined &&
|
|
663
|
-
(toolName === undefined || op.toolName === toolName)
|
|
664
|
-
) {
|
|
665
|
-
return op;
|
|
666
|
-
}
|
|
667
|
-
}
|
|
668
|
-
return undefined;
|
|
669
|
-
}
|
|
670
|
-
|
|
671
625
|
interface ActiveAssistantTurn {
|
|
672
626
|
token: symbol;
|
|
673
627
|
turnId: string;
|
|
@@ -683,7 +637,9 @@ interface ActiveAssistantTurn {
|
|
|
683
637
|
handle: VoiceTurnHandle | null;
|
|
684
638
|
// When the turn launched, for narration's turnElapsedMs.
|
|
685
639
|
launchedAtMs: number;
|
|
686
|
-
progress
|
|
640
|
+
// Tool-activity log and spoken progress narration for the turn
|
|
641
|
+
// (voice.frontModel.progress), shared with the phone driver.
|
|
642
|
+
progress: ProgressCadence;
|
|
687
643
|
assistantCompleted: boolean;
|
|
688
644
|
ttsDone: boolean;
|
|
689
645
|
// Latched when the turn puts something on screen (a ui tool ran); consumed
|
|
@@ -691,6 +647,10 @@ interface ActiveAssistantTurn {
|
|
|
691
647
|
// Never set from anything the model says: the reveal is a consequence of
|
|
692
648
|
// showing a surface, not a token the model has to remember.
|
|
693
649
|
minimizeRequested: boolean;
|
|
650
|
+
// The session control the completed reply ended with (see
|
|
651
|
+
// session-controls.ts); consumed at TTS drain like the minimize, where the
|
|
652
|
+
// session_control frame goes out once the acknowledgement has been spoken.
|
|
653
|
+
sessionControlRequested: ClientSessionControlRequest | null;
|
|
694
654
|
// The activity label the client was last told about, so a run of tools that
|
|
695
655
|
// map to the same line sends one frame rather than one per call. Empty means
|
|
696
656
|
// the client believes nothing is running, which is also where a turn ends.
|
|
@@ -767,10 +727,21 @@ interface ActiveAssistantTurn {
|
|
|
767
727
|
// no user utterance behind it — `content` is CONTINUATION_DELIVERY_CONTENT and
|
|
768
728
|
// the answer rides the control prompt (buildLiveDeliveryNote).
|
|
769
729
|
continuationDelivery: ContinuationDelivery | null;
|
|
730
|
+
// Set only on the turn that answers a look: which look it answers. The turn
|
|
731
|
+
// has no user utterance behind it; the instruction rides the control prompt
|
|
732
|
+
// (lookFollowUpNote).
|
|
733
|
+
lookFollowUp: LookSessionControl | null;
|
|
770
734
|
// The turn's content is an internal instruction rather than user speech (the
|
|
771
735
|
// greeting that opens a session, say). The row still persists and the model
|
|
772
736
|
// still sees it; `hiddenSyntheticPrompt` keeps it out of the transcript.
|
|
773
737
|
hiddenPrompt: boolean;
|
|
738
|
+
// The suspended foreground task this turn may continue. Set only after the user
|
|
739
|
+
// turn commits or on the synthetic resume turn; speculative discards never
|
|
740
|
+
// mutate the session's task state.
|
|
741
|
+
foregroundTaskEpoch: number | null;
|
|
742
|
+
// Snapshot retained while a tool-capable leg owns suspended work. If that
|
|
743
|
+
// leg fails asynchronously, the task returns to the same suspension budget.
|
|
744
|
+
foregroundTaskSuspensionSnapshot: SuspendedForegroundTask | null;
|
|
774
745
|
// Set when a barge-in handed the interrupted work to a background subagent:
|
|
775
746
|
// that request's transcript, so the model can tell the user the work is
|
|
776
747
|
// still running instead of appearing to have dropped it.
|
|
@@ -803,7 +774,9 @@ interface ActiveAssistantTurn {
|
|
|
803
774
|
// Guards the
|
|
804
775
|
// front-door leg's trailing completion from finalizing the turn, and makes
|
|
805
776
|
// the hand-off idempotent.
|
|
806
|
-
|
|
777
|
+
// The front-door leg's coordinator: whether it handed the turn off to the
|
|
778
|
+
// escalated leg. Null until the front-door leg starts.
|
|
779
|
+
frontDoor: FrontDoorLegCoordinator | null;
|
|
807
780
|
ttsBuffer: string;
|
|
808
781
|
// What the caller actually hears this turn, summed over the model's own
|
|
809
782
|
// segments (acks and progress narration do not count). Logged at tts_done
|
|
@@ -829,55 +802,6 @@ interface ActiveAssistantTurn {
|
|
|
829
802
|
assistantAudioSampleRate?: number;
|
|
830
803
|
}
|
|
831
804
|
|
|
832
|
-
/**
|
|
833
|
-
* Control-marker hygiene for one model leg's delta stream, shared by the
|
|
834
|
-
* front-door answer stage and the default/escalated leg. The returned flush
|
|
835
|
-
* forwards the stripped (stripInternalSpeechMarkers) prefix of `raw` that has
|
|
836
|
-
* not been emitted yet and cannot contain a still-streaming control marker:
|
|
837
|
-
* the flush stops at the first "[" whose tail is an incomplete marker
|
|
838
|
-
* (isIncompleteControlMarkerTail) and holds from there until a later delta
|
|
839
|
-
* completes or disproves it; `force` (leg completion) emits the held tail so
|
|
840
|
-
* real text that merely resembles a marker prefix is not dropped. The scan
|
|
841
|
-
* runs forward from the emitted boundary — not from the last "[" — so
|
|
842
|
-
* brackets INSIDE a streaming marker body (a JSON array or "]"-bearing string
|
|
843
|
-
* in ASK_GUARDIAN_APPROVAL) can neither mask the marker's start nor pass as
|
|
844
|
-
* its terminator.
|
|
845
|
-
*
|
|
846
|
-
* **Markers are stripped, never acted on.** Nothing the model can say
|
|
847
|
-
* minimizes the room any more: that is decided by whether a ui tool ran (see
|
|
848
|
-
* the `tool_use_start` handler), so the reveal cannot depend on the model
|
|
849
|
-
* remembering a token, and a reply whose content happens to contain "[-1]" (an
|
|
850
|
-
* array literal, a temperature) cannot move the room either. No prompt teaches
|
|
851
|
-
* a marker, so this stripping is defense against a model that emits one
|
|
852
|
-
* regardless: an unspoken, unpersisted "[-1]" is the correct handling of a
|
|
853
|
-
* token that now means nothing.
|
|
854
|
-
*/
|
|
855
|
-
function createControlMarkerHoldback(
|
|
856
|
-
turn: ActiveAssistantTurn,
|
|
857
|
-
emit: (chunk: string) => void,
|
|
858
|
-
): (raw: string, opts?: { force?: boolean }) => void {
|
|
859
|
-
let emitted = 0;
|
|
860
|
-
return (raw, opts) => {
|
|
861
|
-
let safeEnd = raw.length;
|
|
862
|
-
if (opts?.force !== true) {
|
|
863
|
-
for (
|
|
864
|
-
let i = raw.indexOf("[", emitted);
|
|
865
|
-
i !== -1;
|
|
866
|
-
i = raw.indexOf("[", i + 1)
|
|
867
|
-
) {
|
|
868
|
-
if (isIncompleteControlMarkerTail(raw.slice(i))) {
|
|
869
|
-
safeEnd = i;
|
|
870
|
-
break;
|
|
871
|
-
}
|
|
872
|
-
}
|
|
873
|
-
}
|
|
874
|
-
if (safeEnd > emitted) {
|
|
875
|
-
emit(stripInternalSpeechMarkers(raw.slice(emitted, safeEnd)));
|
|
876
|
-
emitted = safeEnd;
|
|
877
|
-
}
|
|
878
|
-
};
|
|
879
|
-
}
|
|
880
|
-
|
|
881
805
|
// Base control prompt for every live-voice turn. Opens with the spoken-reply
|
|
882
806
|
// rules shared with the phone path (spoken-reply-rules.ts): this is the only
|
|
883
807
|
// place the model learns that its text is spoken, since the system prompt has
|
|
@@ -885,7 +809,7 @@ function createControlMarkerHoldback(
|
|
|
885
809
|
// message. When a turn starts from a barge-in, the interruption merge note is
|
|
886
810
|
// appended to it (see buildInterruptionMergeNote) so the model reconciles the
|
|
887
811
|
// interrupted request with the new utterance.
|
|
888
|
-
const LIVE_VOICE_CONTROL_PROMPT_BASE = `You are speaking in a local live voice session. ${SPOKEN_REPLY_LENGTH_RULE} ${SPOKEN_REPLY_PLAIN_TEXT_RULE} Speech is the main channel: say the answer, and do not narrate a surface instead of answering. You can also put something on screen when it genuinely helps (a form, a list to pick from, a progress card for long work); the call overlay minimizes by itself once you finish speaking, so the user sees it without doing anything. Never tell the user you cannot show them something. Reply in the language the caller is speaking; if they switch languages, switch with them. `;
|
|
812
|
+
const LIVE_VOICE_CONTROL_PROMPT_BASE = `You are speaking in a local live voice session. ${SPOKEN_REPLY_LENGTH_RULE} ${SPOKEN_REPLY_PLAIN_TEXT_RULE} Speech is the main channel: say the answer, and do not narrate a surface instead of answering. You can also put something on screen when it genuinely helps (a form, a list to pick from, a progress card for long work, or a card with an image or a link instead of reading an address out); the call overlay minimizes by itself once you finish speaking, so the user sees it without doing anything. Never tell the user you cannot show them something. Reply in the language the caller is speaking; if they switch languages, switch with them. `;
|
|
889
813
|
|
|
890
814
|
// Appended for the legs that can actually put something on screen: the main
|
|
891
815
|
// leg and the escalated leg. The front-door (fast) leg never receives it, for
|
|
@@ -901,7 +825,7 @@ const LIVE_VOICE_CONTROL_PROMPT_BASE = `You are speaking in a local live voice s
|
|
|
901
825
|
// model can get right, which is speaking as though the thing is already in
|
|
902
826
|
// front of the user, because by the time it stops talking it is.
|
|
903
827
|
const LIVE_VOICE_SCREEN_REVEAL_TEACHING =
|
|
904
|
-
"When the complete answer would run past a few sentences, say the short version out loud and put the detail on screen instead of reading it out. The call renders as a full-screen overlay covering the app. Whenever you put something on screen, the overlay minimizes by itself as soon as you finish speaking, and the user is looking at what you made. So speak as if you are showing it to them right now (for example, close with something like: take a look), and never say you cannot show it, that this is a voice call, or that they should check it later.
|
|
828
|
+
"When the complete answer would run past a few sentences, say the short version out loud and put the detail on screen instead of reading it out. The call renders as a full-screen overlay covering the app. Whenever you put something on screen, the overlay minimizes by itself as soon as you finish speaking, and the user is looking at what you made. So speak as if you are showing it to them right now (for example, close with something like: take a look), and never say you cannot show it, that this is a voice call, or that they should check it later. ";
|
|
905
829
|
|
|
906
830
|
// The setup-flow case, spelled out because it is the one the model gets wrong
|
|
907
831
|
// on its own: connecting an account reads as something a call cannot do, so it
|
|
@@ -922,7 +846,7 @@ const LIVE_VOICE_SETUP_FLOW_TEACHING =
|
|
|
922
846
|
// off answering, rather than a fresh follow-up. Reaches the model only; it is
|
|
923
847
|
// not a user message and never renders as a transcript bubble.
|
|
924
848
|
function buildInterruptionMergeNote(interruptedRequest: string): string {
|
|
925
|
-
return `The user interrupted your previous, unfinished reply. Their earlier request was: "${interruptedRequest}". Treat their current message as a continuation of that request and address both together
|
|
849
|
+
return `The user interrupted your previous, unfinished reply. Their earlier request was: "${interruptedRequest}". Treat their current message as a continuation of that request and address both together. If they corrected or redirected the unfinished work, continue executing it with that correction instead of merely acknowledging it. If they clearly want the unfinished task abandoned, use the task-stop control.`;
|
|
926
850
|
}
|
|
927
851
|
|
|
928
852
|
// System-level guidance appended to the NEXT turn's control prompt after a
|
|
@@ -955,6 +879,14 @@ function describeInterruptedRequest(request: string): string {
|
|
|
955
879
|
: "their earlier request";
|
|
956
880
|
}
|
|
957
881
|
|
|
882
|
+
// A look control waiting on its fresh frame: which look, when it was asked
|
|
883
|
+
// for, and the wait's bound.
|
|
884
|
+
interface PendingLook {
|
|
885
|
+
action: LookSessionControl;
|
|
886
|
+
armedAtMs: number;
|
|
887
|
+
timer: ReturnType<typeof setTimeout>;
|
|
888
|
+
}
|
|
889
|
+
|
|
958
890
|
// A finished background continuation waiting to be delivered: the request it
|
|
959
891
|
// took over and the answer it produced.
|
|
960
892
|
interface ContinuationDelivery {
|
|
@@ -962,6 +894,26 @@ interface ContinuationDelivery {
|
|
|
962
894
|
answer: string;
|
|
963
895
|
}
|
|
964
896
|
|
|
897
|
+
interface OwnedForegroundTask {
|
|
898
|
+
phase: "owned";
|
|
899
|
+
epoch: number;
|
|
900
|
+
request: string;
|
|
901
|
+
ownerToken: symbol;
|
|
902
|
+
hostToolStarted: boolean;
|
|
903
|
+
}
|
|
904
|
+
|
|
905
|
+
interface SuspendedForegroundTask {
|
|
906
|
+
phase: "suspended";
|
|
907
|
+
epoch: number;
|
|
908
|
+
request: string;
|
|
909
|
+
suspendedAtMs: number;
|
|
910
|
+
interveningTurns: number;
|
|
911
|
+
resumePending: boolean;
|
|
912
|
+
hostToolStarted: boolean;
|
|
913
|
+
}
|
|
914
|
+
|
|
915
|
+
type ForegroundTaskState = OwnedForegroundTask | SuspendedForegroundTask;
|
|
916
|
+
|
|
965
917
|
// Appended to an announcement turn's control prompt: the turn the session
|
|
966
918
|
// starts on its own when a background continuation finishes while the call is
|
|
967
919
|
// live and nobody is speaking. There is no user utterance behind it, so this
|
|
@@ -973,6 +925,12 @@ function buildLiveDeliveryNote(request: string, answer: string): string {
|
|
|
973
925
|
return `The user interrupted you earlier, and in the background you finished ${what}. The call is still live and nobody is speaking, so tell them briefly that it is done and give them the result. Do not re-run any tool calls; the work is already complete, and do not repeat it verbatim if it no longer fits. What you produced was:\n\n${answer}`;
|
|
974
926
|
}
|
|
975
927
|
|
|
928
|
+
function buildForegroundTaskResumePrompt(request: string): string {
|
|
929
|
+
const task =
|
|
930
|
+
request.length > 0 ? JSON.stringify(request) : "the unfinished task";
|
|
931
|
+
return `Continue the unfinished task that began with ${task}. Follow the user's latest corrections and all newer conversation context. If the conversation has clearly moved on or the user no longer wants the task, do not continue it and output only ${TASK_STOP_MARKER}, with no spoken words before the marker.`;
|
|
932
|
+
}
|
|
933
|
+
|
|
976
934
|
// Assemble a leg's model-facing control prompt: the base live-voice rules, the
|
|
977
935
|
// screen-reveal and setup-flow teaching (both withheld from the front-door leg,
|
|
978
936
|
// see LIVE_VOICE_SCREEN_REVEAL_TEACHING), plus any pending barge-in merge
|
|
@@ -982,12 +940,21 @@ function buildLiveDeliveryNote(request: string, answer: string): string {
|
|
|
982
940
|
function buildVoiceControlPrompt(
|
|
983
941
|
turn: ActiveAssistantTurn,
|
|
984
942
|
leg: { frontDoor?: boolean },
|
|
943
|
+
sessionControls: readonly LiveVoiceSessionControl[],
|
|
944
|
+
client: { lookFrames: boolean },
|
|
985
945
|
): string {
|
|
986
946
|
let prompt =
|
|
987
947
|
LIVE_VOICE_CONTROL_PROMPT_BASE +
|
|
988
948
|
(leg.frontDoor === true
|
|
989
949
|
? ""
|
|
990
|
-
: LIVE_VOICE_SCREEN_REVEAL_TEACHING + LIVE_VOICE_SETUP_FLOW_TEACHING)
|
|
950
|
+
: LIVE_VOICE_SCREEN_REVEAL_TEACHING + LIVE_VOICE_SETUP_FLOW_TEACHING) +
|
|
951
|
+
sessionControlTeaching(sessionControls, leg, {
|
|
952
|
+
...client,
|
|
953
|
+
unfinishedTaskPending:
|
|
954
|
+
turn.foregroundTaskEpoch !== null ||
|
|
955
|
+
turn.interruptedRequest !== null ||
|
|
956
|
+
turn.handedOffRequest !== null,
|
|
957
|
+
});
|
|
991
958
|
if (turn.language !== undefined) {
|
|
992
959
|
prompt = `${prompt}\n\nThe caller has been speaking the language with code "${turn.language}" this turn. Reply in that language unless they clearly switch to another.`;
|
|
993
960
|
}
|
|
@@ -1000,6 +967,9 @@ function buildVoiceControlPrompt(
|
|
|
1000
967
|
if (turn.handedOffRequest) {
|
|
1001
968
|
prompt = `${prompt}\n\n${buildHandoffAnnouncementNote(turn.handedOffRequest)}`;
|
|
1002
969
|
}
|
|
970
|
+
if (turn.lookFollowUp !== null) {
|
|
971
|
+
prompt = `${prompt}\n\n${lookFollowUpNote(turn.lookFollowUp)}`;
|
|
972
|
+
}
|
|
1003
973
|
if (turn.continuationDelivery) {
|
|
1004
974
|
prompt = `${prompt}\n\n${buildLiveDeliveryNote(
|
|
1005
975
|
turn.continuationDelivery.request,
|
|
@@ -1152,6 +1122,20 @@ function foregroundToolContendsWithContinuation(
|
|
|
1152
1122
|
return isRefusedInReadOnlyPass(toolName, ownerKind);
|
|
1153
1123
|
}
|
|
1154
1124
|
|
|
1125
|
+
function hostInteractiveToolName(
|
|
1126
|
+
toolName: string,
|
|
1127
|
+
input?: Record<string, unknown>,
|
|
1128
|
+
allowedToolNames?: ReadonlySet<string>,
|
|
1129
|
+
): string | null {
|
|
1130
|
+
const effectiveToolName =
|
|
1131
|
+
toolName === "skill_execute"
|
|
1132
|
+
? resolveSkillExecuteInvocation(input ?? {}, allowedToolNames).name
|
|
1133
|
+
: toolName;
|
|
1134
|
+
return getTool(effectiveToolName)?.executionTarget === "host"
|
|
1135
|
+
? effectiveToolName
|
|
1136
|
+
: null;
|
|
1137
|
+
}
|
|
1138
|
+
|
|
1155
1139
|
// Upper bound on how long a barge-in waits for the interrupted turn's teardown
|
|
1156
1140
|
// to settle before giving up on the continuation. The teardown settles once the
|
|
1157
1141
|
// aborted turn's agent loop reaches its `finally`, which can wait out BOTH the
|
|
@@ -1247,6 +1231,16 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1247
1231
|
// take a turn without the microphone. Governs one thing only: whether a
|
|
1248
1232
|
// missing speech-to-text leg is fatal to startup (see start()).
|
|
1249
1233
|
private readonly textInput: boolean;
|
|
1234
|
+
// The session controls the client declared it can carry out; the only ones
|
|
1235
|
+
// the model is taught and the only ones a reply's marker can trigger.
|
|
1236
|
+
private readonly sessionControls: readonly LiveVoiceSessionControl[];
|
|
1237
|
+
// The client declared `lookFrames`: it sends a fresh frame for every look
|
|
1238
|
+
// control it carries out, so the session answers a look on its own turn and
|
|
1239
|
+
// teaches the model that the reply asking for one is only the acknowledgement.
|
|
1240
|
+
private readonly lookFrames: boolean;
|
|
1241
|
+
// How often progress updates are spoken, as the user last asked out loud.
|
|
1242
|
+
// Session-scoped: it applies from the next turn to the end of the call.
|
|
1243
|
+
private progressCadence: "fewer" | "normal" = "normal";
|
|
1250
1244
|
// Whether this session's speech-to-text leg came up. False only when the
|
|
1251
1245
|
// preflight found it missing and `textInput` let the session open anyway, in
|
|
1252
1246
|
// which case nothing arms a transcriber and typed turns are the only input.
|
|
@@ -1289,22 +1283,17 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1289
1283
|
// assistant turn is audibly speaking: above-gate speech-chunk duration
|
|
1290
1284
|
// accumulates until it reaches bargeInMinSpeechMs, then the deferred
|
|
1291
1285
|
// speech_started + barge-in fire (at most once per onset). Brief sub-threshold
|
|
1292
|
-
// gaps are tolerated (see
|
|
1286
|
+
// gaps are tolerated (see barge-in-guard.ts); the run resets on a
|
|
1293
1287
|
// single longer continuous silence, or once cumulative tolerated silence
|
|
1294
|
-
// exceeds the duty-cycle ceiling (see
|
|
1288
|
+
// exceeds the duty-cycle ceiling (see barge-in-guard.ts).
|
|
1295
1289
|
// The detector's utterance end discards the guard.
|
|
1296
1290
|
private pendingBargeIn: {
|
|
1297
1291
|
// Null when guarding only the post-tts_done drain window (the turn is
|
|
1298
1292
|
// already finalized but the client is still playing its tail).
|
|
1299
1293
|
turn: ActiveAssistantTurn | null;
|
|
1300
|
-
|
|
1301
|
-
//
|
|
1302
|
-
|
|
1303
|
-
silenceMs: number;
|
|
1304
|
-
// Cumulative sub-threshold time over the whole run (not reset by speech
|
|
1305
|
-
// chunks); resets speechMs once it exceeds the duty-cycle ceiling so sparse
|
|
1306
|
-
// periodic blips cannot sum into a barge-in.
|
|
1307
|
-
toleratedSilenceMs: number;
|
|
1294
|
+
// The shared sustained-speech accounting (gap tolerance, duty-cycle
|
|
1295
|
+
// ceiling, threshold).
|
|
1296
|
+
guard: BargeInGuard;
|
|
1308
1297
|
} | null = null;
|
|
1309
1298
|
// Estimated wall-clock ms until the client finishes draining the
|
|
1310
1299
|
// assistant audio sent so far. The server clears the turn right after
|
|
@@ -1336,6 +1325,23 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1336
1325
|
// answer.
|
|
1337
1326
|
private pendingAnnouncement: ContinuationDelivery | null = null;
|
|
1338
1327
|
private announcementTimer: ReturnType<typeof setTimeout> | null = null;
|
|
1328
|
+
// Host-backed work stays on the parent conversation across barge-ins. An
|
|
1329
|
+
// owned task belongs to the tool-capable turn doing the work; a suspended
|
|
1330
|
+
// task is waiting for either a merged escalation or a synthetic resume.
|
|
1331
|
+
private foregroundTaskState: ForegroundTaskState | null = null;
|
|
1332
|
+
private foregroundTaskEpoch = 0;
|
|
1333
|
+
private foregroundTaskResumeTimer: ReturnType<typeof setTimeout> | null =
|
|
1334
|
+
null;
|
|
1335
|
+
private readonly foregroundTaskResumeSilenceMs: number;
|
|
1336
|
+
private readonly foregroundTaskMaxSuspendedMs: number;
|
|
1337
|
+
private readonly foregroundTaskMaxInterveningTurns: number;
|
|
1338
|
+
// A look control sent to a client that declared `lookFrames`, waiting for
|
|
1339
|
+
// the fresh frame the client takes for it. The session answers the look on a
|
|
1340
|
+
// turn of its own once that frame is in the conversation (see
|
|
1341
|
+
// answerLookWhenFloorIsFree). Cleared when the frame lands, when the wait
|
|
1342
|
+
// runs out, and when the session closes.
|
|
1343
|
+
private pendingLook: PendingLook | null = null;
|
|
1344
|
+
private lookFollowUpTimer: ReturnType<typeof setTimeout> | null = null;
|
|
1339
1345
|
// Set when a continuation actually spawns: the request it took over, so the
|
|
1340
1346
|
// NEXT turn can tell the user the work is still running. Consumed by that
|
|
1341
1347
|
// turn; cleared when the continuation finishes (by then the result note
|
|
@@ -1350,6 +1356,10 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1350
1356
|
// the distance from it to a keep arriving is the client leg of the frame the
|
|
1351
1357
|
// onset asked for, measured from the daemon's own clock.
|
|
1352
1358
|
private lastSpeechStartedAtMs: number | null = null;
|
|
1359
|
+
// How many assistant turns have launched, so a look can tell whether one has
|
|
1360
|
+
// started since its frame landed. A count rather than a time: two launches
|
|
1361
|
+
// and a frame can share a millisecond.
|
|
1362
|
+
private turnsLaunched = 0;
|
|
1353
1363
|
private readonly maxPendingAudioBytes: number;
|
|
1354
1364
|
// Set on VAD speech onset; consumed when the first speech chunk is routed
|
|
1355
1365
|
// to an utterance so the metric lands on the right turn.
|
|
@@ -1400,9 +1410,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1400
1410
|
// never emits an answer.
|
|
1401
1411
|
private readonly progressNarrator: VoiceProgressNarrator | null;
|
|
1402
1412
|
// Complete front-model tunables: the constructor schema-parses the partial
|
|
1403
|
-
// option once, so every field carries its `
|
|
1413
|
+
// option once, so every field carries its `voice.frontModel` schema
|
|
1404
1414
|
// default when unset.
|
|
1405
|
-
private readonly frontModelConfig:
|
|
1415
|
+
private readonly frontModelConfig: VoiceFrontModelConfig;
|
|
1406
1416
|
// Complete Flux tunables (the constructor schema-parses the partial option
|
|
1407
1417
|
// once, so every field carries its `liveVoice.flux` schema default).
|
|
1408
1418
|
private readonly fluxConfig: LiveVoiceFluxConfig;
|
|
@@ -1471,6 +1481,14 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1471
1481
|
defaultDetachTeardownSettleTimeoutMs();
|
|
1472
1482
|
this.continuationAnnounceSilenceMs =
|
|
1473
1483
|
options.continuationAnnounceSilenceMs ?? CONTINUATION_ANNOUNCE_SILENCE_MS;
|
|
1484
|
+
this.foregroundTaskResumeSilenceMs =
|
|
1485
|
+
options.foregroundTaskResumeSilenceMs ??
|
|
1486
|
+
FOREGROUND_TASK_RESUME_SILENCE_MS;
|
|
1487
|
+
this.foregroundTaskMaxSuspendedMs =
|
|
1488
|
+
options.foregroundTaskMaxSuspendedMs ?? FOREGROUND_TASK_MAX_SUSPENDED_MS;
|
|
1489
|
+
this.foregroundTaskMaxInterveningTurns =
|
|
1490
|
+
options.foregroundTaskMaxInterveningTurns ??
|
|
1491
|
+
FOREGROUND_TASK_MAX_INTERVENING_TURNS;
|
|
1474
1492
|
this.emitMetrics = options.emitMetrics ?? false;
|
|
1475
1493
|
this.createTurnId = options.createTurnId ?? randomUUID;
|
|
1476
1494
|
this.conversationId =
|
|
@@ -1502,7 +1520,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1502
1520
|
options.echoDrainSlackMs ?? DEFAULT_ECHO_DRAIN_SLACK_MS;
|
|
1503
1521
|
this.finalizeGraceMs = options.finalizeGraceMs ?? FINALIZE_GRACE_MS;
|
|
1504
1522
|
this.progressNarrator = options.progressNarrator ?? null;
|
|
1505
|
-
this.frontModelConfig =
|
|
1523
|
+
this.frontModelConfig = VoiceFrontModelConfigSchema.parse(
|
|
1506
1524
|
options.frontModelConfig ?? {},
|
|
1507
1525
|
);
|
|
1508
1526
|
this.fluxConfig = LiveVoiceFluxConfigSchema.parse(options.fluxConfig ?? {});
|
|
@@ -1526,6 +1544,8 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1526
1544
|
2 *
|
|
1527
1545
|
SERVER_VAD_PENDING_AUDIO_MAX_SECONDS;
|
|
1528
1546
|
this.textInput = context.startFrame.textInput === true;
|
|
1547
|
+
this.sessionControls = context.startFrame.sessionControls ?? [];
|
|
1548
|
+
this.lookFrames = context.startFrame.lookFrames === true;
|
|
1529
1549
|
}
|
|
1530
1550
|
|
|
1531
1551
|
get finalTranscriptText(): string {
|
|
@@ -1774,6 +1794,11 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1774
1794
|
},
|
|
1775
1795
|
"Sight frame timing",
|
|
1776
1796
|
);
|
|
1797
|
+
// Only once the row is in: the turn that answers the look reads the
|
|
1798
|
+
// conversation, so the frame has to be there before that turn starts.
|
|
1799
|
+
if (result.ok && frame.timing?.reason === LOOK_FRAME_REASON) {
|
|
1800
|
+
this.lookFrameLanded();
|
|
1801
|
+
}
|
|
1777
1802
|
if (!result.ok && !this.isClosed) {
|
|
1778
1803
|
void this.sendFrame({
|
|
1779
1804
|
type: "error",
|
|
@@ -1960,6 +1985,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1960
1985
|
|
|
1961
1986
|
const shouldEmitSessionEndMetrics = this.state !== "failed";
|
|
1962
1987
|
this.state = "closed";
|
|
1988
|
+
this.clearForegroundTask("session_closed");
|
|
1963
1989
|
// Retire the island before the teardown below starts awaiting things. A
|
|
1964
1990
|
// close can take a while (a pending continuation is delivered first), and
|
|
1965
1991
|
// an activity left asserting "Speaking…" through it is exactly the stale
|
|
@@ -1979,6 +2005,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1979
2005
|
// cleared: the announcement is dead either way.
|
|
1980
2006
|
await this.deliverPendingContinuationToConversation();
|
|
1981
2007
|
this.clearContinuationAnnouncement();
|
|
2008
|
+
this.clearPendingLook();
|
|
1982
2009
|
this.stopSessionTranscriber();
|
|
1983
2010
|
// Detached continuations outlive the call. A deliberate `interrupt()`
|
|
1984
2011
|
// aborts them; ending the session leaves them running. With no next voice
|
|
@@ -2488,7 +2515,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2488
2515
|
|
|
2489
2516
|
if (this.echoWindowTotalAudioMs === 0) {
|
|
2490
2517
|
this.echoWindowGuardCarryover =
|
|
2491
|
-
this.pendingBargeIn !== null && this.pendingBargeIn.speechMs > 0;
|
|
2518
|
+
this.pendingBargeIn !== null && this.pendingBargeIn.guard.speechMs > 0;
|
|
2492
2519
|
} else if (this.pendingBargeIn === null) {
|
|
2493
2520
|
this.echoWindowGuardCarryover = false;
|
|
2494
2521
|
}
|
|
@@ -2533,7 +2560,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2533
2560
|
);
|
|
2534
2561
|
if (meanAmplitude > speechThreshold) {
|
|
2535
2562
|
const guardHasSpeech =
|
|
2536
|
-
this.pendingBargeIn !== null && this.pendingBargeIn.speechMs > 0;
|
|
2563
|
+
this.pendingBargeIn !== null && this.pendingBargeIn.guard.speechMs > 0;
|
|
2537
2564
|
if (!guardHasSpeech && this.echoMatchesAssistant(chunk)) {
|
|
2538
2565
|
this.updateEchoEnergy(meanAmplitude, chunkMs);
|
|
2539
2566
|
return [{ chunk, classification: "echo" }];
|
|
@@ -2831,9 +2858,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2831
2858
|
// accumulates (trackBargeInGuard), so no speech is lost either way.
|
|
2832
2859
|
this.pendingBargeIn = {
|
|
2833
2860
|
turn: bargeableTurn,
|
|
2834
|
-
|
|
2835
|
-
silenceMs: 0,
|
|
2836
|
-
toleratedSilenceMs: 0,
|
|
2861
|
+
guard: createBargeInGuard(this.bargeInMinSpeechMs),
|
|
2837
2862
|
};
|
|
2838
2863
|
return;
|
|
2839
2864
|
}
|
|
@@ -2857,77 +2882,50 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2857
2882
|
|
|
2858
2883
|
// Advance the sustained-speech barge-in guard by one server-VAD chunk.
|
|
2859
2884
|
// Speech accumulates toward bargeInMinSpeechMs, short true-silence gaps are
|
|
2860
|
-
// tolerated, and classified playback echo resets the run immediately
|
|
2861
|
-
//
|
|
2885
|
+
// tolerated, and classified playback echo resets the run immediately (the
|
|
2886
|
+
// shared guard owns that accounting). A reset also drops the echo state a
|
|
2887
|
+
// run carried over, so the classifier re-learns the playback level.
|
|
2862
2888
|
private trackBargeInGuard(
|
|
2863
2889
|
classification: VadEnergyClassification,
|
|
2864
2890
|
chunk: Buffer,
|
|
2865
2891
|
): void {
|
|
2866
|
-
const
|
|
2867
|
-
if (!
|
|
2892
|
+
const pending = this.pendingBargeIn;
|
|
2893
|
+
if (!pending) {
|
|
2868
2894
|
return;
|
|
2869
2895
|
}
|
|
2870
2896
|
const chunkMs = pcm16DurationMs(
|
|
2871
2897
|
chunk.byteLength,
|
|
2872
2898
|
this.context.startFrame.audio.sampleRate,
|
|
2873
2899
|
);
|
|
2874
|
-
|
|
2875
|
-
|
|
2876
|
-
|
|
2877
|
-
|
|
2878
|
-
|
|
2879
|
-
|
|
2880
|
-
guard.toleratedSilenceMs += chunkMs;
|
|
2881
|
-
// Strictly greater on the per-gap check: a gap of exactly
|
|
2882
|
-
// BARGE_IN_GAP_TOLERANCE_MS is still tolerated. The web client batches PCM
|
|
2883
|
-
// into 50 ms frames, so a run of ducked frames lands on the boundary
|
|
2884
|
-
// exactly (e.g. four frames = 200 ms). The run also resets once its total
|
|
2885
|
-
// tolerated silence outweighs the speech by the duty-cycle ceiling, so
|
|
2886
|
-
// sparse periodic blips can never sum to the guard.
|
|
2887
|
-
if (
|
|
2888
|
-
guard.silenceMs > BARGE_IN_GAP_TOLERANCE_MS ||
|
|
2889
|
-
guard.toleratedSilenceMs >
|
|
2890
|
-
this.bargeInMinSpeechMs * BARGE_IN_MAX_TOLERATED_SILENCE_RATIO
|
|
2891
|
-
) {
|
|
2892
|
-
this.resetBargeInGuardRun();
|
|
2900
|
+
const step = pending.guard.track(classification, chunkMs);
|
|
2901
|
+
if (step === "reset") {
|
|
2902
|
+
if (this.echoWindowGuardCarryover) {
|
|
2903
|
+
this.echoWindowGuardCarryover = false;
|
|
2904
|
+
this.echoEnergyEma = 0;
|
|
2905
|
+
this.echoProbeChunks = [];
|
|
2893
2906
|
}
|
|
2894
2907
|
return;
|
|
2895
2908
|
}
|
|
2896
|
-
|
|
2897
|
-
guard.speechMs += chunkMs;
|
|
2898
|
-
if (guard.speechMs < this.bargeInMinSpeechMs) {
|
|
2909
|
+
if (step !== "fired") {
|
|
2899
2910
|
return;
|
|
2900
2911
|
}
|
|
2901
2912
|
this.pendingBargeIn = null;
|
|
2902
2913
|
this.assistantPlaybackTailUntilMs = 0;
|
|
2903
2914
|
this.sendSpeechStarted();
|
|
2904
|
-
const { turn } =
|
|
2915
|
+
const { turn } = pending;
|
|
2905
2916
|
if (turn && turn === this.activeAssistantTurn && !turn.finalized) {
|
|
2906
2917
|
this.bargeIn(turn);
|
|
2907
2918
|
}
|
|
2908
2919
|
}
|
|
2909
2920
|
|
|
2910
|
-
private resetBargeInGuardRun(): void {
|
|
2911
|
-
const guard = this.pendingBargeIn;
|
|
2912
|
-
if (!guard) {
|
|
2913
|
-
return;
|
|
2914
|
-
}
|
|
2915
|
-
guard.speechMs = 0;
|
|
2916
|
-
guard.silenceMs = 0;
|
|
2917
|
-
guard.toleratedSilenceMs = 0;
|
|
2918
|
-
if (this.echoWindowGuardCarryover) {
|
|
2919
|
-
this.echoWindowGuardCarryover = false;
|
|
2920
|
-
this.echoEnergyEma = 0;
|
|
2921
|
-
this.echoProbeChunks = [];
|
|
2922
|
-
}
|
|
2923
|
-
}
|
|
2924
|
-
|
|
2925
2921
|
private bargeIn(turn: ActiveAssistantTurn): void {
|
|
2926
2922
|
// Abort synchronously so no tts_audio frame can follow turn_cancelled,
|
|
2927
2923
|
// and settle the cancelled turn's metrics so the next utterance's marks
|
|
2928
2924
|
// do not collide with it in the collector. turn_cancelled flushes
|
|
2929
2925
|
// client playback, so the drain estimate resets with it.
|
|
2930
2926
|
this.assistantPlaybackTailUntilMs = 0;
|
|
2927
|
+
const keepForegroundTaskOnParent =
|
|
2928
|
+
!turn.assistantCompleted && this.suspendOwnedForegroundTask(turn);
|
|
2931
2929
|
// Carry the interrupted request into the next turn so it merges with the
|
|
2932
2930
|
// barge-in utterance rather than being answered as a fresh follow-up.
|
|
2933
2931
|
const interruptedRequest = turn.utterance.finalTranscriptSegments
|
|
@@ -3008,6 +3006,13 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3008
3006
|
this.publishActivity(turn, "");
|
|
3009
3007
|
await this.sendFrame({ type: "turn_cancelled", turnId: turn.turnId });
|
|
3010
3008
|
await this.cancelAssistantTurn("barge_in");
|
|
3009
|
+
if (keepForegroundTaskOnParent) {
|
|
3010
|
+
log.info(
|
|
3011
|
+
{ turnId: turn.turnId },
|
|
3012
|
+
"Voice duplex continuation skipped for parent-owned foreground task",
|
|
3013
|
+
);
|
|
3014
|
+
return;
|
|
3015
|
+
}
|
|
3011
3016
|
// Keep the interrupted turn's work alive on a background subagent; the
|
|
3012
3017
|
// detach waits for its teardown to settle the partial into history before
|
|
3013
3018
|
// forking.
|
|
@@ -3053,16 +3058,20 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3053
3058
|
// already finished, so bargeIn returns it to the stash instead.
|
|
3054
3059
|
turn.continuationDelivery !== null
|
|
3055
3060
|
? "announcement_turn"
|
|
3056
|
-
: //
|
|
3057
|
-
//
|
|
3058
|
-
|
|
3059
|
-
|
|
3060
|
-
|
|
3061
|
-
|
|
3062
|
-
//
|
|
3063
|
-
|
|
3064
|
-
? "
|
|
3065
|
-
:
|
|
3061
|
+
: // Nor over the answer to a look: there is no request behind it
|
|
3062
|
+
// either, and the user talking over it is them moving on.
|
|
3063
|
+
turn.lookFollowUp !== null
|
|
3064
|
+
? "look_follow_up"
|
|
3065
|
+
: // The model already finished generating (barge-in during TTS playback
|
|
3066
|
+
// of a complete reply): there is nothing to continue, so a
|
|
3067
|
+
// continuation would just re-do a finished answer.
|
|
3068
|
+
turn.assistantCompleted
|
|
3069
|
+
? "assistant_already_completed"
|
|
3070
|
+
: // A stop (interrupt/close) or a superseding invalidation landed
|
|
3071
|
+
// during the barge-in teardown: honor it.
|
|
3072
|
+
this.detachStopGeneration !== stopGeneration
|
|
3073
|
+
? "invalidated_during_barge_teardown"
|
|
3074
|
+
: null;
|
|
3066
3075
|
if (skipReason !== null) {
|
|
3067
3076
|
log.info(
|
|
3068
3077
|
{ turnId: turn.turnId, skipReason },
|
|
@@ -3281,6 +3290,350 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3281
3290
|
}
|
|
3282
3291
|
}
|
|
3283
3292
|
|
|
3293
|
+
private foregroundTaskRequestForTurn(turn: ActiveAssistantTurn): string {
|
|
3294
|
+
const currentRequest = turn.utterance.finalTranscriptSegments
|
|
3295
|
+
.join(" ")
|
|
3296
|
+
.trim();
|
|
3297
|
+
return currentRequest || turn.interruptedRequest || "";
|
|
3298
|
+
}
|
|
3299
|
+
|
|
3300
|
+
private createForegroundTaskOwnership(
|
|
3301
|
+
turn: ActiveAssistantTurn,
|
|
3302
|
+
hostToolStarted: boolean,
|
|
3303
|
+
): OwnedForegroundTask {
|
|
3304
|
+
const task: OwnedForegroundTask = {
|
|
3305
|
+
phase: "owned",
|
|
3306
|
+
epoch: ++this.foregroundTaskEpoch,
|
|
3307
|
+
request: this.foregroundTaskRequestForTurn(turn),
|
|
3308
|
+
ownerToken: turn.token,
|
|
3309
|
+
hostToolStarted,
|
|
3310
|
+
};
|
|
3311
|
+
turn.foregroundTaskEpoch = task.epoch;
|
|
3312
|
+
this.foregroundTaskState = task;
|
|
3313
|
+
return task;
|
|
3314
|
+
}
|
|
3315
|
+
|
|
3316
|
+
private noteEscalatedForegroundTask(turn: ActiveAssistantTurn): void {
|
|
3317
|
+
if (
|
|
3318
|
+
this.activeAssistantTurn?.token !== turn.token ||
|
|
3319
|
+
turn.discardRequested ||
|
|
3320
|
+
turn.abortController.signal.aborted
|
|
3321
|
+
) {
|
|
3322
|
+
return;
|
|
3323
|
+
}
|
|
3324
|
+
if (
|
|
3325
|
+
this.claimSuspendedForegroundTask(turn) ||
|
|
3326
|
+
this.foregroundTaskState !== null
|
|
3327
|
+
) {
|
|
3328
|
+
return;
|
|
3329
|
+
}
|
|
3330
|
+
|
|
3331
|
+
const task = this.createForegroundTaskOwnership(turn, false);
|
|
3332
|
+
log.info(
|
|
3333
|
+
{ turnId: turn.turnId, epoch: task.epoch },
|
|
3334
|
+
"Escalated live voice turn provisionally owns a foreground task",
|
|
3335
|
+
);
|
|
3336
|
+
}
|
|
3337
|
+
|
|
3338
|
+
private noteHostToolStarted(
|
|
3339
|
+
turn: ActiveAssistantTurn,
|
|
3340
|
+
effectiveToolName: string,
|
|
3341
|
+
): void {
|
|
3342
|
+
const current = this.foregroundTaskState;
|
|
3343
|
+
if (current?.phase === "owned" && current.ownerToken === turn.token) {
|
|
3344
|
+
if (!current.hostToolStarted) {
|
|
3345
|
+
this.foregroundTaskState = { ...current, hostToolStarted: true };
|
|
3346
|
+
log.info(
|
|
3347
|
+
{ turnId: turn.turnId, epoch: current.epoch, effectiveToolName },
|
|
3348
|
+
"Foreground task confirmed by host tool use",
|
|
3349
|
+
);
|
|
3350
|
+
}
|
|
3351
|
+
return;
|
|
3352
|
+
}
|
|
3353
|
+
if (this.claimSuspendedForegroundTask(turn, true)) {
|
|
3354
|
+
return;
|
|
3355
|
+
}
|
|
3356
|
+
|
|
3357
|
+
const task = this.createForegroundTaskOwnership(turn, true);
|
|
3358
|
+
log.info(
|
|
3359
|
+
{ turnId: turn.turnId, epoch: task.epoch, effectiveToolName },
|
|
3360
|
+
"Live voice turn started a foreground task",
|
|
3361
|
+
);
|
|
3362
|
+
}
|
|
3363
|
+
|
|
3364
|
+
private claimSuspendedForegroundTask(
|
|
3365
|
+
turn: ActiveAssistantTurn,
|
|
3366
|
+
hostToolStarted = false,
|
|
3367
|
+
): boolean {
|
|
3368
|
+
const current = this.foregroundTaskState;
|
|
3369
|
+
if (
|
|
3370
|
+
current?.phase !== "suspended" ||
|
|
3371
|
+
turn.foregroundTaskEpoch !== current.epoch ||
|
|
3372
|
+
this.activeAssistantTurn?.token !== turn.token ||
|
|
3373
|
+
turn.discardRequested ||
|
|
3374
|
+
turn.abortController.signal.aborted
|
|
3375
|
+
) {
|
|
3376
|
+
return false;
|
|
3377
|
+
}
|
|
3378
|
+
this.foregroundTaskState = {
|
|
3379
|
+
phase: "owned",
|
|
3380
|
+
epoch: current.epoch,
|
|
3381
|
+
request: current.request,
|
|
3382
|
+
ownerToken: turn.token,
|
|
3383
|
+
hostToolStarted: current.hostToolStarted || hostToolStarted,
|
|
3384
|
+
};
|
|
3385
|
+
this.clearForegroundTaskResumeTimer();
|
|
3386
|
+
log.info(
|
|
3387
|
+
{
|
|
3388
|
+
turnId: turn.turnId,
|
|
3389
|
+
epoch: current.epoch,
|
|
3390
|
+
hostToolStarted: current.hostToolStarted || hostToolStarted,
|
|
3391
|
+
},
|
|
3392
|
+
"Suspended foreground task claimed by tool-capable turn",
|
|
3393
|
+
);
|
|
3394
|
+
return true;
|
|
3395
|
+
}
|
|
3396
|
+
|
|
3397
|
+
private suspendOwnedForegroundTask(turn: ActiveAssistantTurn): boolean {
|
|
3398
|
+
const current = this.foregroundTaskState;
|
|
3399
|
+
if (current?.phase === "owned" && current.ownerToken === turn.token) {
|
|
3400
|
+
this.foregroundTaskState = {
|
|
3401
|
+
phase: "suspended",
|
|
3402
|
+
epoch: current.epoch,
|
|
3403
|
+
request: current.request,
|
|
3404
|
+
suspendedAtMs: Date.now(),
|
|
3405
|
+
interveningTurns: 0,
|
|
3406
|
+
resumePending: false,
|
|
3407
|
+
hostToolStarted: current.hostToolStarted,
|
|
3408
|
+
};
|
|
3409
|
+
this.clearForegroundTaskResumeTimer();
|
|
3410
|
+
log.info(
|
|
3411
|
+
{ turnId: turn.turnId, epoch: current.epoch },
|
|
3412
|
+
"Foreground task suspended by voice barge-in",
|
|
3413
|
+
);
|
|
3414
|
+
return true;
|
|
3415
|
+
}
|
|
3416
|
+
return current !== null;
|
|
3417
|
+
}
|
|
3418
|
+
|
|
3419
|
+
private clearForegroundTask(reason: string): void {
|
|
3420
|
+
const current = this.foregroundTaskState;
|
|
3421
|
+
this.foregroundTaskEpoch += 1;
|
|
3422
|
+
this.foregroundTaskState = null;
|
|
3423
|
+
this.clearForegroundTaskResumeTimer();
|
|
3424
|
+
if (current !== null) {
|
|
3425
|
+
log.info(
|
|
3426
|
+
{
|
|
3427
|
+
epoch: current.epoch,
|
|
3428
|
+
phase: current.phase,
|
|
3429
|
+
hostToolStarted: current.hostToolStarted,
|
|
3430
|
+
reason,
|
|
3431
|
+
},
|
|
3432
|
+
"Live voice foreground task cleared",
|
|
3433
|
+
);
|
|
3434
|
+
}
|
|
3435
|
+
}
|
|
3436
|
+
|
|
3437
|
+
private stopOutstandingInterruptedWork(reason: string): void {
|
|
3438
|
+
this.clearForegroundTask(reason);
|
|
3439
|
+
this.abortDetachedRuns({ reason });
|
|
3440
|
+
}
|
|
3441
|
+
|
|
3442
|
+
private foregroundTaskSuspensionExpired(
|
|
3443
|
+
task: SuspendedForegroundTask,
|
|
3444
|
+
): "turn_limit" | "time_limit" | null {
|
|
3445
|
+
if (task.interveningTurns > this.foregroundTaskMaxInterveningTurns) {
|
|
3446
|
+
return "turn_limit";
|
|
3447
|
+
}
|
|
3448
|
+
if (Date.now() - task.suspendedAtMs >= this.foregroundTaskMaxSuspendedMs) {
|
|
3449
|
+
return "time_limit";
|
|
3450
|
+
}
|
|
3451
|
+
return null;
|
|
3452
|
+
}
|
|
3453
|
+
|
|
3454
|
+
private queueForegroundTaskResume(turn: ActiveAssistantTurn): void {
|
|
3455
|
+
const current = this.foregroundTaskState;
|
|
3456
|
+
if (
|
|
3457
|
+
current?.phase !== "suspended" ||
|
|
3458
|
+
turn.foregroundTaskEpoch !== current.epoch ||
|
|
3459
|
+
turn.hiddenPrompt ||
|
|
3460
|
+
turn.discardRequested ||
|
|
3461
|
+
turn.speculativePending
|
|
3462
|
+
) {
|
|
3463
|
+
return;
|
|
3464
|
+
}
|
|
3465
|
+
const next: SuspendedForegroundTask = {
|
|
3466
|
+
...current,
|
|
3467
|
+
interveningTurns: current.interveningTurns + 1,
|
|
3468
|
+
resumePending: true,
|
|
3469
|
+
};
|
|
3470
|
+
const expired = this.foregroundTaskSuspensionExpired(next);
|
|
3471
|
+
if (expired !== null) {
|
|
3472
|
+
this.clearForegroundTask(`suspended_${expired}`);
|
|
3473
|
+
return;
|
|
3474
|
+
}
|
|
3475
|
+
this.foregroundTaskState = next;
|
|
3476
|
+
if (this.activeAssistantTurn === null) {
|
|
3477
|
+
this.scheduleForegroundTaskResume();
|
|
3478
|
+
}
|
|
3479
|
+
}
|
|
3480
|
+
|
|
3481
|
+
private markForegroundTaskResumePending(epoch: number): boolean {
|
|
3482
|
+
const current = this.foregroundTaskState;
|
|
3483
|
+
if (current?.phase !== "suspended" || current.epoch !== epoch) {
|
|
3484
|
+
return false;
|
|
3485
|
+
}
|
|
3486
|
+
this.foregroundTaskState = { ...current, resumePending: true };
|
|
3487
|
+
return true;
|
|
3488
|
+
}
|
|
3489
|
+
|
|
3490
|
+
private rearmForegroundTaskAfterLegFailure(
|
|
3491
|
+
turn: ActiveAssistantTurn,
|
|
3492
|
+
): boolean {
|
|
3493
|
+
const epoch = turn.foregroundTaskEpoch;
|
|
3494
|
+
if (epoch === null) {
|
|
3495
|
+
return false;
|
|
3496
|
+
}
|
|
3497
|
+
const current = this.foregroundTaskState;
|
|
3498
|
+
if (current?.phase === "suspended" && current.epoch === epoch) {
|
|
3499
|
+
return this.markForegroundTaskResumePending(epoch);
|
|
3500
|
+
}
|
|
3501
|
+
const snapshot = turn.foregroundTaskSuspensionSnapshot;
|
|
3502
|
+
if (
|
|
3503
|
+
current?.phase !== "owned" ||
|
|
3504
|
+
current.epoch !== epoch ||
|
|
3505
|
+
current.ownerToken !== turn.token ||
|
|
3506
|
+
(snapshot !== null && snapshot.epoch !== epoch)
|
|
3507
|
+
) {
|
|
3508
|
+
return false;
|
|
3509
|
+
}
|
|
3510
|
+
const next: SuspendedForegroundTask = {
|
|
3511
|
+
...(snapshot ?? {
|
|
3512
|
+
phase: "suspended",
|
|
3513
|
+
epoch,
|
|
3514
|
+
request: current.request,
|
|
3515
|
+
suspendedAtMs: Date.now(),
|
|
3516
|
+
interveningTurns: 0,
|
|
3517
|
+
hostToolStarted: current.hostToolStarted,
|
|
3518
|
+
}),
|
|
3519
|
+
resumePending: true,
|
|
3520
|
+
};
|
|
3521
|
+
const expired = this.foregroundTaskSuspensionExpired(next);
|
|
3522
|
+
if (expired !== null) {
|
|
3523
|
+
this.clearForegroundTask(`suspended_${expired}`);
|
|
3524
|
+
return false;
|
|
3525
|
+
}
|
|
3526
|
+
this.foregroundTaskState = next;
|
|
3527
|
+
log.info(
|
|
3528
|
+
{ turnId: turn.turnId, epoch },
|
|
3529
|
+
"Foreground task re-armed after tool-capable leg failure",
|
|
3530
|
+
);
|
|
3531
|
+
return true;
|
|
3532
|
+
}
|
|
3533
|
+
|
|
3534
|
+
private clearForegroundTaskResumeTimer(): void {
|
|
3535
|
+
if (this.foregroundTaskResumeTimer !== null) {
|
|
3536
|
+
clearTimeout(this.foregroundTaskResumeTimer);
|
|
3537
|
+
this.foregroundTaskResumeTimer = null;
|
|
3538
|
+
}
|
|
3539
|
+
}
|
|
3540
|
+
|
|
3541
|
+
private scheduleFloorCheck(
|
|
3542
|
+
currentTimer: ReturnType<typeof setTimeout> | null,
|
|
3543
|
+
silenceMs: number,
|
|
3544
|
+
check: (blockedBy: string | null) => void,
|
|
3545
|
+
): ReturnType<typeof setTimeout> {
|
|
3546
|
+
if (currentTimer !== null) {
|
|
3547
|
+
clearTimeout(currentTimer);
|
|
3548
|
+
}
|
|
3549
|
+
const drainMs = Math.max(0, this.assistantPlaybackTailUntilMs - Date.now());
|
|
3550
|
+
return setTimeout(
|
|
3551
|
+
() => check(this.sessionTurnFloorBlocker()),
|
|
3552
|
+
drainMs + silenceMs,
|
|
3553
|
+
);
|
|
3554
|
+
}
|
|
3555
|
+
|
|
3556
|
+
private scheduleForegroundTaskResume(): void {
|
|
3557
|
+
const task = this.foregroundTaskState;
|
|
3558
|
+
if (
|
|
3559
|
+
task?.phase !== "suspended" ||
|
|
3560
|
+
!task.resumePending ||
|
|
3561
|
+
this.isClosed ||
|
|
3562
|
+
this.state === "failed"
|
|
3563
|
+
) {
|
|
3564
|
+
this.clearForegroundTaskResumeTimer();
|
|
3565
|
+
return;
|
|
3566
|
+
}
|
|
3567
|
+
const expired = this.foregroundTaskSuspensionExpired(task);
|
|
3568
|
+
if (expired !== null) {
|
|
3569
|
+
this.clearForegroundTask(`suspended_${expired}`);
|
|
3570
|
+
return;
|
|
3571
|
+
}
|
|
3572
|
+
const epoch = task.epoch;
|
|
3573
|
+
this.foregroundTaskResumeTimer = this.scheduleFloorCheck(
|
|
3574
|
+
this.foregroundTaskResumeTimer,
|
|
3575
|
+
this.foregroundTaskResumeSilenceMs,
|
|
3576
|
+
(blockedBy) => {
|
|
3577
|
+
this.foregroundTaskResumeTimer = null;
|
|
3578
|
+
const current = this.foregroundTaskState;
|
|
3579
|
+
if (
|
|
3580
|
+
current?.phase !== "suspended" ||
|
|
3581
|
+
current.epoch !== epoch ||
|
|
3582
|
+
!current.resumePending
|
|
3583
|
+
) {
|
|
3584
|
+
return;
|
|
3585
|
+
}
|
|
3586
|
+
const currentExpired = this.foregroundTaskSuspensionExpired(current);
|
|
3587
|
+
if (currentExpired !== null) {
|
|
3588
|
+
this.clearForegroundTask(`suspended_${currentExpired}`);
|
|
3589
|
+
return;
|
|
3590
|
+
}
|
|
3591
|
+
if (blockedBy === null) {
|
|
3592
|
+
void this.resumeForegroundTask(current).catch((err: unknown) => {
|
|
3593
|
+
log.warn({ err, epoch }, "Foreground task resume failed");
|
|
3594
|
+
});
|
|
3595
|
+
return;
|
|
3596
|
+
}
|
|
3597
|
+
if (blockedBy === "session_unavailable") {
|
|
3598
|
+
this.clearForegroundTask("session_unavailable");
|
|
3599
|
+
return;
|
|
3600
|
+
}
|
|
3601
|
+
if (blockedBy !== "turn_active") {
|
|
3602
|
+
this.scheduleForegroundTaskResume();
|
|
3603
|
+
}
|
|
3604
|
+
},
|
|
3605
|
+
);
|
|
3606
|
+
}
|
|
3607
|
+
|
|
3608
|
+
private async resumeForegroundTask(
|
|
3609
|
+
task: SuspendedForegroundTask,
|
|
3610
|
+
): Promise<void> {
|
|
3611
|
+
const current = this.foregroundTaskState;
|
|
3612
|
+
if (
|
|
3613
|
+
current?.phase !== "suspended" ||
|
|
3614
|
+
current.epoch !== task.epoch ||
|
|
3615
|
+
!current.resumePending
|
|
3616
|
+
) {
|
|
3617
|
+
return;
|
|
3618
|
+
}
|
|
3619
|
+
this.foregroundTaskState = { ...current, resumePending: false };
|
|
3620
|
+
const started = await this.launchAssistantTurn(
|
|
3621
|
+
createSyntheticUtterance(),
|
|
3622
|
+
buildForegroundTaskResumePrompt(current.request),
|
|
3623
|
+
{
|
|
3624
|
+
hiddenPrompt: true,
|
|
3625
|
+
initialLeg: "escalated",
|
|
3626
|
+
foregroundTaskEpoch: current.epoch,
|
|
3627
|
+
},
|
|
3628
|
+
);
|
|
3629
|
+
if (started) {
|
|
3630
|
+
return;
|
|
3631
|
+
}
|
|
3632
|
+
if (this.markForegroundTaskResumePending(current.epoch)) {
|
|
3633
|
+
this.scheduleForegroundTaskResume();
|
|
3634
|
+
}
|
|
3635
|
+
}
|
|
3636
|
+
|
|
3284
3637
|
// Abort every background continuation this session started and drop its
|
|
3285
3638
|
// handle. Called on a hard stop (client interrupt / session close), when a
|
|
3286
3639
|
// newer barge-in supersedes the detached runs, and on a foreground-wins
|
|
@@ -3407,50 +3760,53 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3407
3760
|
retry = false,
|
|
3408
3761
|
drainRearms = 0,
|
|
3409
3762
|
): void {
|
|
3410
|
-
|
|
3411
|
-
|
|
3412
|
-
|
|
3413
|
-
|
|
3414
|
-
|
|
3415
|
-
|
|
3416
|
-
|
|
3417
|
-
|
|
3418
|
-
|
|
3419
|
-
|
|
3420
|
-
|
|
3421
|
-
|
|
3422
|
-
|
|
3423
|
-
|
|
3424
|
-
|
|
3425
|
-
|
|
3426
|
-
|
|
3427
|
-
|
|
3428
|
-
|
|
3429
|
-
|
|
3430
|
-
|
|
3431
|
-
|
|
3432
|
-
|
|
3433
|
-
|
|
3434
|
-
|
|
3435
|
-
|
|
3436
|
-
|
|
3437
|
-
|
|
3438
|
-
|
|
3439
|
-
|
|
3440
|
-
|
|
3441
|
-
|
|
3442
|
-
|
|
3443
|
-
|
|
3444
|
-
|
|
3445
|
-
|
|
3446
|
-
|
|
3447
|
-
|
|
3448
|
-
|
|
3449
|
-
|
|
3450
|
-
|
|
3451
|
-
|
|
3452
|
-
|
|
3453
|
-
|
|
3763
|
+
this.announcementTimer = this.scheduleFloorCheck(
|
|
3764
|
+
this.announcementTimer,
|
|
3765
|
+
this.continuationAnnounceSilenceMs,
|
|
3766
|
+
() => {
|
|
3767
|
+
this.announcementTimer = null;
|
|
3768
|
+
const blockedBy = this.continuationAnnouncementBlocker();
|
|
3769
|
+
if (blockedBy === null) {
|
|
3770
|
+
void this.announceContinuation().catch((err: unknown) => {
|
|
3771
|
+
log.warn({ err }, "Voice duplex continuation announcement failed");
|
|
3772
|
+
});
|
|
3773
|
+
return;
|
|
3774
|
+
}
|
|
3775
|
+
log.debug(
|
|
3776
|
+
{
|
|
3777
|
+
conversationId: this.conversationId,
|
|
3778
|
+
blockedBy,
|
|
3779
|
+
retry,
|
|
3780
|
+
drainRearms,
|
|
3781
|
+
},
|
|
3782
|
+
"Voice duplex continuation announcement deferred",
|
|
3783
|
+
);
|
|
3784
|
+
// A tail that grew while the timer ran (the reply was still streaming
|
|
3785
|
+
// TTS when it was armed) is a wait of known length rather than a busy
|
|
3786
|
+
// call, so it re-arms against the new deadline without spending the
|
|
3787
|
+
// single retry the busy-call blockers get. Otherwise a long reply's
|
|
3788
|
+
// playback alone outlasts the budget and the announcement is dropped
|
|
3789
|
+
// even though the call goes idle right after. The tail only extends
|
|
3790
|
+
// while a turn is active, which blocks earlier, so the re-arms converge;
|
|
3791
|
+
// the cap bounds the interleaving where a fresh turn's audio lands
|
|
3792
|
+
// between two checks.
|
|
3793
|
+
if (
|
|
3794
|
+
blockedBy === "playback_draining" &&
|
|
3795
|
+
drainRearms < CONTINUATION_ANNOUNCE_MAX_DRAIN_REARMS
|
|
3796
|
+
) {
|
|
3797
|
+
this.scheduleContinuationAnnouncement(retry, drainRearms + 1);
|
|
3798
|
+
return;
|
|
3799
|
+
}
|
|
3800
|
+
if (blockedBy === "turn_active") {
|
|
3801
|
+
return;
|
|
3802
|
+
}
|
|
3803
|
+
if (retry || this.pendingAnnouncement === null) {
|
|
3804
|
+
this.pendingAnnouncement = null;
|
|
3805
|
+
return;
|
|
3806
|
+
}
|
|
3807
|
+
this.scheduleContinuationAnnouncement(true, drainRearms);
|
|
3808
|
+
},
|
|
3809
|
+
);
|
|
3454
3810
|
}
|
|
3455
3811
|
|
|
3456
3812
|
/**
|
|
@@ -3561,11 +3917,15 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3561
3917
|
// down. One line, not narration: the turn is not working, it is waiting,
|
|
3562
3918
|
// and it says which, in the turn's spoken language, like every other
|
|
3563
3919
|
// filler phrase.
|
|
3564
|
-
|
|
3565
|
-
|
|
3566
|
-
|
|
3567
|
-
|
|
3568
|
-
|
|
3920
|
+
if (
|
|
3921
|
+
this.enqueueFillerPhrase(
|
|
3922
|
+
turn,
|
|
3923
|
+
approvalPendingPhraseFor(turn.language),
|
|
3924
|
+
this.fixedPhraseLanguage(turn, APPROVAL_PENDING_PHRASE_BY_LANGUAGE),
|
|
3925
|
+
)
|
|
3926
|
+
) {
|
|
3927
|
+
turn.progress.noteFloorHolder();
|
|
3928
|
+
}
|
|
3569
3929
|
}
|
|
3570
3930
|
|
|
3571
3931
|
/** Clear the wait once a decision lands, so the turn narrates normally again. */
|
|
@@ -3636,6 +3996,14 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3636
3996
|
) {
|
|
3637
3997
|
this.scheduleContinuationAnnouncement();
|
|
3638
3998
|
}
|
|
3999
|
+
if (
|
|
4000
|
+
this.foregroundTaskState?.phase === "suspended" &&
|
|
4001
|
+
this.foregroundTaskState.resumePending &&
|
|
4002
|
+
!this.isClosed &&
|
|
4003
|
+
this.state !== "failed"
|
|
4004
|
+
) {
|
|
4005
|
+
this.scheduleForegroundTaskResume();
|
|
4006
|
+
}
|
|
3639
4007
|
}
|
|
3640
4008
|
|
|
3641
4009
|
// Why the session must not speak up right now, or null when the call is
|
|
@@ -3705,6 +4073,10 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3705
4073
|
// own it leaves manual mode blind to a user who is talking right now and
|
|
3706
4074
|
// has no text yet — hence the captured-audio flag, which manual ingress
|
|
3707
4075
|
// sets from the first chunk.
|
|
4076
|
+
//
|
|
4077
|
+
// A partial counts only when it has words. Deepgram Flux sends interim
|
|
4078
|
+
// updates through silence too, each an empty partial, and one of those
|
|
4079
|
+
// would otherwise hold the floor until the user next spoke.
|
|
3708
4080
|
if (
|
|
3709
4081
|
utterance !== null &&
|
|
3710
4082
|
!utterance.completed &&
|
|
@@ -3712,13 +4084,129 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3712
4084
|
utterance.assistantTurnStarted ||
|
|
3713
4085
|
(utterance.manualAudioCaptured && opts?.ignoreManualCapture !== true) ||
|
|
3714
4086
|
utterance.finalTranscriptSegments.length > 0 ||
|
|
3715
|
-
utterance.latestPartialText
|
|
4087
|
+
(utterance.latestPartialText?.trim() ?? "").length > 0)
|
|
3716
4088
|
) {
|
|
3717
4089
|
return "utterance_in_flight";
|
|
3718
4090
|
}
|
|
3719
4091
|
return null;
|
|
3720
4092
|
}
|
|
3721
4093
|
|
|
4094
|
+
/**
|
|
4095
|
+
* Wait for the fresh frame a look control asks the client for.
|
|
4096
|
+
*
|
|
4097
|
+
* Only for a client that declared `lookFrames`: any other sends no such
|
|
4098
|
+
* frame, and its look keeps the old shape, where the user's next words are
|
|
4099
|
+
* what the frame gets answered on. A newer look replaces an older one still
|
|
4100
|
+
* waiting, and the wait is bounded so a frame that never comes (a camera
|
|
4101
|
+
* that would not open, a share the desktop refused) cannot turn up minutes
|
|
4102
|
+
* later as a reply to nothing.
|
|
4103
|
+
*/
|
|
4104
|
+
private awaitLookFrame(action: LookSessionControl): void {
|
|
4105
|
+
if (!this.lookFrames) {
|
|
4106
|
+
return;
|
|
4107
|
+
}
|
|
4108
|
+
this.clearPendingLook();
|
|
4109
|
+
const timer = setTimeout(() => {
|
|
4110
|
+
if (this.pendingLook?.timer !== timer) {
|
|
4111
|
+
return;
|
|
4112
|
+
}
|
|
4113
|
+
this.pendingLook = null;
|
|
4114
|
+
log.info(
|
|
4115
|
+
{ conversationId: this.conversationId, action },
|
|
4116
|
+
"Live voice look dropped: no frame arrived",
|
|
4117
|
+
);
|
|
4118
|
+
}, LOOK_FRAME_WAIT_MS);
|
|
4119
|
+
this.pendingLook = { action, armedAtMs: Date.now(), timer };
|
|
4120
|
+
}
|
|
4121
|
+
|
|
4122
|
+
private clearPendingLook(): void {
|
|
4123
|
+
if (this.pendingLook !== null) {
|
|
4124
|
+
clearTimeout(this.pendingLook.timer);
|
|
4125
|
+
this.pendingLook = null;
|
|
4126
|
+
}
|
|
4127
|
+
if (this.lookFollowUpTimer !== null) {
|
|
4128
|
+
clearTimeout(this.lookFollowUpTimer);
|
|
4129
|
+
this.lookFollowUpTimer = null;
|
|
4130
|
+
}
|
|
4131
|
+
}
|
|
4132
|
+
|
|
4133
|
+
/**
|
|
4134
|
+
* The frame a look asked for is in the conversation: answer the look.
|
|
4135
|
+
*
|
|
4136
|
+
* A look frame with no look waiting (the wait ran out, or the session never
|
|
4137
|
+
* armed one) is only a frame, like any other keep.
|
|
4138
|
+
*/
|
|
4139
|
+
private lookFrameLanded(): void {
|
|
4140
|
+
const pending = this.pendingLook;
|
|
4141
|
+
if (pending === null || this.isClosed) {
|
|
4142
|
+
return;
|
|
4143
|
+
}
|
|
4144
|
+
clearTimeout(pending.timer);
|
|
4145
|
+
this.pendingLook = null;
|
|
4146
|
+
this.answerLookWhenFloorIsFree(pending, this.turnsLaunched, 0);
|
|
4147
|
+
}
|
|
4148
|
+
|
|
4149
|
+
/**
|
|
4150
|
+
* Start the turn that answers a look, once nothing else holds the floor.
|
|
4151
|
+
*
|
|
4152
|
+
* The acknowledgement ("taking a look") can still be playing when the frame
|
|
4153
|
+
* lands, and the look's own turn can still be clearing, so both are waited
|
|
4154
|
+
* out, as is the user mid-utterance. A turn that starts once the frame is in
|
|
4155
|
+
* the conversation is not: it reads the frame, so answering the look as well
|
|
4156
|
+
* would answer it twice. A turn that started before the frame landed did not
|
|
4157
|
+
* see it, so the look is still answered once that turn is done.
|
|
4158
|
+
*
|
|
4159
|
+
* `turnsAtFrame` is how many turns had launched when the frame landed.
|
|
4160
|
+
*/
|
|
4161
|
+
private answerLookWhenFloorIsFree(
|
|
4162
|
+
look: PendingLook,
|
|
4163
|
+
turnsAtFrame: number,
|
|
4164
|
+
rearms: number,
|
|
4165
|
+
): void {
|
|
4166
|
+
if (this.lookFollowUpTimer !== null) {
|
|
4167
|
+
clearTimeout(this.lookFollowUpTimer);
|
|
4168
|
+
this.lookFollowUpTimer = null;
|
|
4169
|
+
}
|
|
4170
|
+
const { action, armedAtMs } = look;
|
|
4171
|
+
// A turn launched since the frame landed read it already. Speech that
|
|
4172
|
+
// never became a turn (a cough, noise that transcribed to nothing) is only
|
|
4173
|
+
// waited out.
|
|
4174
|
+
const blockedBy =
|
|
4175
|
+
this.turnsLaunched > turnsAtFrame
|
|
4176
|
+
? "turn_since_look"
|
|
4177
|
+
: this.sessionTurnFloorBlocker();
|
|
4178
|
+
if (blockedBy === null) {
|
|
4179
|
+
void this.launchAssistantTurn(
|
|
4180
|
+
createSyntheticUtterance(),
|
|
4181
|
+
LOOK_FOLLOW_UP_CONTENT,
|
|
4182
|
+
{ lookFollowUp: action, hiddenPrompt: true },
|
|
4183
|
+
).catch((err: unknown) => {
|
|
4184
|
+
log.warn(
|
|
4185
|
+
{ err, conversationId: this.conversationId, action },
|
|
4186
|
+
"Live voice look follow-up failed to start",
|
|
4187
|
+
);
|
|
4188
|
+
});
|
|
4189
|
+
return;
|
|
4190
|
+
}
|
|
4191
|
+
const waitable =
|
|
4192
|
+
blockedBy !== "turn_since_look" && blockedBy !== "session_unavailable";
|
|
4193
|
+
if (!waitable || Date.now() - armedAtMs >= LOOK_ANSWER_DEADLINE_MS) {
|
|
4194
|
+
log.info(
|
|
4195
|
+
{ conversationId: this.conversationId, action, blockedBy, rearms },
|
|
4196
|
+
"Live voice look follow-up skipped",
|
|
4197
|
+
);
|
|
4198
|
+
return;
|
|
4199
|
+
}
|
|
4200
|
+
const drainMs = Math.max(0, this.assistantPlaybackTailUntilMs - Date.now());
|
|
4201
|
+
this.lookFollowUpTimer = setTimeout(
|
|
4202
|
+
() => {
|
|
4203
|
+
this.lookFollowUpTimer = null;
|
|
4204
|
+
this.answerLookWhenFloorIsFree(look, turnsAtFrame, rearms + 1);
|
|
4205
|
+
},
|
|
4206
|
+
Math.max(drainMs, LOOK_FOLLOW_UP_REARM_MS),
|
|
4207
|
+
);
|
|
4208
|
+
}
|
|
4209
|
+
|
|
3722
4210
|
// Speak the finished continuation's result on a turn the session starts
|
|
3723
4211
|
// itself. The turn rides a synthetic cycle rather than the armed VAD one —
|
|
3724
4212
|
// `currentUtterance` belongs to the capture loop, and overwriting it would
|
|
@@ -4172,13 +4660,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
4172
4660
|
if (!alreadyReleased) {
|
|
4173
4661
|
void this.releaseUtterance();
|
|
4174
4662
|
}
|
|
4175
|
-
|
|
4176
|
-
this.frontModelConfig.progress.enabled &&
|
|
4177
|
-
this.streamTtsAudio &&
|
|
4178
|
-
this.progressNarrator
|
|
4179
|
-
) {
|
|
4180
|
-
this.armProgressIdleTimer(turn);
|
|
4181
|
-
}
|
|
4663
|
+
turn.progress.arm();
|
|
4182
4664
|
return true;
|
|
4183
4665
|
}
|
|
4184
4666
|
|
|
@@ -4914,8 +5396,12 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
4914
5396
|
// the next turn is now stale (the interrupted utterance may be discarded
|
|
4915
5397
|
// without ever reaching finalizePendingUtterance).
|
|
4916
5398
|
this.pendingInterruptedRequest = null;
|
|
5399
|
+
this.clearForegroundTask("client_interrupt");
|
|
4917
5400
|
// ...and it hard-stops any detached background continuations.
|
|
4918
5401
|
this.abortDetachedRuns({ reason: "client_interrupt" });
|
|
5402
|
+
// ...and a look still waiting to be answered: the user stopped the call
|
|
5403
|
+
// talking, and a reply starting on its own a moment later is not a stop.
|
|
5404
|
+
this.clearPendingLook();
|
|
4919
5405
|
const utterance = this.currentUtterance;
|
|
4920
5406
|
this.stopSessionTranscriber();
|
|
4921
5407
|
if (utterance) {
|
|
@@ -5137,6 +5623,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5137
5623
|
// Set on an announcement turn: the finished continuation this turn exists
|
|
5138
5624
|
// to deliver. Its answer goes in the control prompt, not in `content`.
|
|
5139
5625
|
continuationDelivery?: ContinuationDelivery | null;
|
|
5626
|
+
// Set on the turn that answers a look: which look. Its instruction goes
|
|
5627
|
+
// in the control prompt, not in `content`.
|
|
5628
|
+
lookFollowUp?: LookSessionControl;
|
|
5140
5629
|
// Unified front-door: dispatch without releasing the utterance. The
|
|
5141
5630
|
// thinking frame and floor-holding timers are deferred until the leg's
|
|
5142
5631
|
// leading verdict commits the turn (see commitSpeculativeTurn); a hold
|
|
@@ -5146,6 +5635,11 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5146
5635
|
// the row persists and drives the turn but never renders: no echo, and
|
|
5147
5636
|
// `/messages` filters it after a reload.
|
|
5148
5637
|
hiddenPrompt?: boolean;
|
|
5638
|
+
// Synthetic host-task resumes start on the tool-capable profile without
|
|
5639
|
+
// passing through the front door.
|
|
5640
|
+
initialLeg?: VoiceRoutingLeg;
|
|
5641
|
+
// The suspended foreground task this synthetic turn resumes.
|
|
5642
|
+
foregroundTaskEpoch?: number;
|
|
5149
5643
|
},
|
|
5150
5644
|
): Promise<boolean> {
|
|
5151
5645
|
utterance.assistantTurnStarted = true;
|
|
@@ -5161,6 +5655,15 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5161
5655
|
this.startMetricsTurnIfNeeded(utterance, turnId);
|
|
5162
5656
|
this.markAssistantDispatch(utterance, turnId);
|
|
5163
5657
|
const abortController = new AbortController();
|
|
5658
|
+
const suspendedForegroundTask =
|
|
5659
|
+
this.foregroundTaskState?.phase === "suspended"
|
|
5660
|
+
? this.foregroundTaskState
|
|
5661
|
+
: null;
|
|
5662
|
+
const foregroundTaskEpoch =
|
|
5663
|
+
opts?.foregroundTaskEpoch ??
|
|
5664
|
+
(opts?.hiddenPrompt !== true
|
|
5665
|
+
? (suspendedForegroundTask?.epoch ?? null)
|
|
5666
|
+
: null);
|
|
5164
5667
|
const activeTurn: ActiveAssistantTurn = {
|
|
5165
5668
|
token,
|
|
5166
5669
|
turnId,
|
|
@@ -5169,23 +5672,46 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5169
5672
|
abortController,
|
|
5170
5673
|
handle: null,
|
|
5171
5674
|
launchedAtMs: Date.now(),
|
|
5172
|
-
progress: {
|
|
5173
|
-
|
|
5174
|
-
|
|
5175
|
-
|
|
5176
|
-
|
|
5177
|
-
//
|
|
5178
|
-
|
|
5179
|
-
|
|
5180
|
-
|
|
5181
|
-
|
|
5182
|
-
|
|
5183
|
-
|
|
5184
|
-
|
|
5185
|
-
|
|
5675
|
+
progress: createProgressCadence({
|
|
5676
|
+
config: progressConfigForCadence(
|
|
5677
|
+
this.frontModelConfig.progress,
|
|
5678
|
+
this.progressCadence,
|
|
5679
|
+
),
|
|
5680
|
+
// Without TTS there is nothing to speak (the idle trigger's static
|
|
5681
|
+
// fallback still needs a generation attempt to fall back from).
|
|
5682
|
+
narrator: this.streamTtsAudio ? this.progressNarrator : null,
|
|
5683
|
+
nextFallbackPhraseIndex: () => this.progressPhraseCounter++,
|
|
5684
|
+
host: {
|
|
5685
|
+
turnId,
|
|
5686
|
+
launchedAtMs: Date.now(),
|
|
5687
|
+
signal: abortController.signal,
|
|
5688
|
+
// Narration may speak whenever the live turn is audibly silent and
|
|
5689
|
+
// has not completed. Escalated turns qualify too: the bridge phrase
|
|
5690
|
+
// holds the floor only while its audio plays (covered by the
|
|
5691
|
+
// audible-idle check), and the strong leg's tool loops are the
|
|
5692
|
+
// longest dead air in the system. Nothing is in flight while a
|
|
5693
|
+
// decision is pending, so every phrase narration has would be a lie
|
|
5694
|
+
// about who the call is waiting on: the turn says so once, when it
|
|
5695
|
+
// starts waiting, and is quiet after that.
|
|
5696
|
+
canNarrate: () =>
|
|
5697
|
+
this.isActiveAssistantTurn(token) &&
|
|
5698
|
+
!activeTurn.assistantCompleted &&
|
|
5699
|
+
activeTurn.pendingApproval === null,
|
|
5700
|
+
isAudioIdle: () => this.turnAudioIdle(activeTurn),
|
|
5701
|
+
playbackTailUntilMs: () => this.assistantPlaybackTailUntilMs,
|
|
5702
|
+
transcriptSoFar: () =>
|
|
5703
|
+
activeTurn.utterance.finalTranscriptSegments.join(" ").trim(),
|
|
5704
|
+
language: () => activeTurn.language,
|
|
5705
|
+
deltaEpoch: () => activeTurn.deltaEpoch,
|
|
5706
|
+
speak: (phrase, language) =>
|
|
5707
|
+
this.enqueueFillerPhrase(activeTurn, phrase, language),
|
|
5708
|
+
onNarrationSpoken: () => this.markProgressSpoken(activeTurn),
|
|
5709
|
+
},
|
|
5710
|
+
}),
|
|
5186
5711
|
assistantCompleted: false,
|
|
5187
5712
|
ttsDone: false,
|
|
5188
5713
|
minimizeRequested: false,
|
|
5714
|
+
sessionControlRequested: null,
|
|
5189
5715
|
activityLabel: "",
|
|
5190
5716
|
publishedApprovalRequestId: null,
|
|
5191
5717
|
pendingApproval: null,
|
|
@@ -5208,9 +5734,15 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5208
5734
|
consumedAnnouncement: pending?.announcement ?? null,
|
|
5209
5735
|
pendingContextStopGeneration: this.detachStopGeneration,
|
|
5210
5736
|
continuationDelivery: opts?.continuationDelivery ?? null,
|
|
5737
|
+
lookFollowUp: opts?.lookFollowUp ?? null,
|
|
5211
5738
|
hiddenPrompt: opts?.hiddenPrompt === true,
|
|
5739
|
+
foregroundTaskEpoch,
|
|
5740
|
+
foregroundTaskSuspensionSnapshot:
|
|
5741
|
+
suspendedForegroundTask?.epoch === foregroundTaskEpoch
|
|
5742
|
+
? { ...suspendedForegroundTask }
|
|
5743
|
+
: null,
|
|
5212
5744
|
deltaEpoch: 0,
|
|
5213
|
-
|
|
5745
|
+
frontDoor: null,
|
|
5214
5746
|
ttsBuffer: "",
|
|
5215
5747
|
spokenSegments: 0,
|
|
5216
5748
|
spokenChars: 0,
|
|
@@ -5223,6 +5755,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5223
5755
|
assistantAudioMimeType: "audio/pcm",
|
|
5224
5756
|
};
|
|
5225
5757
|
this.activeAssistantTurn = activeTurn;
|
|
5758
|
+
this.turnsLaunched += 1;
|
|
5226
5759
|
|
|
5227
5760
|
// A speculative turn defers the thinking frame and both floor-holding
|
|
5228
5761
|
// timers to commitSpeculativeTurn: until the verdict arrives, the pause
|
|
@@ -5268,22 +5801,17 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5268
5801
|
// cadence, wherever in the turn it occurs; without TTS or a narrator there
|
|
5269
5802
|
// is nothing to speak (the idle trigger's static fallback still needs a
|
|
5270
5803
|
// generation attempt to fall back from).
|
|
5271
|
-
if (
|
|
5272
|
-
|
|
5273
|
-
this.streamTtsAudio &&
|
|
5274
|
-
this.progressNarrator &&
|
|
5275
|
-
!opts?.speculative
|
|
5276
|
-
) {
|
|
5277
|
-
this.armProgressIdleTimer(activeTurn);
|
|
5804
|
+
if (!opts?.speculative) {
|
|
5805
|
+
activeTurn.progress.arm();
|
|
5278
5806
|
}
|
|
5279
5807
|
|
|
5280
|
-
|
|
5281
|
-
// call site pins it) and may hand off to a quality leg on the escalate
|
|
5282
|
-
// verdict.
|
|
5808
|
+
const initialLeg = opts?.initialLeg ?? "front-door";
|
|
5283
5809
|
const started = await this.startAssistantLeg(activeTurn, {
|
|
5284
5810
|
content,
|
|
5285
|
-
routingLeg:
|
|
5286
|
-
|
|
5811
|
+
routingLeg: initialLeg,
|
|
5812
|
+
...(initialLeg === "front-door"
|
|
5813
|
+
? { frontDoor: true }
|
|
5814
|
+
: { directEscalated: true }),
|
|
5287
5815
|
// Only this leg: an escalated leg persists its own continuation row, and
|
|
5288
5816
|
// the frame belongs to the row carrying the user's words.
|
|
5289
5817
|
...(activeTurn.turnAttachmentId
|
|
@@ -5328,6 +5856,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5328
5856
|
overrideProfile?: string;
|
|
5329
5857
|
routingLeg?: VoiceRoutingLeg;
|
|
5330
5858
|
frontDoor?: boolean;
|
|
5859
|
+
directEscalated?: boolean;
|
|
5331
5860
|
spokenEscalationBridge?: string;
|
|
5332
5861
|
attachments?: readonly string[];
|
|
5333
5862
|
},
|
|
@@ -5337,17 +5866,15 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5337
5866
|
}
|
|
5338
5867
|
const { token, utterance, turnId } = activeTurn;
|
|
5339
5868
|
|
|
5340
|
-
// `rawText` accumulates this leg's
|
|
5341
|
-
//
|
|
5342
|
-
//
|
|
5343
|
-
// escalation buffers the post-verdict stream
|
|
5344
|
-
// bridge is complete, then hands off. A default
|
|
5345
|
-
// every delta through the same holdback, so a
|
|
5346
|
-
// the main model is stripped instead of
|
|
5869
|
+
// `rawText` accumulates this leg's spoken stream. A front-door leg runs
|
|
5870
|
+
// through the shared coordinator, which reads its stream through the
|
|
5871
|
+
// verdict machine: released answer text flushes through the shared
|
|
5872
|
+
// marker holdback, while an escalation buffers the post-verdict stream
|
|
5873
|
+
// until the capped bridge is complete, then hands off. A default or
|
|
5874
|
+
// escalated leg flushes every delta through the same holdback, so a
|
|
5875
|
+
// stray control marker from the main model is stripped instead of
|
|
5876
|
+
// spoken.
|
|
5347
5877
|
let rawText = "";
|
|
5348
|
-
let frontDoorStage: "deciding" | "answer" | "bridging" | "handedOff" =
|
|
5349
|
-
"deciding";
|
|
5350
|
-
let bridgeRaw = "";
|
|
5351
5878
|
|
|
5352
5879
|
const emitLegText = (chunk: string): void => {
|
|
5353
5880
|
if (chunk.length === 0) {
|
|
@@ -5370,20 +5897,55 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5370
5897
|
this.bufferAssistantTextForTts(token, chunk);
|
|
5371
5898
|
};
|
|
5372
5899
|
|
|
5373
|
-
const flushLegText = createControlMarkerHoldback(
|
|
5374
|
-
|
|
5375
|
-
|
|
5376
|
-
|
|
5377
|
-
|
|
5378
|
-
|
|
5379
|
-
|
|
5380
|
-
|
|
5381
|
-
|
|
5382
|
-
|
|
5383
|
-
|
|
5384
|
-
|
|
5385
|
-
|
|
5386
|
-
|
|
5900
|
+
const flushLegText = createControlMarkerHoldback(emitLegText);
|
|
5901
|
+
|
|
5902
|
+
const coordinator = leg.frontDoor
|
|
5903
|
+
? createFrontDoorLegCoordinator({
|
|
5904
|
+
// The hold branch is only classifiable while the leg is
|
|
5905
|
+
// speculative: its decision rule is the only one that teaches the
|
|
5906
|
+
// hold token.
|
|
5907
|
+
holdEnabled:
|
|
5908
|
+
activeTurn.speculativePending && activeTurn.speculativeHoldAllowed,
|
|
5909
|
+
host: {
|
|
5910
|
+
isLive: () => !activeTurn.finalized,
|
|
5911
|
+
language: () => activeTurn.language,
|
|
5912
|
+
progress: {
|
|
5913
|
+
// The verdict deadline goes with the narration timer: the
|
|
5914
|
+
// hand-off settles the verdict.
|
|
5915
|
+
clear: () => this.clearFillerTimers(activeTurn),
|
|
5916
|
+
noteFloorHolder: () => activeTurn.progress.noteFloorHolder(),
|
|
5917
|
+
arm: () => activeTurn.progress.arm(),
|
|
5918
|
+
},
|
|
5919
|
+
// Hold discards a speculative turn (mid-thought pause, keep
|
|
5920
|
+
// listening); escalate and answer both commit it: utterance
|
|
5921
|
+
// release, thinking frame, and timers all happen inside
|
|
5922
|
+
// commitSpeculativeTurn.
|
|
5923
|
+
onHold: () => {
|
|
5924
|
+
void this.holdSpeculativeTurn(activeTurn);
|
|
5925
|
+
},
|
|
5926
|
+
commit: () =>
|
|
5927
|
+
!activeTurn.speculativePending ||
|
|
5928
|
+
this.commitSpeculativeTurn(activeTurn),
|
|
5929
|
+
onAnswerText: (text) => {
|
|
5930
|
+
rawText += text;
|
|
5931
|
+
flushLegText(rawText);
|
|
5932
|
+
},
|
|
5933
|
+
abortLeg: () => {
|
|
5934
|
+
activeTurn.handle?.abort();
|
|
5935
|
+
activeTurn.handle = null;
|
|
5936
|
+
},
|
|
5937
|
+
speakBridge: (bridge) =>
|
|
5938
|
+
this.speakEscalationBridge(activeTurn, bridge),
|
|
5939
|
+
startEscalatedLeg: (escalated) => {
|
|
5940
|
+
this.noteEscalatedForegroundTask(activeTurn);
|
|
5941
|
+
void this.startAssistantLeg(activeTurn, escalated);
|
|
5942
|
+
},
|
|
5943
|
+
},
|
|
5944
|
+
})
|
|
5945
|
+
: null;
|
|
5946
|
+
if (coordinator !== null) {
|
|
5947
|
+
activeTurn.frontDoor = coordinator;
|
|
5948
|
+
}
|
|
5387
5949
|
|
|
5388
5950
|
try {
|
|
5389
5951
|
// Latched before the await, not after: this flag only decides whether the
|
|
@@ -5418,6 +5980,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5418
5980
|
// as telemetry instead, on `voiceTelemetry` below.
|
|
5419
5981
|
userMessageInterface: "macos",
|
|
5420
5982
|
assistantMessageInterface: "macos",
|
|
5983
|
+
...(this.context.startFrame.client === "macos"
|
|
5984
|
+
? { macosDesktopSession: true }
|
|
5985
|
+
: {}),
|
|
5421
5986
|
voiceTelemetry: {
|
|
5422
5987
|
sessionId: this.context.sessionId,
|
|
5423
5988
|
...(this.context.startFrame.client
|
|
@@ -5427,9 +5992,16 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5427
5992
|
? { entry: this.context.startFrame.entry }
|
|
5428
5993
|
: {}),
|
|
5429
5994
|
},
|
|
5430
|
-
voiceControlPrompt: buildVoiceControlPrompt(
|
|
5431
|
-
|
|
5432
|
-
|
|
5995
|
+
voiceControlPrompt: buildVoiceControlPrompt(
|
|
5996
|
+
activeTurn,
|
|
5997
|
+
{
|
|
5998
|
+
...(leg.frontDoor !== undefined
|
|
5999
|
+
? { frontDoor: leg.frontDoor }
|
|
6000
|
+
: {}),
|
|
6001
|
+
},
|
|
6002
|
+
this.sessionControls,
|
|
6003
|
+
{ lookFrames: this.lookFrames },
|
|
6004
|
+
),
|
|
5433
6005
|
onApprovalPending: (requestId) => {
|
|
5434
6006
|
this.revealRoomForPendingApproval(activeTurn, requestId);
|
|
5435
6007
|
},
|
|
@@ -5454,6 +6026,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5454
6026
|
...(leg.spokenEscalationBridge != null
|
|
5455
6027
|
? { spokenEscalationBridge: leg.spokenEscalationBridge }
|
|
5456
6028
|
: {}),
|
|
6029
|
+
...(leg.directEscalated === true ? { directEscalated: true } : {}),
|
|
5457
6030
|
// A speculative front-door leg's decision rule includes the hold
|
|
5458
6031
|
// branch so its leading tokens can be the hold verdict — but only
|
|
5459
6032
|
// on the utterance's FIRST dispatch. Extension replays (the
|
|
@@ -5470,53 +6043,16 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5470
6043
|
if (!this.isForwardingAssistantText(token)) {
|
|
5471
6044
|
return;
|
|
5472
6045
|
}
|
|
5473
|
-
if (leg.frontDoor) {
|
|
5474
|
-
|
|
5475
|
-
|
|
5476
|
-
|
|
5477
|
-
}
|
|
5478
|
-
if (frontDoorStage === "bridging") {
|
|
5479
|
-
bridgeRaw += msg.text;
|
|
5480
|
-
maybeHandOffBridge();
|
|
5481
|
-
return;
|
|
5482
|
-
}
|
|
6046
|
+
if (!leg.frontDoor) {
|
|
6047
|
+
this.claimSuspendedForegroundTask(activeTurn);
|
|
6048
|
+
}
|
|
6049
|
+
if (coordinator !== null) {
|
|
5483
6050
|
// Verdict-first: the leg's leading tokens decide the turn's
|
|
5484
|
-
// fate
|
|
5485
|
-
//
|
|
5486
|
-
//
|
|
5487
|
-
//
|
|
5488
|
-
|
|
5489
|
-
// rule is the only one that teaches the hold token).
|
|
5490
|
-
if (frontDoorStage === "deciding") {
|
|
5491
|
-
const verdict = classifyFrontDoorLeading(
|
|
5492
|
-
rawText.trimStart(),
|
|
5493
|
-
activeTurn.speculativePending &&
|
|
5494
|
-
activeTurn.speculativeHoldAllowed,
|
|
5495
|
-
);
|
|
5496
|
-
if (verdict === "pending") {
|
|
5497
|
-
return;
|
|
5498
|
-
}
|
|
5499
|
-
if (verdict === "hold") {
|
|
5500
|
-
void this.holdSpeculativeTurn(activeTurn);
|
|
5501
|
-
return;
|
|
5502
|
-
}
|
|
5503
|
-
if (
|
|
5504
|
-
activeTurn.speculativePending &&
|
|
5505
|
-
!this.commitSpeculativeTurn(activeTurn)
|
|
5506
|
-
) {
|
|
5507
|
-
return;
|
|
5508
|
-
}
|
|
5509
|
-
if (verdict === "escalate") {
|
|
5510
|
-
frontDoorStage = "bridging";
|
|
5511
|
-
bridgeRaw = rawText
|
|
5512
|
-
.trimStart()
|
|
5513
|
-
.slice(ESCALATE_VERDICT_TOKEN.length);
|
|
5514
|
-
maybeHandOffBridge();
|
|
5515
|
-
return;
|
|
5516
|
-
}
|
|
5517
|
-
frontDoorStage = "answer";
|
|
5518
|
-
}
|
|
5519
|
-
flushLegText(rawText);
|
|
6051
|
+
// fate, and the coordinator acts on them. The bridge hands
|
|
6052
|
+
// off in one piece once the machine caps it, so the audio,
|
|
6053
|
+
// the persisted row, and the phrase quoted to the escalated
|
|
6054
|
+
// leg are all the same text.
|
|
6055
|
+
coordinator.push(msg.text);
|
|
5520
6056
|
return;
|
|
5521
6057
|
}
|
|
5522
6058
|
// Defensive: speculative legs are always front-door today, but a
|
|
@@ -5545,6 +6081,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5545
6081
|
) {
|
|
5546
6082
|
return;
|
|
5547
6083
|
}
|
|
6084
|
+
if (!leg.frontDoor) {
|
|
6085
|
+
this.claimSuspendedForegroundTask(current);
|
|
6086
|
+
}
|
|
5548
6087
|
// A speculative leg that finished without a single delta (empty
|
|
5549
6088
|
// output, provider hiccup) carries no verdict — fail open to a
|
|
5550
6089
|
// committed turn so the utterance releases and finalizes like a
|
|
@@ -5552,27 +6091,20 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5552
6091
|
if (current.speculativePending) {
|
|
5553
6092
|
this.commitSpeculativeTurn(current);
|
|
5554
6093
|
}
|
|
5555
|
-
// A front-door leg that stopped mid-bridge
|
|
5556
|
-
//
|
|
5557
|
-
//
|
|
5558
|
-
//
|
|
5559
|
-
//
|
|
5560
|
-
//
|
|
5561
|
-
|
|
5562
|
-
|
|
5563
|
-
|
|
5564
|
-
|
|
5565
|
-
|
|
5566
|
-
|
|
5567
|
-
|
|
5568
|
-
|
|
5569
|
-
return;
|
|
5570
|
-
}
|
|
5571
|
-
// A front-door leg that handed off is finished; the escalated leg
|
|
5572
|
-
// drives completion. The front-door leg's own trailing completion
|
|
5573
|
-
// (including the generation_cancelled from its abort) is a no-op.
|
|
5574
|
-
if (leg.frontDoor && current.escalationHandedOff) {
|
|
5575
|
-
return;
|
|
6094
|
+
// A front-door leg that stopped mid-bridge hands off now with
|
|
6095
|
+
// whatever arrived. A cancellation mid-bridge falls through to
|
|
6096
|
+
// normal cancelled finalization instead: a dead turn must not
|
|
6097
|
+
// spawn an escalated leg. A front-door leg that handed off is
|
|
6098
|
+
// finished; the escalated leg drives completion, so this leg's
|
|
6099
|
+
// own trailing completion (including the generation_cancelled
|
|
6100
|
+
// from its abort) is a no-op.
|
|
6101
|
+
if (coordinator !== null) {
|
|
6102
|
+
if (msg.type === "message_complete") {
|
|
6103
|
+
coordinator.complete();
|
|
6104
|
+
}
|
|
6105
|
+
if (coordinator.handedOff) {
|
|
6106
|
+
return;
|
|
6107
|
+
}
|
|
5576
6108
|
}
|
|
5577
6109
|
// A held "[…"-tail that never completed a marker is real text —
|
|
5578
6110
|
// force-flush it before assistantCompleted closes the TTS buffer
|
|
@@ -5581,6 +6113,34 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5581
6113
|
if (!leg.frontDoor && msg.type === "message_complete") {
|
|
5582
6114
|
flushLegText(rawText, { force: true });
|
|
5583
6115
|
}
|
|
6116
|
+
// Read off the leg that finished the reply: a front-door answer
|
|
6117
|
+
// or the escalated leg. A handed-off front-door leg returned
|
|
6118
|
+
// above, so its holding phrase can never end a call.
|
|
6119
|
+
if (msg.type === "message_complete") {
|
|
6120
|
+
const request = requestedSessionControl(
|
|
6121
|
+
rawText,
|
|
6122
|
+
this.sessionControls,
|
|
6123
|
+
);
|
|
6124
|
+
if (request?.action === "updates") {
|
|
6125
|
+
// The session's own control: nothing to send, and nothing to
|
|
6126
|
+
// wait for, since it shapes turns that have not started yet.
|
|
6127
|
+
this.progressCadence = request.cadence;
|
|
6128
|
+
log.info(
|
|
6129
|
+
{ turnId, cadence: request.cadence },
|
|
6130
|
+
"Live voice progress cadence changed",
|
|
6131
|
+
);
|
|
6132
|
+
} else if (request?.action === "task_stop") {
|
|
6133
|
+
this.stopOutstandingInterruptedWork("spoken_task_stop");
|
|
6134
|
+
} else {
|
|
6135
|
+
if (request?.action === "end") {
|
|
6136
|
+
this.clearForegroundTask("call_end_requested");
|
|
6137
|
+
}
|
|
6138
|
+
current.sessionControlRequested = request;
|
|
6139
|
+
}
|
|
6140
|
+
}
|
|
6141
|
+
if (leg.frontDoor) {
|
|
6142
|
+
this.queueForegroundTaskResume(current);
|
|
6143
|
+
}
|
|
5584
6144
|
current.assistantCompleted = true;
|
|
5585
6145
|
if (msg.type === "generation_cancelled") {
|
|
5586
6146
|
void this.finalizeAssistantTurn(
|
|
@@ -5615,6 +6175,14 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5615
6175
|
if (current?.token !== token) {
|
|
5616
6176
|
return;
|
|
5617
6177
|
}
|
|
6178
|
+
const effectiveHostTool = hostInteractiveToolName(
|
|
6179
|
+
toolName,
|
|
6180
|
+
detail?.input,
|
|
6181
|
+
detail?.allowedToolNames,
|
|
6182
|
+
);
|
|
6183
|
+
if (effectiveHostTool !== null) {
|
|
6184
|
+
this.noteHostToolStarted(current, effectiveHostTool);
|
|
6185
|
+
}
|
|
5618
6186
|
// Foreground wins the workspace: the continuation runs with full
|
|
5619
6187
|
// subagent abilities (it can write files, run commands), so the
|
|
5620
6188
|
// moment a live turn starts a consequential tool the two could
|
|
@@ -5649,57 +6217,16 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5649
6217
|
toolName,
|
|
5650
6218
|
});
|
|
5651
6219
|
}
|
|
5652
|
-
|
|
5653
|
-
// completion) so a burst of slow tools still trips the ops
|
|
5654
|
-
// trigger while they run.
|
|
5655
|
-
current.progress.ops.push({
|
|
5656
|
-
toolName,
|
|
5657
|
-
...(detail?.toolUseId !== undefined
|
|
5658
|
-
? { toolUseId: detail.toolUseId }
|
|
5659
|
-
: {}),
|
|
5660
|
-
startedAtMs: Date.now(),
|
|
5661
|
-
});
|
|
5662
|
-
current.progress.opsSinceNarration += 1;
|
|
5663
|
-
current.progress.stateEpoch += 1;
|
|
6220
|
+
current.progress.toolStarted(toolName, detail?.toolUseId);
|
|
5664
6221
|
this.refreshActivity(current);
|
|
5665
6222
|
log.debug({ turnId, toolName }, "Live voice turn started tool use");
|
|
5666
|
-
this.maybeNarrateProgress(current, "ops");
|
|
5667
6223
|
},
|
|
5668
6224
|
tool_result: (event) => {
|
|
5669
6225
|
const current = this.activeAssistantTurn;
|
|
5670
6226
|
if (current?.token !== token) {
|
|
5671
6227
|
return;
|
|
5672
6228
|
}
|
|
5673
|
-
|
|
5674
|
-
// incomplete op with the same name (parallel same-name ops
|
|
5675
|
-
// resolve newest-first).
|
|
5676
|
-
const op =
|
|
5677
|
-
(event.toolUseId !== undefined
|
|
5678
|
-
? current.progress.ops.find(
|
|
5679
|
-
(o) => o.toolUseId === event.toolUseId,
|
|
5680
|
-
)
|
|
5681
|
-
: undefined) ??
|
|
5682
|
-
findLastIncompleteOp(current.progress.ops, event.toolName);
|
|
5683
|
-
// A long-running op finishing is the beat the user has been
|
|
5684
|
-
// waiting through: it narrates immediately rather than waiting for
|
|
5685
|
-
// `opsThreshold` more ops, which on a one-slow-tool turn never
|
|
5686
|
-
// arrive. Short ops stay on the ops trigger — narrating every
|
|
5687
|
-
// quick lookup is the chatter this cadence exists to avoid.
|
|
5688
|
-
let trigger: "ops" | "op_complete" = "ops";
|
|
5689
|
-
if (op) {
|
|
5690
|
-
op.completedAtMs = Date.now();
|
|
5691
|
-
if (event.isError !== undefined) {
|
|
5692
|
-
op.isError = event.isError;
|
|
5693
|
-
}
|
|
5694
|
-
op.resultPreview = event.resultPreview;
|
|
5695
|
-
if (
|
|
5696
|
-
op.completedAtMs - op.startedAtMs >=
|
|
5697
|
-
this.frontModelConfig.progress.longOpMs
|
|
5698
|
-
) {
|
|
5699
|
-
trigger = "op_complete";
|
|
5700
|
-
}
|
|
5701
|
-
}
|
|
5702
|
-
current.progress.stateEpoch += 1;
|
|
6229
|
+
current.progress.toolFinished(event);
|
|
5703
6230
|
// Showing a surface implies revealing it: the room is a
|
|
5704
6231
|
// full-screen overlay, so a surface rendered behind it is a
|
|
5705
6232
|
// surface nobody sees.
|
|
@@ -5722,7 +6249,6 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5722
6249
|
}
|
|
5723
6250
|
}
|
|
5724
6251
|
this.refreshActivity(current);
|
|
5725
|
-
this.maybeNarrateProgress(current, trigger);
|
|
5726
6252
|
},
|
|
5727
6253
|
},
|
|
5728
6254
|
onError: (message) => {
|
|
@@ -5743,7 +6269,15 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5743
6269
|
if (currentTurn?.token !== token) {
|
|
5744
6270
|
return;
|
|
5745
6271
|
}
|
|
6272
|
+
const foregroundTaskResumeRearmed =
|
|
6273
|
+
!leg.frontDoor &&
|
|
6274
|
+
!currentTurn.discardRequested &&
|
|
6275
|
+
!currentTurn.abortController.signal.aborted &&
|
|
6276
|
+
this.rearmForegroundTaskAfterLegFailure(currentTurn);
|
|
5746
6277
|
await this.finalizeAssistantTurn(currentTurn, "cancelled", "error");
|
|
6278
|
+
if (foregroundTaskResumeRearmed) {
|
|
6279
|
+
this.scheduleForegroundTaskResume();
|
|
6280
|
+
}
|
|
5747
6281
|
})();
|
|
5748
6282
|
},
|
|
5749
6283
|
});
|
|
@@ -5779,9 +6313,12 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5779
6313
|
this.clearActiveAssistantTurn(token);
|
|
5780
6314
|
return true;
|
|
5781
6315
|
}
|
|
6316
|
+
if (!leg.frontDoor) {
|
|
6317
|
+
this.claimSuspendedForegroundTask(current);
|
|
6318
|
+
}
|
|
5782
6319
|
// The front-door leg may have handed off before its handle resolved;
|
|
5783
6320
|
// abort it rather than exposing it as the turn's live handle.
|
|
5784
|
-
if (
|
|
6321
|
+
if (coordinator?.handedOff === true) {
|
|
5785
6322
|
handle.abort();
|
|
5786
6323
|
return true;
|
|
5787
6324
|
}
|
|
@@ -5793,8 +6330,17 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5793
6330
|
return false;
|
|
5794
6331
|
}
|
|
5795
6332
|
|
|
6333
|
+
const foregroundTaskResumeRearmed =
|
|
6334
|
+
!leg.frontDoor &&
|
|
6335
|
+
leg.directEscalated !== true &&
|
|
6336
|
+
!activeTurn.discardRequested &&
|
|
6337
|
+
!activeTurn.abortController.signal.aborted &&
|
|
6338
|
+
this.rearmForegroundTaskAfterLegFailure(activeTurn);
|
|
5796
6339
|
this.clearFillerTimers(activeTurn);
|
|
5797
6340
|
this.clearActiveAssistantTurn(token);
|
|
6341
|
+
if (foregroundTaskResumeRearmed) {
|
|
6342
|
+
this.scheduleForegroundTaskResume();
|
|
6343
|
+
}
|
|
5798
6344
|
await this.sendFrame({
|
|
5799
6345
|
type: "error",
|
|
5800
6346
|
code: LiveVoiceProtocolErrorCode.InvalidField,
|
|
@@ -5809,37 +6355,17 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5809
6355
|
}
|
|
5810
6356
|
|
|
5811
6357
|
/**
|
|
5812
|
-
*
|
|
5813
|
-
*
|
|
5814
|
-
*
|
|
5815
|
-
*
|
|
5816
|
-
* starts the escalated leg on the same ActiveAssistantTurn. Idempotent.
|
|
6358
|
+
* Speak the escalation bridge so the strong-model call has no dead air.
|
|
6359
|
+
* The model's own bridge is real assistant speech (captions + TTS); the
|
|
6360
|
+
* canned fallback stays audio-only, matching the persisted-row hygiene (a
|
|
6361
|
+
* deleted row for a bridge the model never produced).
|
|
5817
6362
|
*/
|
|
5818
|
-
private
|
|
6363
|
+
private speakEscalationBridge(
|
|
5819
6364
|
activeTurn: ActiveAssistantTurn,
|
|
5820
|
-
|
|
6365
|
+
bridge: SpokenEscalationBridge,
|
|
5821
6366
|
): void {
|
|
5822
|
-
|
|
5823
|
-
|
|
5824
|
-
}
|
|
5825
|
-
activeTurn.escalationHandedOff = true;
|
|
5826
|
-
// The escalation bridge below holds the floor, so pending progress
|
|
5827
|
-
// narration would only stack a second filler on top of it.
|
|
5828
|
-
this.clearFillerTimers(activeTurn);
|
|
5829
|
-
// Abort the front-door leg so a model that keeps generating past the
|
|
5830
|
-
// bridge cap adds no latency before the escalated leg starts.
|
|
5831
|
-
activeTurn.handle?.abort();
|
|
5832
|
-
activeTurn.handle = null;
|
|
5833
|
-
|
|
5834
|
-
// Speak the bridge so the strong-model call has no dead air. The model's
|
|
5835
|
-
// own bridge is real assistant speech (captions + TTS); the canned
|
|
5836
|
-
// fallback stays audio-only, matching the persisted-row hygiene (a
|
|
5837
|
-
// deleted row for a bridge the model never produced).
|
|
5838
|
-
const usesFallbackBridge = cappedBridge.length < MIN_SPOKEN_BRIDGE_CHARS;
|
|
5839
|
-
const spokenBridge = usesFallbackBridge
|
|
5840
|
-
? fallbackEscalationBridgeFor(activeTurn.language)
|
|
5841
|
-
: cappedBridge;
|
|
5842
|
-
if (!usesFallbackBridge) {
|
|
6367
|
+
const { spokenBridge, usesFallback, language } = bridge;
|
|
6368
|
+
if (!usesFallback) {
|
|
5843
6369
|
this.markFirstAssistantDelta(activeTurn.utterance, activeTurn.turnId);
|
|
5844
6370
|
this.markAssistantDelta(activeTurn);
|
|
5845
6371
|
void this.sendFrame(
|
|
@@ -5851,46 +6377,14 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5851
6377
|
// otherwise sit buffered until a sentence boundary and leave the
|
|
5852
6378
|
// caller in silence during the escalated model's call.
|
|
5853
6379
|
this.flushTtsBuffer(activeTurn.token, true);
|
|
5854
|
-
|
|
5855
|
-
// The canned bridge is a fixed localized-table phrase, enqueued
|
|
5856
|
-
// directly (it is already one complete sentence) so the segment can
|
|
5857
|
-
// carry the "en" override when the table lacks the turn's language.
|
|
5858
|
-
const speakable = sanitizeForTts(spokenBridge).trim();
|
|
5859
|
-
if (speakable.length > 0) {
|
|
5860
|
-
this.enqueueTtsSegment(activeTurn.token, speakable, {
|
|
5861
|
-
language: this.fixedPhraseLanguage(
|
|
5862
|
-
activeTurn,
|
|
5863
|
-
FALLBACK_ESCALATION_BRIDGE_BY_LANGUAGE,
|
|
5864
|
-
),
|
|
5865
|
-
});
|
|
5866
|
-
}
|
|
6380
|
+
return;
|
|
5867
6381
|
}
|
|
5868
|
-
|
|
5869
|
-
//
|
|
5870
|
-
//
|
|
5871
|
-
|
|
5872
|
-
|
|
5873
|
-
|
|
5874
|
-
// row).
|
|
5875
|
-
void this.startAssistantLeg(activeTurn, {
|
|
5876
|
-
content: ESCALATION_CONTINUATION_CONTENT,
|
|
5877
|
-
routingLeg: "escalated",
|
|
5878
|
-
spokenEscalationBridge: spokenBridge,
|
|
5879
|
-
});
|
|
5880
|
-
|
|
5881
|
-
// Escalated legs run the slowest work in the system (strong-model
|
|
5882
|
-
// thinking + tool loops), and the bridge only covers the first couple of
|
|
5883
|
-
// seconds of it — exactly the dead air progress narration exists for.
|
|
5884
|
-
// Re-arm the idle narration timer (cleared above with the ack): its
|
|
5885
|
-
// audible-silence gating means nothing speaks until the bridge audio has
|
|
5886
|
-
// fully drained plus a whole idle interval. Acks stay suppressed
|
|
5887
|
-
// post-handoff — the bridge already served that role.
|
|
5888
|
-
if (
|
|
5889
|
-
this.frontModelConfig.progress.enabled &&
|
|
5890
|
-
this.streamTtsAudio &&
|
|
5891
|
-
this.progressNarrator
|
|
5892
|
-
) {
|
|
5893
|
-
this.armProgressIdleTimer(activeTurn);
|
|
6382
|
+
// The canned bridge is a fixed localized-table phrase, enqueued
|
|
6383
|
+
// directly (it is already one complete sentence) so the segment can
|
|
6384
|
+
// carry the "en" override when the table lacks the turn's language.
|
|
6385
|
+
const speakable = sanitizeForTts(spokenBridge).trim();
|
|
6386
|
+
if (speakable.length > 0) {
|
|
6387
|
+
this.enqueueTtsSegment(activeTurn.token, speakable, { language });
|
|
5894
6388
|
}
|
|
5895
6389
|
}
|
|
5896
6390
|
|
|
@@ -5967,61 +6461,6 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5967
6461
|
turn.deltaEpoch += 1;
|
|
5968
6462
|
}
|
|
5969
6463
|
|
|
5970
|
-
// Arms (or re-arms) the dead-air narration timer. The countdown measures
|
|
5971
|
-
// audible silence — time since the turn's audio last (estimatedly) reached
|
|
5972
|
-
// the user's ears — not time since launch, so it covers mid-turn silences
|
|
5973
|
-
// for the whole turn. On expiry with audio still pending, or with the
|
|
5974
|
-
// silence not yet a full interval old, it re-arms for the remainder; only a
|
|
5975
|
-
// full interval of audible silence reaches the narration gatekeeper. The
|
|
5976
|
-
// interval is a polling cadence, not a speaking cadence: most ticks find
|
|
5977
|
-
// nothing new to report and stay quiet, so what the user hears follows the
|
|
5978
|
-
// turn's tool activity (with `maxSilenceMs` as the heartbeat ceiling).
|
|
5979
|
-
private armProgressIdleTimer(
|
|
5980
|
-
turn: ActiveAssistantTurn,
|
|
5981
|
-
delayMs?: number,
|
|
5982
|
-
): void {
|
|
5983
|
-
const { token } = turn;
|
|
5984
|
-
this.clearProgressIdleTimer(turn);
|
|
5985
|
-
turn.progress.idleTimer = setTimeout(() => {
|
|
5986
|
-
turn.progress.idleTimer = null;
|
|
5987
|
-
if (!this.isActiveAssistantTurn(token) || turn.assistantCompleted) {
|
|
5988
|
-
return;
|
|
5989
|
-
}
|
|
5990
|
-
if (!this.turnAudioIdle(turn)) {
|
|
5991
|
-
// Audio is buffered, queued, or still playing: a fresh silence can
|
|
5992
|
-
// only be a full interval old one interval from now.
|
|
5993
|
-
this.armProgressIdleTimer(turn);
|
|
5994
|
-
return;
|
|
5995
|
-
}
|
|
5996
|
-
const remaining = this.progressIdleDeadlineMs(turn) - Date.now();
|
|
5997
|
-
if (remaining > 0) {
|
|
5998
|
-
this.armProgressIdleTimer(turn, remaining);
|
|
5999
|
-
return;
|
|
6000
|
-
}
|
|
6001
|
-
this.maybeNarrateProgress(turn, "idle");
|
|
6002
|
-
this.armProgressIdleTimer(turn);
|
|
6003
|
-
}, delayMs ?? this.frontModelConfig.progress.idleIntervalMs);
|
|
6004
|
-
}
|
|
6005
|
-
|
|
6006
|
-
// Wall-clock instant the current audible silence turns a full interval old.
|
|
6007
|
-
private progressIdleDeadlineMs(turn: ActiveAssistantTurn): number {
|
|
6008
|
-
return (
|
|
6009
|
-
this.progressSilenceSinceMs(turn) +
|
|
6010
|
-
this.frontModelConfig.progress.idleIntervalMs
|
|
6011
|
-
);
|
|
6012
|
-
}
|
|
6013
|
-
|
|
6014
|
-
// When the turn's current audible silence began: the latest of the last
|
|
6015
|
-
// emitted segment, the estimated client playback end, and the last enqueued
|
|
6016
|
-
// filler.
|
|
6017
|
-
private progressSilenceSinceMs(turn: ActiveAssistantTurn): number {
|
|
6018
|
-
return Math.max(
|
|
6019
|
-
turn.progress.lastAudibleAtMs,
|
|
6020
|
-
this.assistantPlaybackTailUntilMs,
|
|
6021
|
-
turn.progress.lastFloorHolderAtMs ?? 0,
|
|
6022
|
-
);
|
|
6023
|
-
}
|
|
6024
|
-
|
|
6025
6464
|
// No assistant audio is pending or (estimatedly) still playing: nothing
|
|
6026
6465
|
// buffered toward the next sentence, every queued TTS segment fully
|
|
6027
6466
|
// emitted, and the client-side playback-tail estimate expired.
|
|
@@ -6033,177 +6472,18 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
6033
6472
|
);
|
|
6034
6473
|
}
|
|
6035
6474
|
|
|
6036
|
-
private clearProgressIdleTimer(turn: ActiveAssistantTurn): void {
|
|
6037
|
-
if (turn.progress.idleTimer !== null) {
|
|
6038
|
-
clearTimeout(turn.progress.idleTimer);
|
|
6039
|
-
turn.progress.idleTimer = null;
|
|
6040
|
-
}
|
|
6041
|
-
}
|
|
6042
|
-
|
|
6043
6475
|
// Clears progress narration and verdict timers for events that end the
|
|
6044
6476
|
// current filler lifecycle.
|
|
6045
6477
|
private clearFillerTimers(turn: ActiveAssistantTurn): void {
|
|
6046
|
-
|
|
6478
|
+
turn.progress.clear();
|
|
6047
6479
|
if (turn.verdictDeadlineTimer !== null) {
|
|
6048
6480
|
clearTimeout(turn.verdictDeadlineTimer);
|
|
6049
6481
|
turn.verdictDeadlineTimer = null;
|
|
6050
6482
|
}
|
|
6051
6483
|
}
|
|
6052
6484
|
|
|
6053
|
-
//
|
|
6054
|
-
//
|
|
6055
|
-
// floor only while its audio plays (covered by the audible-idle check),
|
|
6056
|
-
// and the strong leg's tool loops are the longest dead air in the system.
|
|
6057
|
-
private turnCanNarrateProgress(turn: ActiveAssistantTurn): boolean {
|
|
6058
|
-
return (
|
|
6059
|
-
this.isActiveAssistantTurn(turn.token) &&
|
|
6060
|
-
!turn.assistantCompleted &&
|
|
6061
|
-
// Nothing is in flight while a decision is pending, so every phrase
|
|
6062
|
-
// narration has would be a lie about who the call is waiting on. The
|
|
6063
|
-
// turn says so once, when it starts waiting, and is quiet after that.
|
|
6064
|
-
turn.pendingApproval === null &&
|
|
6065
|
-
this.turnAudioIdle(turn)
|
|
6066
|
-
);
|
|
6067
|
-
}
|
|
6068
|
-
|
|
6069
|
-
// The idle tick has something worth saying when the turn's tool activity has
|
|
6070
|
-
// moved since the last narration described it, or when the silence has run
|
|
6071
|
-
// past `maxSilenceMs` — the heartbeat ceiling that proves the assistant is
|
|
6072
|
-
// still alive on a turn with no observable activity at all. Every other tick
|
|
6073
|
-
// stays quiet, so the cadence follows the work rather than the clock.
|
|
6074
|
-
private progressIdleHasSomethingToSay(turn: ActiveAssistantTurn): boolean {
|
|
6075
|
-
const { progress } = turn;
|
|
6076
|
-
if (progress.stateEpoch !== progress.narratedEpoch) {
|
|
6077
|
-
return true;
|
|
6078
|
-
}
|
|
6079
|
-
const silentForMs = Date.now() - this.progressSilenceSinceMs(turn);
|
|
6080
|
-
if (silentForMs >= this.frontModelConfig.progress.maxSilenceMs) {
|
|
6081
|
-
return true;
|
|
6082
|
-
}
|
|
6083
|
-
log.debug(
|
|
6084
|
-
{ turnId: turn.turnId, silentForMs },
|
|
6085
|
-
"Live voice progress narration held — nothing new since the last update",
|
|
6086
|
-
);
|
|
6087
|
-
return false;
|
|
6088
|
-
}
|
|
6089
|
-
|
|
6090
|
-
// Gatekeeper for progress narration. It speaks only while the turn is
|
|
6091
|
-
// audibly silent, with one generation at a time and the configured spacing.
|
|
6092
|
-
private maybeNarrateProgress(
|
|
6093
|
-
turn: ActiveAssistantTurn,
|
|
6094
|
-
trigger: "ops" | "idle" | "op_complete",
|
|
6095
|
-
): void {
|
|
6096
|
-
const cfg = this.frontModelConfig.progress;
|
|
6097
|
-
const { progress } = turn;
|
|
6098
|
-
const progressNarrator = this.progressNarrator;
|
|
6099
|
-
if (
|
|
6100
|
-
!cfg.enabled ||
|
|
6101
|
-
!this.streamTtsAudio ||
|
|
6102
|
-
!progressNarrator ||
|
|
6103
|
-
!this.turnCanNarrateProgress(turn) ||
|
|
6104
|
-
progress.narrationInFlight ||
|
|
6105
|
-
(progress.lastFloorHolderAtMs !== null &&
|
|
6106
|
-
Date.now() - progress.lastFloorHolderAtMs < cfg.minGapMs) ||
|
|
6107
|
-
(trigger === "ops" && progress.opsSinceNarration < cfg.opsThreshold) ||
|
|
6108
|
-
(trigger === "idle" && !this.progressIdleHasSomethingToSay(turn))
|
|
6109
|
-
) {
|
|
6110
|
-
return;
|
|
6111
|
-
}
|
|
6112
|
-
void this.speakProgressUpdate(turn, progressNarrator, trigger);
|
|
6113
|
-
}
|
|
6114
|
-
|
|
6115
|
-
// Generate and speak one audio-only progress narration. On a null result,
|
|
6116
|
-
// idle narration falls back to a static phrase while activity-triggered
|
|
6117
|
-
// narration stays silent.
|
|
6118
|
-
private async speakProgressUpdate(
|
|
6119
|
-
turn: ActiveAssistantTurn,
|
|
6120
|
-
progressNarrator: VoiceProgressNarrator,
|
|
6121
|
-
trigger: "ops" | "idle" | "op_complete",
|
|
6122
|
-
): Promise<void> {
|
|
6123
|
-
const { progress } = turn;
|
|
6124
|
-
progress.narrationInFlight = true;
|
|
6125
|
-
// Any delta that lands while generation is in flight makes the text stale.
|
|
6126
|
-
const deltaEpochAtLaunch = turn.deltaEpoch;
|
|
6127
|
-
// The activity this update describes. Tool events that land mid-generation
|
|
6128
|
-
// are news the generated text cannot carry, so they must leave the idle
|
|
6129
|
-
// trigger armed rather than count as already narrated.
|
|
6130
|
-
const stateEpochAtLaunch = progress.stateEpoch;
|
|
6131
|
-
try {
|
|
6132
|
-
const now = Date.now();
|
|
6133
|
-
const currentOp = findLastIncompleteOp(progress.ops);
|
|
6134
|
-
const input: VoiceProgressTextInput = {
|
|
6135
|
-
transcriptSoFar: turn.utterance.finalTranscriptSegments
|
|
6136
|
-
.join(" ")
|
|
6137
|
-
.trim(),
|
|
6138
|
-
completedOps: progress.ops
|
|
6139
|
-
.filter(
|
|
6140
|
-
(op): op is TurnProgressOp & { completedAtMs: number } =>
|
|
6141
|
-
op.completedAtMs !== undefined,
|
|
6142
|
-
)
|
|
6143
|
-
// `ops` is in start order; the narrator wants completion
|
|
6144
|
-
// order, which differs when parallel tools finish out of order.
|
|
6145
|
-
.sort((a, b) => a.completedAtMs - b.completedAtMs)
|
|
6146
|
-
.map((op) => ({
|
|
6147
|
-
toolName: op.toolName,
|
|
6148
|
-
...(op.isError !== undefined ? { isError: op.isError } : {}),
|
|
6149
|
-
...(op.resultPreview !== undefined
|
|
6150
|
-
? { resultPreview: op.resultPreview }
|
|
6151
|
-
: {}),
|
|
6152
|
-
})),
|
|
6153
|
-
currentOp: currentOp
|
|
6154
|
-
? {
|
|
6155
|
-
toolName: currentOp.toolName,
|
|
6156
|
-
elapsedMs: now - currentOp.startedAtMs,
|
|
6157
|
-
}
|
|
6158
|
-
: null,
|
|
6159
|
-
turnElapsedMs: now - turn.launchedAtMs,
|
|
6160
|
-
updateIndex: progress.updatesSpoken + 1,
|
|
6161
|
-
...(turn.language !== undefined ? { languageHint: turn.language } : {}),
|
|
6162
|
-
};
|
|
6163
|
-
const generated = await progressNarrator
|
|
6164
|
-
.generateProgressText(input, turn.abortController.signal)
|
|
6165
|
-
// The narrator contract never rejects. Keep test stubs fail-soft too.
|
|
6166
|
-
.catch(() => null);
|
|
6167
|
-
// Re-check liveness and staleness after the provider call.
|
|
6168
|
-
if (
|
|
6169
|
-
!this.turnCanNarrateProgress(turn) ||
|
|
6170
|
-
turn.deltaEpoch !== deltaEpochAtLaunch ||
|
|
6171
|
-
(progress.lastFloorHolderAtMs !== null &&
|
|
6172
|
-
Date.now() - progress.lastFloorHolderAtMs <
|
|
6173
|
-
this.frontModelConfig.progress.minGapMs)
|
|
6174
|
-
) {
|
|
6175
|
-
return;
|
|
6176
|
-
}
|
|
6177
|
-
let raw = generated;
|
|
6178
|
-
// Generated text is in the turn's language; only the static
|
|
6179
|
-
// fallback comes from a localized table and may need the "en" override.
|
|
6180
|
-
let fillerLanguage: string | undefined;
|
|
6181
|
-
if (raw === null) {
|
|
6182
|
-
if (trigger !== "idle") {
|
|
6183
|
-
return;
|
|
6184
|
-
}
|
|
6185
|
-
raw = pickProgressPhrase(this.progressPhraseCounter++, turn.language);
|
|
6186
|
-
fillerLanguage = this.fixedPhraseLanguage(
|
|
6187
|
-
turn,
|
|
6188
|
-
PROGRESS_FALLBACK_PHRASES_BY_LANGUAGE,
|
|
6189
|
-
);
|
|
6190
|
-
}
|
|
6191
|
-
if (!this.enqueueFillerPhrase(turn, raw, fillerLanguage)) {
|
|
6192
|
-
return;
|
|
6193
|
-
}
|
|
6194
|
-
progress.opsSinceNarration = 0;
|
|
6195
|
-
progress.narratedEpoch = stateEpochAtLaunch;
|
|
6196
|
-
progress.updatesSpoken += 1;
|
|
6197
|
-
// Record only when narration audio actually enqueued.
|
|
6198
|
-
this.markProgressSpoken(turn);
|
|
6199
|
-
// Restart the dead-air countdown from this narration.
|
|
6200
|
-
this.armProgressIdleTimer(turn);
|
|
6201
|
-
} finally {
|
|
6202
|
-
progress.narrationInFlight = false;
|
|
6203
|
-
}
|
|
6204
|
-
}
|
|
6205
|
-
|
|
6206
|
-
// Sanitize and enqueue one progress sentence on the ordered TTS queue.
|
|
6485
|
+
// Sanitize and enqueue one filler sentence (narration, the approval line)
|
|
6486
|
+
// on the ordered TTS queue, audio-only.
|
|
6207
6487
|
private enqueueFillerPhrase(
|
|
6208
6488
|
turn: ActiveAssistantTurn,
|
|
6209
6489
|
raw: string,
|
|
@@ -6217,8 +6497,6 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
6217
6497
|
countsAsFirstSegment: false,
|
|
6218
6498
|
...(language !== undefined ? { language } : {}),
|
|
6219
6499
|
});
|
|
6220
|
-
// A spoken filler holds the floor, so narration's minGapMs spaces from it.
|
|
6221
|
-
turn.progress.lastFloorHolderAtMs = Date.now();
|
|
6222
6500
|
return true;
|
|
6223
6501
|
}
|
|
6224
6502
|
|
|
@@ -6231,10 +6509,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
6231
6509
|
turn: ActiveAssistantTurn,
|
|
6232
6510
|
table: Readonly<Record<string, unknown>>,
|
|
6233
6511
|
): string | undefined {
|
|
6234
|
-
return turn.language
|
|
6235
|
-
!hasLocalizedEntry(table, turn.language)
|
|
6236
|
-
? "en"
|
|
6237
|
-
: undefined;
|
|
6512
|
+
return fixedPhraseLanguage(table, turn.language);
|
|
6238
6513
|
}
|
|
6239
6514
|
|
|
6240
6515
|
private bufferAssistantTextForTts(token: symbol, text: string): void {
|
|
@@ -6277,7 +6552,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
6277
6552
|
log.info(
|
|
6278
6553
|
{
|
|
6279
6554
|
turnId: currentTurn.turnId,
|
|
6280
|
-
escalated: currentTurn.
|
|
6555
|
+
escalated: currentTurn.frontDoor?.handedOff ?? false,
|
|
6281
6556
|
spokenSegments: currentTurn.spokenSegments,
|
|
6282
6557
|
spokenChars: currentTurn.spokenChars,
|
|
6283
6558
|
},
|
|
@@ -6306,6 +6581,44 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
6306
6581
|
// leave the last tool it touched on screen through the next silence.
|
|
6307
6582
|
this.publishActivity(currentTurn, "");
|
|
6308
6583
|
|
|
6584
|
+
// Drain-scoped session control, under the same terms as the minimize
|
|
6585
|
+
// below: after the acknowledgement has been spoken, never for a
|
|
6586
|
+
// barged-in turn (talking over "okay, bye" means they are not
|
|
6587
|
+
// leaving), at most once per turn. Ending the call makes revealing
|
|
6588
|
+
// the screen moot, so an end takes the minimize's place.
|
|
6589
|
+
const sessionControl = currentTurn.sessionControlRequested;
|
|
6590
|
+
currentTurn.sessionControlRequested = null;
|
|
6591
|
+
if (
|
|
6592
|
+
sessionControl !== null &&
|
|
6593
|
+
!currentTurn.abortController.signal.aborted
|
|
6594
|
+
) {
|
|
6595
|
+
log.info(
|
|
6596
|
+
{ turnId: currentTurn.turnId, action: sessionControl.action },
|
|
6597
|
+
"Live voice reply requested a session control",
|
|
6598
|
+
);
|
|
6599
|
+
if (sessionControl.action === "end") {
|
|
6600
|
+
currentTurn.minimizeRequested = false;
|
|
6601
|
+
}
|
|
6602
|
+
// Armed before the send, so a client quick enough to answer with
|
|
6603
|
+
// its frame before this await resolves still finds the look waiting.
|
|
6604
|
+
// Never from the turn that answers a look: that turn is the answer,
|
|
6605
|
+
// and a marker it emits anyway must not chain another.
|
|
6606
|
+
if (
|
|
6607
|
+
isLookSessionControl(sessionControl.action) &&
|
|
6608
|
+
currentTurn.lookFollowUp === null
|
|
6609
|
+
) {
|
|
6610
|
+
this.awaitLookFrame(sessionControl.action);
|
|
6611
|
+
}
|
|
6612
|
+
await this.sendFrame(
|
|
6613
|
+
{
|
|
6614
|
+
type: "session_control",
|
|
6615
|
+
turnId: currentTurn.turnId,
|
|
6616
|
+
...sessionControl,
|
|
6617
|
+
},
|
|
6618
|
+
() => !this.isClosed,
|
|
6619
|
+
);
|
|
6620
|
+
}
|
|
6621
|
+
|
|
6309
6622
|
// Drain-scoped minimize: the latched marker is consumed here, after
|
|
6310
6623
|
// the turn's speech has fully drained — never mid-speech, never for
|
|
6311
6624
|
// a barged-in turn, at most once per turn.
|
|
@@ -6518,7 +6831,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
6518
6831
|
if (settledTurn?.token === token) {
|
|
6519
6832
|
// Anchor the dead-air countdown to the end of emission; the playback
|
|
6520
6833
|
// -tail estimate covers any client-side buffer still draining.
|
|
6521
|
-
settledTurn.progress.
|
|
6834
|
+
settledTurn.progress.noteAudioSettled();
|
|
6522
6835
|
}
|
|
6523
6836
|
this.pumpTtsSynthesis(token);
|
|
6524
6837
|
}
|
|
@@ -6835,6 +7148,14 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
6835
7148
|
|
|
6836
7149
|
turn.finalized = true;
|
|
6837
7150
|
this.clearFillerTimers(turn);
|
|
7151
|
+
if (
|
|
7152
|
+
this.foregroundTaskState?.phase === "owned" &&
|
|
7153
|
+
this.foregroundTaskState.ownerToken === turn.token
|
|
7154
|
+
) {
|
|
7155
|
+
this.clearForegroundTask(
|
|
7156
|
+
status === "completed" ? "owner_completed" : `owner_${reason}`,
|
|
7157
|
+
);
|
|
7158
|
+
}
|
|
6838
7159
|
turn.utterance.completed = true;
|
|
6839
7160
|
await this.archiveBufferedAudio({
|
|
6840
7161
|
turnId: turn.turnId,
|
|
@@ -7105,8 +7426,8 @@ export function createLiveVoiceSession(
|
|
|
7105
7426
|
const vadConfig = liveVoiceConfig?.vad;
|
|
7106
7427
|
// Parsed once here into a complete config, shared by endpointing and the
|
|
7107
7428
|
// session (the constructor's own parse is then a no-op re-validation).
|
|
7108
|
-
const frontModelConfig =
|
|
7109
|
-
options.frontModelConfig ??
|
|
7429
|
+
const frontModelConfig = VoiceFrontModelConfigSchema.parse(
|
|
7430
|
+
options.frontModelConfig ?? getConfig().voice?.frontModel ?? {},
|
|
7110
7431
|
);
|
|
7111
7432
|
return new LiveVoiceSession(context, {
|
|
7112
7433
|
...options,
|
|
@@ -7271,10 +7592,10 @@ export async function defaultSpawnBackgroundContinuation(args: {
|
|
|
7271
7592
|
// spoken by a session turn, on the session's terms; it was never meant to
|
|
7272
7593
|
// mean invisible.
|
|
7273
7594
|
//
|
|
7274
|
-
// NOT
|
|
7275
|
-
//
|
|
7276
|
-
//
|
|
7277
|
-
// spawning
|
|
7595
|
+
// NOT a per-turn sender: the conversation's event sink is fixed at
|
|
7596
|
+
// construction (see "Conversation event delivery" in assistant/AGENTS.md),
|
|
7597
|
+
// and the detach deliberately waits for the interrupted turn's bridge
|
|
7598
|
+
// teardown before spawning, so nothing turn-scoped is alive by the time
|
|
7278
7599
|
// these events fire. `broadcastMessage` is the same path the bridge itself
|
|
7279
7600
|
// uses to reach an attached web client. The subagent events carry
|
|
7280
7601
|
// `parentConversationId`, not `conversationId`, so scope explicitly.
|