@vellumai/assistant 0.12.1 → 0.12.2-staging.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +4 -0
- package/ARCHITECTURE.md +20 -21
- package/docs/architecture/turn-actor.md +15 -8
- package/docs/guardian-request-flow.md +16 -0
- package/node_modules/@vellumai/slack-text/src/index.ts +13 -8
- package/node_modules/@vellumai/slack-text/src/label-resolution-entities.test.ts +95 -0
- package/openapi.yaml +360 -51
- package/package.json +3 -1
- package/scripts/smoke-desktop-window-theme.ts +186 -0
- package/src/__tests__/always-loaded-tools-guard.test.ts +18 -10
- package/src/__tests__/anthropic-provider.test.ts +0 -63
- package/src/__tests__/app-compiler.test.ts +6 -0
- package/src/__tests__/app-control-tool-schemas.test.ts +5 -2
- package/src/__tests__/app-dir-path-guard.test.ts +1 -3
- package/src/__tests__/assistant-event-hub-machine-name.test.ts +6 -7
- package/src/__tests__/assistant-event-hub.test.ts +44 -0
- package/src/__tests__/computer-use-skill-manifest-regression.test.ts +1 -1
- package/src/__tests__/computer-use-skill-proxy-bridge.test.ts +5 -2
- package/src/__tests__/computer-use-tools.test.ts +19 -7
- package/src/__tests__/config-schema-cmd.test.ts +2 -2
- package/src/__tests__/config-schema.test.ts +28 -29
- package/src/__tests__/config-watcher.test.ts +23 -8
- package/src/__tests__/connection-model-compat.test.ts +2 -2
- package/src/__tests__/conversation-key-store-disk-view.test.ts +1 -1
- package/src/__tests__/conversation-pairing.test.ts +26 -0
- package/src/__tests__/conversation-process-app-control-preactivation.test.ts +4 -10
- package/src/__tests__/conversation-queue.test.ts +48 -207
- package/src/__tests__/conversation-retry-discard.test.ts +54 -1
- package/src/__tests__/conversation-runtime-assembly.test.ts +28 -0
- package/src/__tests__/conversation-skill-tools.test.ts +27 -0
- package/src/__tests__/conversation-surfaces-app-control.test.ts +8 -3
- package/src/__tests__/conversation-surfaces-point-at-budget.test.ts +1 -0
- package/src/__tests__/conversation-surfaces-point-at-capability.test.ts +1 -0
- package/src/__tests__/conversation-surfaces-task-progress.test.ts +67 -0
- package/src/__tests__/conversation-title-service.test.ts +14 -10
- package/src/__tests__/conversation-tool-setup-app-refresh.test.ts +3 -5
- package/src/__tests__/conversation-tool-setup-attribution.test.ts +51 -0
- package/src/__tests__/credential-record-retire.test.ts +171 -1
- package/src/__tests__/credential-routes.test.ts +250 -3
- package/src/__tests__/credential-security-invariants.test.ts +5 -3
- package/src/__tests__/cu-unified-flow.test.ts +14 -4
- package/src/__tests__/document-sync-tags.test.ts +25 -0
- package/src/__tests__/drain-kick-guard.test.ts +0 -2
- package/src/__tests__/drain-requeue-on-contention.test.ts +0 -6
- package/src/__tests__/evict-conversations-for-reload.test.ts +1 -1
- package/src/__tests__/git-porcelain.test.ts +47 -0
- package/src/__tests__/heartbeat-service.test.ts +56 -40
- package/src/__tests__/helpers/set-workspace-mcp.ts +11 -0
- package/src/__tests__/host-bash-proxy.test.ts +26 -45
- package/src/__tests__/host-browser-proxy.test.ts +6 -17
- package/src/__tests__/host-cu-proxy.test.ts +596 -0
- package/src/__tests__/host-file-proxy-targeted.test.ts +23 -0
- package/src/__tests__/host-file-proxy.test.ts +11 -23
- package/src/__tests__/host-transfer-proxy-targeted.test.ts +30 -9
- package/src/__tests__/list-messages-camera-frame.test.ts +121 -0
- package/src/__tests__/llm-catalog-parity.test.ts +25 -0
- package/src/__tests__/mcp-auth-routes.test.ts +6 -9
- package/src/__tests__/mcp-config-secret-boundary.test.ts +78 -136
- package/src/__tests__/mcp-health-check.test.ts +8 -11
- package/src/__tests__/mcp-list-plugin-servers.test.ts +8 -10
- package/src/__tests__/message-deleted-hook-dispatch.test.ts +119 -0
- package/src/__tests__/messages-after-tiebreaker.test.ts +91 -0
- package/src/__tests__/messages-read-boundary-guard.test.ts +1 -1
- package/src/__tests__/messaging-send-tool.test.ts +355 -548
- package/src/__tests__/model-intents.test.ts +3 -0
- package/src/__tests__/notification-vellum-adapter.test.ts +85 -2
- package/src/__tests__/oauth-commands-routes.test.ts +168 -0
- package/src/__tests__/oauth-provider-profiles.test.ts +18 -0
- package/src/__tests__/oauth-provider-serializer.test.ts +22 -0
- package/src/__tests__/oauth-providers-routes.test.ts +1 -0
- package/src/__tests__/openai-provider.test.ts +1 -0
- package/src/__tests__/outlook-messaging-provider.test.ts +6 -6
- package/src/__tests__/platform-callback-registration.test.ts +8 -16
- package/src/__tests__/platform-client-verify-credential.test.ts +2 -0
- package/src/__tests__/plugin-import-boundary-guard.test.ts +14 -0
- package/src/__tests__/qdrant-manager.test.ts +132 -0
- package/src/__tests__/resolve-app.test.ts +135 -0
- package/src/__tests__/scaffold-managed-skill-tool.test.ts +229 -6
- package/src/__tests__/script-proxy-certs.test.ts +35 -0
- package/src/__tests__/secret-routes-acp-guard.test.ts +59 -1
- package/src/__tests__/service-account-token.test.ts +247 -0
- package/src/__tests__/skill-feature-flags-integration.test.ts +1 -1
- package/src/__tests__/skill-feature-flags.test.ts +2 -2
- package/src/__tests__/skills.test.ts +9 -4
- package/src/__tests__/source-fingerprint.test.ts +184 -0
- package/src/__tests__/subagent-tool-gate-mode.test.ts +70 -3
- package/src/__tests__/subagent-tools.test.ts +3 -2
- package/src/__tests__/telephony-tts-guard.test.ts +38 -3
- package/src/__tests__/test-support/computer-use-skill-harness.ts +3 -2
- package/src/__tests__/ui-channel-variants.test.ts +1 -56
- package/src/__tests__/unicode.test.ts +13 -0
- package/src/__tests__/vellum-self-knowledge-inline-command.test.ts +4 -0
- package/src/__tests__/watch-retro-tool-availability.test.ts +8 -17
- package/src/__tests__/workspace-git-service.test.ts +26 -0
- package/src/__tests__/workspace-migration-154-repair-retired-codex-gpt-5-4-model-ids.test.ts +336 -0
- package/src/acp/__tests__/acp-claude-oauth.test.ts +257 -6
- package/src/acp/__tests__/acp-credentials.test.ts +13 -0
- package/src/acp/__tests__/agent-process.test.ts +258 -4
- package/src/acp/__tests__/auth-required.test.ts +96 -0
- package/src/acp/__tests__/claude-token-refresh.test.ts +257 -0
- package/src/acp/__tests__/client-handler.test.ts +90 -2
- package/src/acp/__tests__/helpers/acp-model-option.ts +51 -0
- package/src/acp/__tests__/prepare-agent-env.test.ts +60 -1
- package/src/acp/__tests__/session-manager-persistence.test.ts +5 -1
- package/src/acp/__tests__/session-manager-resume.test.ts +270 -6
- package/src/acp/__tests__/session-manager-usage.test.ts +4 -2
- package/src/acp/__tests__/session-manager.test.ts +895 -4
- package/src/acp/acp-claude-oauth.ts +328 -14
- package/src/acp/acp-credentials.ts +19 -0
- package/src/acp/agent-process.test.ts +4 -1
- package/src/acp/agent-process.ts +96 -10
- package/src/acp/auth-required.ts +50 -1
- package/src/acp/auto-install.test.ts +9 -7
- package/src/acp/claude-token-refresh.ts +150 -0
- package/src/acp/client-handler.ts +16 -3
- package/src/acp/model-config.test.ts +166 -0
- package/src/acp/model-config.ts +138 -0
- package/src/acp/prepare-agent-env.ts +21 -6
- package/src/acp/resolve-agent.test.ts +220 -6
- package/src/acp/resolve-agent.ts +88 -23
- package/src/acp/session-manager.test.ts +196 -3
- package/src/acp/session-manager.ts +441 -18
- package/src/acp/types.ts +50 -5
- package/src/agent/loop.ts +1 -0
- package/src/api/events/acp-session-model-update.test.ts +88 -0
- package/src/api/events/acp-session-model-update.ts +46 -0
- package/src/api/events/app-files-changed.ts +4 -3
- package/src/api/events/notification-intent.ts +2 -0
- package/src/api/events/user-message-echo.ts +7 -1
- package/src/api/index.ts +6 -0
- package/src/api/responses/conversation-message.ts +5 -0
- package/src/apps/app-store.ts +1 -1
- package/src/apps/resolve-app.ts +81 -0
- package/src/apps/source-fingerprint.ts +266 -0
- package/src/avatar/identity-avatar.ts +7 -2
- package/src/bundler/app-compiler.ts +15 -0
- package/src/calls/__tests__/barge-in-guard.test.ts +81 -0
- package/src/{__tests__ → calls/__tests__}/call-controller.test.ts +503 -82
- package/src/{__tests__ → calls/__tests__}/media-stream-audio-transcode.test.ts +1 -1
- package/src/{__tests__ → calls/__tests__}/media-stream-output.test.ts +37 -8
- package/src/{__tests__ → calls/__tests__}/media-stream-parser.test.ts +1 -1
- package/src/{__tests__ → calls/__tests__}/media-stream-server-integration.test.ts +150 -57
- package/src/{__tests__ → calls/__tests__}/media-stream-stt-session.test.ts +5 -5
- package/src/{__tests__ → calls/__tests__}/media-turn-detector.test.ts +1 -1
- package/src/{live-voice → calls}/__tests__/progress-narration.test.ts +4 -4
- package/src/{live-voice → calls}/__tests__/progress-phrases.test.ts +1 -1
- package/src/calls/__tests__/voice-control-protocol.test.ts +120 -25
- package/src/calls/__tests__/voice-leg-coordinator.test.ts +229 -0
- package/src/calls/__tests__/voice-progress-cadence.test.ts +274 -0
- package/src/{__tests__/voice-session-bridge.test.ts → calls/__tests__/voice-session-bridge-turn-flow.test.ts} +20 -12
- package/src/calls/__tests__/voice-session-bridge.test.ts +56 -1
- package/src/calls/__tests__/voice-triage-escalate.test.ts +105 -0
- package/src/calls/barge-in-guard.ts +118 -0
- package/src/calls/call-constants.ts +24 -4
- package/src/calls/call-controller.ts +499 -199
- package/src/calls/call-transport.ts +16 -0
- package/src/calls/media-stream-output.ts +32 -0
- package/src/calls/media-stream-server.ts +82 -30
- package/src/calls/media-stream-stt-session.ts +15 -0
- package/src/{live-voice → calls}/progress-narration.ts +2 -2
- package/src/calls/resolve-call-tts-provider.ts +33 -2
- package/src/calls/tts-call-strategy.ts +9 -5
- package/src/calls/types.ts +2 -0
- package/src/calls/voice-control-protocol.ts +155 -21
- package/src/calls/voice-leg-coordinator.ts +189 -0
- package/src/calls/voice-progress-cadence.ts +423 -0
- package/src/calls/voice-session-bridge.ts +92 -72
- package/src/calls/voice-triage-escalate.ts +178 -59
- package/src/cli/commands/__tests__/apps.test.ts +245 -0
- package/src/cli/commands/__tests__/bash.test.ts +53 -0
- package/src/cli/commands/__tests__/roadmap.help.test.ts +19 -0
- package/src/cli/commands/apps.help.ts +67 -13
- package/src/cli/commands/apps.ts +183 -0
- package/src/cli/commands/bash.help.ts +8 -13
- package/src/cli/commands/bash.ts +7 -25
- package/src/cli/commands/channels/__tests__/send.test.ts +218 -0
- package/src/cli/commands/channels/index.help.ts +70 -2
- package/src/cli/commands/channels/index.ts +16 -3
- package/src/cli/commands/channels/send.ts +128 -0
- package/src/cli/commands/credentials.help.ts +6 -2
- package/src/cli/commands/credentials.ts +2 -2
- package/src/cli/commands/mcp.help.ts +8 -8
- package/src/cli/commands/mcp.ts +1 -1
- package/src/cli/commands/oauth/status.ts +5 -0
- package/src/cli/commands/roadmap.help.ts +5 -1
- package/src/cli/commands/roadmap.ts +1 -1
- package/src/cli/lib/bundled-marketplace.json +103 -90
- package/src/config/__tests__/default-profile-catalog.test.ts +22 -0
- package/src/config/__tests__/memory-retrospective-schema.test.ts +15 -1
- package/src/config/__tests__/platform-identity.test.ts +255 -0
- package/src/config/__tests__/platform-rehydration.test.ts +99 -9
- package/src/config/acp-agent-write.ts +43 -0
- package/src/config/acp-defaults.test.ts +13 -3
- package/src/config/acp-defaults.ts +23 -6
- package/src/config/acp-schema.test.ts +81 -0
- package/src/config/acp-schema.ts +30 -2
- package/src/config/bundled-skills/acp/SKILL.md +22 -14
- package/src/config/bundled-skills/acp/TOOLS.json +5 -1
- package/src/config/bundled-skills/app-builder/SKILL.md +12 -9
- package/src/config/bundled-skills/app-builder/references/CUSTOM_ROUTES.md +1 -1
- package/src/config/bundled-skills/computer-use/SKILL.md +29 -2
- package/src/config/bundled-skills/computer-use/TOOLS.json +99 -0
- package/src/config/bundled-skills/computer-use/tools/computer-use-sequence.ts +12 -0
- package/src/config/bundled-skills/media-processing/services/preprocess.ts +2 -1
- package/src/config/bundled-skills/media-processing/services/processing-pipeline.ts +2 -1
- package/src/config/bundled-skills/media-processing/tools/generate-clip.ts +2 -1
- package/src/config/bundled-skills/messaging/SKILL.md +3 -3
- package/src/config/bundled-skills/messaging/TOOLS.json +3 -3
- package/src/config/bundled-skills/messaging/tools/messaging-send.ts +67 -139
- package/src/config/bundled-skills/phone-calls/references/TRANSCRIPTS.md +1 -1
- package/src/config/bundled-skills/playbooks/tools/playbook-create.ts +2 -1
- package/src/config/bundled-skills/playbooks/tools/playbook-update.ts +2 -1
- package/src/config/bundled-skills/schedule/SKILL.md +3 -4
- package/src/config/bundled-skills/screen-annotation/SKILL.md +8 -1
- package/src/config/bundled-skills/screen-annotation/tools/screen-annotation.test.ts +4 -1
- package/src/config/bundled-skills/skill-management/TOOLS.json +6 -2
- package/src/config/bundled-skills/transcribe/tools/transcribe-media.ts +5 -2
- package/src/config/default-profile-catalog.ts +1 -1
- package/src/config/env.ts +2 -2
- package/src/config/feature-flag-registry.json +33 -13
- package/src/config/loader.ts +2 -0
- package/src/config/memory-v3-gate.test.ts +31 -0
- package/src/config/memory-v3-gate.ts +20 -16
- package/src/config/platform-identity.ts +201 -0
- package/src/config/platform-rehydration.ts +11 -27
- package/src/config/schema.ts +2 -2
- package/src/config/schemas/__tests__/live-voice.test.ts +8 -158
- package/src/config/schemas/__tests__/voice.test.ts +148 -0
- package/src/config/schemas/calls.ts +1 -28
- package/src/config/schemas/live-voice.ts +1 -173
- package/src/config/schemas/mcp.ts +1 -1
- package/src/config/schemas/memory-retrospective.ts +10 -0
- package/src/config/schemas/services.ts +18 -0
- package/src/config/schemas/skills.ts +0 -14
- package/src/config/schemas/voice.ts +150 -0
- package/src/credential-execution/process-manager.test.ts +18 -19
- package/src/credential-health/credential-health-service.ts +11 -17
- package/src/daemon/__tests__/conversation-tool-setup.test.ts +66 -121
- package/src/daemon/__tests__/plugin-mcp-reconcile.test.ts +4 -1
- package/src/daemon/__tests__/skill-memory-refresh-mtime.test.ts +133 -0
- package/src/daemon/app-change-notify.ts +45 -0
- package/src/daemon/config-watcher.ts +13 -194
- package/src/daemon/conversation-error.ts +3 -2
- package/src/daemon/conversation-media-retry.ts +2 -1
- package/src/daemon/conversation-process.ts +35 -64
- package/src/daemon/conversation-runtime-assembly.ts +20 -7
- package/src/daemon/conversation-skill-tools.ts +13 -31
- package/src/daemon/conversation-store.ts +1 -1
- package/src/daemon/conversation-surfaces.ts +21 -10
- package/src/daemon/conversation-tool-setup.ts +45 -59
- package/src/daemon/conversation.ts +0 -3
- package/src/daemon/dictation-profile-store.ts +2 -1
- package/src/daemon/handlers/shared.ts +2 -1
- package/src/daemon/handlers/skills.ts +6 -3
- package/src/daemon/host-bash-proxy.ts +8 -12
- package/src/daemon/host-browser-proxy.ts +15 -1
- package/src/daemon/host-cu-proxy.ts +224 -19
- package/src/daemon/host-file-proxy.ts +8 -10
- package/src/daemon/host-proxy-preactivation.ts +2 -1
- package/src/daemon/host-transfer-proxy.ts +14 -18
- package/src/daemon/lifecycle.ts +0 -5
- package/src/daemon/mcp-reload-service.ts +5 -5
- package/src/daemon/message-types/sync.ts +1 -0
- package/src/daemon/orphan-reaper.ts +1 -1
- package/src/daemon/providers-setup.ts +1 -1
- package/src/daemon/reaction-history-render.ts +2 -1
- package/src/daemon/shutdown-handlers.ts +1 -9
- package/src/daemon/skill-memory-refresh.ts +64 -0
- package/src/daemon/tool-side-effects.ts +7 -18
- package/src/desktop/__tests__/fake-desktop.ts +122 -0
- package/src/desktop/desktop-chrome-policy.test.ts +79 -0
- package/src/desktop/desktop-chrome-policy.ts +47 -0
- package/src/desktop/desktop-chrome-session.ts +37 -0
- package/src/desktop/desktop-dependencies.test.ts +102 -0
- package/src/desktop/desktop-dependencies.ts +308 -0
- package/src/desktop/desktop-feature.test.ts +18 -0
- package/src/desktop/desktop-feature.ts +19 -0
- package/src/desktop/desktop-panel-config.test.ts +135 -0
- package/src/desktop/desktop-panel-config.ts +149 -0
- package/src/desktop/desktop-session-manager.test.ts +730 -0
- package/src/desktop/desktop-session-manager.ts +850 -0
- package/src/desktop/desktop-stream-bridge.test.ts +395 -0
- package/src/desktop/desktop-stream-bridge.ts +214 -0
- package/src/desktop/desktop-wallpaper.test.ts +191 -0
- package/src/desktop/desktop-wallpaper.ts +113 -0
- package/src/desktop/desktop-window-drag.ts +177 -0
- package/src/desktop/desktop-window-manager-config.test.ts +230 -0
- package/src/desktop/desktop-window-manager-config.ts +238 -0
- package/src/desktop/desktop-window-theme.ts +170 -0
- package/src/heartbeat/heartbeat-service.ts +21 -3
- package/src/home/__tests__/feed-writer.test.ts +22 -0
- package/src/home/feed-writer.ts +11 -2
- package/src/hooks/types.ts +33 -0
- package/src/i18n/AGENTS.md +22 -0
- package/src/i18n/__tests__/i18n.test.ts +108 -0
- package/src/i18n/index.ts +16 -0
- package/src/i18n/locales.ts +93 -0
- package/src/i18n/messages.ts +61 -0
- package/src/i18n/title.ts +25 -0
- package/src/inbound/platform-callback-registration.ts +4 -5
- package/src/live-voice/__tests__/live-voice-agent-turn.test.ts +121 -0
- package/src/live-voice/__tests__/live-voice-events.test.ts +8 -7
- package/src/live-voice/__tests__/live-voice-metrics.test.ts +0 -3
- package/src/live-voice/__tests__/live-voice-progress.test.ts +94 -15
- package/src/live-voice/__tests__/live-voice-sight-frame-echo.test.ts +238 -0
- package/src/live-voice/__tests__/live-voice-vad.test.ts +2 -2
- package/src/live-voice/__tests__/protocol.test.ts +39 -0
- package/src/live-voice/__tests__/runtime-websocket-shell.test.ts +13 -69
- package/src/live-voice/__tests__/session-controls.test.ts +79 -0
- package/src/live-voice/live-voice-photo.ts +25 -6
- package/src/live-voice/live-voice-session.ts +297 -699
- package/src/live-voice/protocol.ts +60 -2
- package/src/live-voice/session-controls.ts +111 -0
- package/src/mcp/__tests__/effective-config.test.ts +2 -2
- package/src/mcp/__tests__/workspace-mcp-config.test.ts +135 -0
- package/src/mcp/effective-config.ts +3 -3
- package/src/mcp/mcp-header-store.ts +29 -45
- package/src/mcp/spec-schema.ts +111 -0
- package/src/mcp/startup.ts +8 -4
- package/src/mcp/workspace-mcp-config.ts +188 -0
- package/src/messaging/provider.ts +10 -3
- package/src/messaging/providers/__tests__/proactive-address.test.ts +95 -0
- package/src/messaging/providers/callback-routing.ts +18 -0
- package/src/messaging/providers/channel-transport.ts +56 -0
- package/src/messaging/providers/discord/transport.ts +17 -0
- package/src/messaging/providers/slack/__tests__/adapter-token-routing.test.ts +6 -17
- package/src/messaging/providers/slack/adapter.ts +7 -44
- package/src/messaging/providers/slack/transport.ts +16 -0
- package/src/messaging/providers/telegram-bot/adapter.ts +1 -48
- package/src/messaging/providers/telegram-bot/client.ts +0 -26
- package/src/messaging/providers/telegram-bot/transport.ts +20 -0
- package/src/messaging/providers/whatsapp/adapter.ts +1 -48
- package/src/messaging/providers/whatsapp/transport.ts +18 -0
- package/src/monitoring/__tests__/workspace-git-heartbeat.test.ts +8 -0
- package/src/notifications/__tests__/assistant-reply-producer.test.ts +55 -0
- package/src/notifications/__tests__/guardian-feed-projection.test.ts +158 -2
- package/src/notifications/__tests__/home-feed-side-effect.test.ts +153 -3
- package/src/notifications/__tests__/schedule-result-producer.test.ts +99 -3
- package/src/notifications/adapters/macos.ts +3 -0
- package/src/notifications/assistant-reply-producer.ts +8 -3
- package/src/notifications/conversation-pairing.ts +36 -16
- package/src/notifications/delivered-post-record.ts +11 -1
- package/src/notifications/guardian-feed-projection.ts +125 -5
- package/src/notifications/home-feed-side-effect.ts +55 -7
- package/src/notifications/schedule-result-producer.ts +66 -22
- package/src/oauth/AGENTS.md +2 -0
- package/src/oauth/__tests__/seed-providers-managed.test.ts +149 -0
- package/src/oauth/byo-connection.test.ts +105 -0
- package/src/oauth/byo-connection.ts +36 -6
- package/src/oauth/connection-resolver.test.ts +100 -0
- package/src/oauth/connection-resolver.ts +120 -3
- package/src/oauth/provider-serializer.ts +9 -0
- package/src/oauth/scope-utils.ts +24 -0
- package/src/oauth/seed-providers.ts +314 -27
- package/src/outbound-proxy/certs.ts +3 -5
- package/src/permissions/checker.test.ts +11 -6
- package/src/persistence/compaction-log-store-clickhouse.ts +3 -3
- package/src/persistence/conversation-crud.ts +71 -63
- package/src/persistence/conversation-lineage.ts +12 -4
- package/src/persistence/conversation-queries.ts +2 -2
- package/src/persistence/conversation-title-placeholders.ts +23 -0
- package/src/persistence/conversation-title-service.ts +9 -26
- package/src/persistence/embeddings/qdrant-manager.ts +163 -18
- package/src/persistence/llm-request-log-sink-clickhouse.ts +3 -3
- package/src/persistence/llm-request-log-source-clickhouse.ts +3 -3
- package/src/persistence/message-cursor.ts +78 -0
- package/src/persistence/message-reads.ts +30 -0
- package/src/persistence/migrations/157-invite-contact-id.ts +18 -6
- package/src/persistence/migrations/__tests__/157-invite-contact-id.test.ts +149 -0
- package/src/persistence/schema/memory-core.ts +5 -0
- package/src/platform/client.test.ts +4 -2
- package/src/platform/client.ts +2 -10
- package/src/plugin-api/constants.ts +2 -0
- package/src/plugin-api/index.ts +1 -0
- package/src/plugin-api/types.ts +2 -0
- package/src/plugins/defaults/index.ts +2 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-accounting.test.ts +46 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-cursor-backfill.test.ts +118 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-cursor-preserve.test.ts +83 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-enqueue.test.ts +9 -5
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-job.test.ts +20 -10
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-prompt.test.ts +5 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-state.test.ts +219 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-sweep.test.ts +31 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-trigger-check.test.ts +63 -0
- package/src/plugins/defaults/memory/__tests__/shutdown-hook.test.ts +4 -0
- package/src/plugins/defaults/memory/context-search/agent-runner.ts +3 -2
- package/src/plugins/defaults/memory/context-search/format.ts +3 -2
- package/src/plugins/defaults/memory/context-search/sources/memory-v2.ts +6 -3
- package/src/plugins/defaults/memory/context-search/sources/workspace.ts +3 -2
- package/src/plugins/defaults/memory/graph/capability-seed.ts +6 -7
- package/src/plugins/defaults/memory/hooks/init.ts +10 -0
- package/src/plugins/defaults/memory/hooks/message-deleted.ts +21 -0
- package/src/plugins/defaults/memory/hooks/shutdown.ts +2 -0
- package/src/plugins/defaults/memory/host-utils.ts +10 -0
- package/src/plugins/defaults/memory/jobs-worker.ts +13 -0
- package/src/plugins/defaults/memory/memory-retrospective-accounting.ts +34 -60
- package/src/plugins/defaults/memory/memory-retrospective-cursor-backfill.ts +126 -0
- package/src/plugins/defaults/memory/memory-retrospective-cursor-preserve.ts +79 -0
- package/src/plugins/defaults/memory/memory-retrospective-cursor.ts +28 -0
- package/src/plugins/defaults/memory/memory-retrospective-enqueue.ts +3 -5
- package/src/plugins/defaults/memory/memory-retrospective-job.ts +12 -3
- package/src/plugins/defaults/memory/memory-retrospective-prompt.ts +1 -1
- package/src/plugins/defaults/memory/memory-retrospective-state.ts +312 -71
- package/src/plugins/defaults/memory/memory-retrospective-sweep.ts +2 -1
- package/src/plugins/defaults/memory/memory-retrospective-trigger-check.ts +5 -3
- package/src/plugins/defaults/memory/skill-md-mtime-poll.ts +9 -0
- package/src/plugins/defaults/memory/startup.ts +28 -2
- package/src/plugins/defaults/memory/substrate/__tests__/skill-content.test.ts +7 -9
- package/src/plugins/defaults/memory/substrate/page-index.ts +6 -1
- package/src/plugins/defaults/memory/substrate/skill-content.ts +2 -2
- package/src/plugins/defaults/memory/v1/graph/consolidation.ts +4 -2
- package/src/plugins/defaults/memory/v1/graph/extraction.ts +4 -2
- package/src/plugins/defaults/memory/v1/graph/retriever.ts +4 -1
- package/src/plugins/defaults/memory/v2/reranker.ts +6 -1
- package/src/plugins/defaults/memory/v3/__tests__/card.test.ts +24 -0
- package/src/plugins/defaults/memory/v3/__tests__/pool-select.test.ts +125 -0
- package/src/plugins/defaults/memory/v3/__tests__/sections.test.ts +26 -0
- package/src/plugins/defaults/memory/v3/__tests__/shadow-plugin.test.ts +30 -0
- package/src/plugins/defaults/memory/v3/card.ts +3 -2
- package/src/plugins/defaults/memory/v3/pool-select.ts +5 -3
- package/src/plugins/defaults/memory/v3/sections.ts +9 -3
- package/src/plugins/defaults/memory/v3/shadow-plugin.ts +5 -2
- package/src/plugins/defaults/memory/worker.ts +6 -0
- package/src/plugins/mcp-servers.ts +15 -75
- package/src/providers/__tests__/connection-model-compat.test.ts +3 -3
- package/src/providers/__tests__/retry-callsite.test.ts +41 -0
- package/src/providers/__tests__/retry-outbound-sanitize.test.ts +176 -0
- package/src/providers/anthropic/client.ts +0 -47
- package/src/providers/gemini/client.ts +9 -23
- package/src/providers/gemini-thought-signature.test.ts +63 -0
- package/src/providers/gemini-thought-signature.ts +44 -0
- package/src/providers/inference/__tests__/endpoint-probe.test.ts +42 -3
- package/src/providers/inference/auth.ts +2 -3
- package/src/providers/inference/connection-availability.ts +27 -8
- package/src/providers/inference/endpoint-probe.ts +10 -1
- package/src/providers/inference/resolve-auth.ts +22 -4
- package/src/providers/inference/service-account-token.ts +248 -0
- package/src/providers/model-catalog.ts +34 -0
- package/src/providers/model-intents.ts +10 -5
- package/src/providers/openai/__tests__/chat-completions-output-token-limit.test.ts +141 -0
- package/src/providers/openai/__tests__/chat-completions-provider-reasoning.test.ts +146 -18
- package/src/providers/openai/__tests__/chat-completions-provider-thought-signature.test.ts +613 -0
- package/src/providers/openai/__tests__/google-thought-signature.test.ts +235 -0
- package/src/providers/openai/__tests__/tool-choice-mapping.test.ts +83 -1
- package/src/providers/openai/chat-completions-provider.ts +212 -79
- package/src/providers/openai/codex-models.ts +0 -4
- package/src/providers/openai/google-thought-signature.ts +320 -0
- package/src/providers/opencode/client.test.ts +73 -4
- package/src/providers/opencode/client.ts +60 -3
- package/src/providers/openrouter/client.ts +20 -0
- package/src/providers/outbound-request-sanitize.ts +96 -0
- package/src/providers/retry.ts +30 -19
- package/src/providers/unparseable-tool-args.ts +5 -1
- package/src/runtime/AGENTS.md +3 -1
- package/src/runtime/__tests__/channel-send.test.ts +443 -0
- package/src/runtime/__tests__/desktop-stream-upgrade.test.ts +89 -0
- package/src/runtime/__tests__/host-observe.test.ts +40 -1
- package/src/runtime/__tests__/live-voice-upgrade.test.ts +58 -0
- package/src/runtime/__tests__/runtime-ws-test-utils.ts +72 -0
- package/src/runtime/__tests__/watch-stream-upgrade.test.ts +59 -0
- package/src/runtime/agent-wake.ts +2 -1
- package/src/runtime/assistant-event-hub.ts +21 -3
- package/src/runtime/auth/same-actor.ts +38 -3
- package/src/runtime/channel-send.ts +316 -0
- package/src/runtime/host-observe.ts +11 -1
- package/src/runtime/http-server.ts +160 -151
- package/src/runtime/routes/__tests__/acp-claude-auth-routes.test.ts +16 -5
- package/src/runtime/routes/__tests__/acp-routes.test.ts +159 -4
- package/src/runtime/routes/__tests__/apps-refresh-route.test.ts +275 -0
- package/src/runtime/routes/__tests__/channel-send-routes.test.ts +127 -0
- package/src/runtime/routes/__tests__/conversation-query-routes.test.ts +108 -1
- package/src/runtime/routes/__tests__/default-provider-routes.test.ts +2 -2
- package/src/runtime/routes/__tests__/dictation-cleanup-guard.test.ts +107 -0
- package/src/runtime/routes/__tests__/mcp-add-default-risk.test.ts +16 -12
- package/src/runtime/routes/acp-claude-auth-routes.ts +11 -3
- package/src/runtime/routes/acp-routes.test.ts +15 -0
- package/src/runtime/routes/acp-routes.ts +72 -9
- package/src/runtime/routes/app-management-routes.ts +90 -2
- package/src/runtime/routes/app-routes.ts +1 -1
- package/src/runtime/routes/channel-send-routes.ts +85 -0
- package/src/runtime/routes/conversation-list-routes.ts +14 -3
- package/src/runtime/routes/conversation-query-routes.ts +40 -0
- package/src/runtime/routes/conversation-routes.ts +16 -3
- package/src/runtime/routes/credential-routes.ts +20 -14
- package/src/runtime/routes/desktop-setup-routes.test.ts +74 -0
- package/src/runtime/routes/desktop-setup-routes.ts +35 -0
- package/src/runtime/routes/diagnostics-routes.ts +61 -4
- package/src/runtime/routes/events-routes.ts +4 -0
- package/src/runtime/routes/host-cu-routes.ts +6 -0
- package/src/runtime/routes/index.ts +4 -2
- package/src/runtime/routes/log-export-routes.ts +2 -0
- package/src/runtime/routes/mcp-auth-routes.ts +39 -47
- package/src/runtime/routes/migration-routes.ts +6 -20
- package/src/runtime/routes/oauth-commands-routes.ts +50 -0
- package/src/runtime/routes/oauth-providers.ts +7 -0
- package/src/runtime/routes/platform-routes.ts +18 -15
- package/src/runtime/routes/workspace-routes.test.ts +55 -26
- package/src/runtime/routes/workspace-routes.ts +46 -63
- package/src/runtime/routes/workspace-tree-walk.test.ts +244 -0
- package/src/runtime/routes/workspace-tree-walk.ts +240 -0
- package/src/runtime/routes/workspace-utils.ts +17 -0
- package/src/runtime/services/__tests__/conversation-serializer.test.ts +28 -0
- package/src/runtime/services/conversation-serializer.ts +16 -3
- package/src/runtime/sync/resource-sync-events.ts +2 -1
- package/src/schedule/run-script.ts +2 -1
- package/src/schedule/schedule-store.ts +9 -2
- package/src/schedule/worker.ts +1 -1
- package/src/security/__tests__/ces-http-record-backend.test.ts +137 -0
- package/src/security/ces-http-record-backend.ts +120 -0
- package/src/security/ces-rpc-record-backend.ts +7 -0
- package/src/security/untrusted-content.ts +3 -1
- package/src/skills/__tests__/workspace-skill-md-signature.test.ts +60 -0
- package/src/skills/workspace-skill-md-signature.ts +50 -0
- package/src/subagent/consult-context.ts +12 -25
- package/src/tools/__tests__/side-effect-abort-guard.test.ts +1 -0
- package/src/tools/acp/list-agents.test.ts +3 -1
- package/src/tools/acp/spawn.test.ts +146 -11
- package/src/tools/acp/spawn.ts +21 -3
- package/src/tools/acp/status.test.ts +94 -0
- package/src/tools/acp/status.ts +40 -3
- package/src/tools/app-control/skill-proxy-bridge.ts +2 -1
- package/src/tools/browser/browser-execution.ts +1 -1
- package/src/tools/browser/cdp-client/accessibility-snapshot.ts +9 -3
- package/src/tools/browser/cdp-client/cdp-inspect/discovery.ts +2 -1
- package/src/tools/browser/cdp-client/extension-cdp-client.ts +3 -2
- package/src/tools/browser/network-recorder.ts +2 -1
- package/src/tools/capability-offer.test.ts +22 -2
- package/src/tools/capability-offer.ts +16 -1
- package/src/tools/computer-use/definitions.ts +121 -2
- package/src/tools/computer-use/skill-proxy-bridge.ts +2 -1
- package/src/tools/credentials/metadata-store.ts +100 -0
- package/src/tools/filesystem/search.ts +2 -1
- package/src/tools/host-filesystem/edit.test.ts +2 -1
- package/src/tools/host-filesystem/edit.ts +2 -2
- package/src/tools/host-filesystem/read.test.ts +2 -1
- package/src/tools/host-filesystem/read.ts +2 -2
- package/src/tools/host-filesystem/transfer.test.ts +2 -1
- package/src/tools/host-filesystem/transfer.ts +2 -2
- package/src/tools/host-filesystem/write.test.ts +2 -1
- package/src/tools/host-filesystem/write.ts +2 -2
- package/src/tools/host-terminal/host-shell.test.ts +16 -0
- package/src/tools/host-terminal/host-shell.ts +4 -20
- package/src/tools/network/web-search.ts +6 -1
- package/src/tools/policy-context.ts +6 -7
- package/src/tools/skills/scaffold-managed.ts +61 -3
- package/src/tools/subagent/spawn.ts +5 -8
- package/src/tools/terminal/__tests__/sanitized-bash.test.ts +57 -0
- package/src/tools/terminal/sanitized-bash.ts +105 -0
- package/src/tools/ui-surface/channel-variants.ts +10 -59
- package/src/tools/watch/watch-retro-report.ts +0 -8
- package/src/types/host-capabilities.ts +5 -3
- package/src/util/__tests__/language-subtag.test.ts +23 -1
- package/src/util/ca-bundle.ts +13 -0
- package/src/util/language-subtag.ts +16 -0
- package/src/util/platform.ts +8 -0
- package/src/util/unicode.ts +15 -1
- package/src/watch/watch-retro.ts +6 -8
- package/src/watch/watch-timeline.ts +2 -2
- package/src/workflows/run-manager.ts +2 -1
- package/src/workspace/git-porcelain.ts +66 -0
- package/src/workspace/git-service.ts +6 -55
- package/src/workspace/migrations/154-repair-retired-codex-gpt-5-4-model-ids.ts +258 -0
- package/src/workspace/migrations/155-move-front-model-config-to-voice.ts +78 -0
- package/src/workspace/migrations/156-extract-workspace-mcp-json.ts +214 -0
- package/src/workspace/migrations/__tests__/155-move-front-model-config-to-voice.test.ts +102 -0
- package/src/workspace/migrations/__tests__/156-extract-workspace-mcp-json.test.ts +199 -0
- package/src/workspace/migrations/registry.ts +6 -0
- package/src/__tests__/app-source-watcher.test.ts +0 -218
- package/src/__tests__/config-watcher-skill-reseed.test.ts +0 -359
- package/src/__tests__/speaker-identification.test.ts +0 -56
- package/src/calls/speaker-identification.ts +0 -233
- package/src/daemon/app-source-watcher.ts +0 -218
- package/src/messaging/providers/whatsapp/client.ts +0 -29
- package/src/runtime/routes/debug-bash-routes.ts +0 -173
- package/src/util/fs-watcher-error.ts +0 -36
- /package/src/{live-voice → calls}/progress-phrases.ts +0 -0
|
@@ -17,6 +17,8 @@ import {
|
|
|
17
17
|
getRequestByPendingQuestionOrNull,
|
|
18
18
|
listGuardianRequestDeliveriesOrEmpty,
|
|
19
19
|
} from "../channels/gateway-guardian-requests.js";
|
|
20
|
+
import { loadConfig } from "../config/loader.js";
|
|
21
|
+
import type { VoiceProgressConfig } from "../config/schemas/voice.js";
|
|
20
22
|
import type { TrustContext } from "../daemon/trust-context-types.js";
|
|
21
23
|
import { DAEMON_INTERNAL_ASSISTANT_ID } from "../runtime/assistant-scope.js";
|
|
22
24
|
import { computeToolApprovalDigest } from "../security/tool-approval-digest.js";
|
|
@@ -32,11 +34,14 @@ import type { TtsProvider, TtsProviderId } from "../tts/types.js";
|
|
|
32
34
|
import { getLogger } from "../util/logger.js";
|
|
33
35
|
import type { CallAudioFormat } from "./audio-store.js";
|
|
34
36
|
import {
|
|
37
|
+
BARGE_IN_TEARDOWN_WAIT_MS,
|
|
38
|
+
DURATION_WARNING_LEAD_MS,
|
|
35
39
|
getEndCallDrainMaxWaitMs,
|
|
36
40
|
getEndCallListenWindowMs,
|
|
37
41
|
getMaxCallDurationMs,
|
|
38
42
|
getSilenceTimeoutMs,
|
|
39
43
|
getUserConsultationTimeoutMs,
|
|
44
|
+
POST_GOODBYE_HANGUP_DELAY_MS,
|
|
40
45
|
} from "./call-constants.js";
|
|
41
46
|
import {
|
|
42
47
|
formatDuration,
|
|
@@ -60,12 +65,15 @@ import type { CallTransport } from "./call-transport.js";
|
|
|
60
65
|
import { finalizeCall } from "./finalize-call.js";
|
|
61
66
|
import { sendGuardianExpiryNotices } from "./guardian-action-sweep.js";
|
|
62
67
|
import { dispatchGuardianQuestion } from "./guardian-dispatch.js";
|
|
68
|
+
import {
|
|
69
|
+
createVoiceProgressNarrator,
|
|
70
|
+
type VoiceProgressNarrator,
|
|
71
|
+
} from "./progress-narration.js";
|
|
63
72
|
import {
|
|
64
73
|
findPlayableTelephonyTtsFallbackProvider,
|
|
65
74
|
resolveCallTtsProvider,
|
|
66
75
|
resolveSynthesisFormats,
|
|
67
76
|
} from "./resolve-call-tts-provider.js";
|
|
68
|
-
import type { PromptSpeakerContext } from "./speaker-identification.js";
|
|
69
77
|
import {
|
|
70
78
|
resolveTelephonyLanguageVoice,
|
|
71
79
|
resolveTelephonySynthesisLanguage,
|
|
@@ -76,16 +84,24 @@ import {
|
|
|
76
84
|
CALL_OPENING_ACK_MARKER,
|
|
77
85
|
CALL_OPENING_MARKER,
|
|
78
86
|
CALL_VERIFICATION_COMPLETE_MARKER,
|
|
79
|
-
|
|
87
|
+
createControlMarkerHoldback,
|
|
80
88
|
END_CALL_MARKER,
|
|
81
89
|
extractBalancedJson,
|
|
82
90
|
stripInternalSpeechMarkers,
|
|
83
91
|
} from "./voice-control-protocol.js";
|
|
92
|
+
import {
|
|
93
|
+
createFrontDoorLegCoordinator,
|
|
94
|
+
type FrontDoorLegCoordinator,
|
|
95
|
+
type SpokenEscalationBridge,
|
|
96
|
+
} from "./voice-leg-coordinator.js";
|
|
97
|
+
import { createProgressCadence } from "./voice-progress-cadence.js";
|
|
84
98
|
import {
|
|
85
99
|
CONVERSATION_BUSY_MESSAGE,
|
|
86
100
|
startVoiceTurn,
|
|
101
|
+
type VoiceTurnCallbacks,
|
|
87
102
|
type VoiceTurnHandle,
|
|
88
103
|
} from "./voice-session-bridge.js";
|
|
104
|
+
import { type VoiceRoutingLeg } from "./voice-triage-escalate.js";
|
|
89
105
|
|
|
90
106
|
const log = getLogger("call-controller");
|
|
91
107
|
|
|
@@ -117,6 +133,32 @@ interface PendingEndCall {
|
|
|
117
133
|
wake: (() => void) | null;
|
|
118
134
|
}
|
|
119
135
|
|
|
136
|
+
/**
|
|
137
|
+
* What one voice turn produced, across both of its legs when it escalated.
|
|
138
|
+
* Feeds post-turn marker detection and the call event log.
|
|
139
|
+
*/
|
|
140
|
+
interface VoiceTurnOutcome {
|
|
141
|
+
/**
|
|
142
|
+
* The filtered response text: what the caller heard, plus the control
|
|
143
|
+
* markers the model emitted for the controller to act on.
|
|
144
|
+
*/
|
|
145
|
+
text: string;
|
|
146
|
+
/** Persisted row of the caller's utterance, when the bridge reported it. */
|
|
147
|
+
userMessageId: string | null;
|
|
148
|
+
/**
|
|
149
|
+
* Persisted row of the leg that answered: the escalated leg's when the
|
|
150
|
+
* turn escalated, else the front-door leg's own.
|
|
151
|
+
*/
|
|
152
|
+
assistantMessageId: string | null;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/** One leg of a triaged phone turn, as handed to the session bridge. */
|
|
156
|
+
interface PhoneVoiceLeg {
|
|
157
|
+
content: string;
|
|
158
|
+
routingLeg: VoiceRoutingLeg;
|
|
159
|
+
spokenEscalationBridge?: string;
|
|
160
|
+
}
|
|
161
|
+
|
|
120
162
|
export class CallController {
|
|
121
163
|
private callSessionId: string;
|
|
122
164
|
private transport: CallTransport;
|
|
@@ -201,6 +243,11 @@ export class CallController {
|
|
|
201
243
|
* resolution only.
|
|
202
244
|
*/
|
|
203
245
|
private resolveSynthesisLanguage: () => string | undefined;
|
|
246
|
+
/** Spoken progress narration (voice.frontModel.progress), null when off. */
|
|
247
|
+
private readonly progressNarrator: VoiceProgressNarrator | null;
|
|
248
|
+
private readonly progressConfig: VoiceProgressConfig;
|
|
249
|
+
/** Rotates the static narration fallback across turns. */
|
|
250
|
+
private progressPhraseCounter = 0;
|
|
204
251
|
|
|
205
252
|
constructor(
|
|
206
253
|
callSessionId: string,
|
|
@@ -211,6 +258,9 @@ export class CallController {
|
|
|
211
258
|
assistantId?: string;
|
|
212
259
|
trustContext?: TrustContext;
|
|
213
260
|
resolveSynthesisLanguage?: () => string | undefined;
|
|
261
|
+
/** Test seam: an explicit null disables narration for the call. */
|
|
262
|
+
progressNarrator?: VoiceProgressNarrator | null;
|
|
263
|
+
progressConfig?: VoiceProgressConfig;
|
|
214
264
|
},
|
|
215
265
|
) {
|
|
216
266
|
this.callSessionId = callSessionId;
|
|
@@ -223,6 +273,12 @@ export class CallController {
|
|
|
223
273
|
this.resolveSynthesisLanguage =
|
|
224
274
|
opts?.resolveSynthesisLanguage ??
|
|
225
275
|
(() => resolveTelephonySynthesisLanguage());
|
|
276
|
+
this.progressConfig =
|
|
277
|
+
opts?.progressConfig ?? loadConfig().voice.frontModel.progress;
|
|
278
|
+
this.progressNarrator =
|
|
279
|
+
opts?.progressNarrator !== undefined
|
|
280
|
+
? opts.progressNarrator
|
|
281
|
+
: createVoiceProgressNarrator({ config: this.progressConfig });
|
|
226
282
|
|
|
227
283
|
// Resolve the conversation ID and skipDisclosure from the call session
|
|
228
284
|
const session = getCallSession(callSessionId);
|
|
@@ -313,10 +369,7 @@ export class CallController {
|
|
|
313
369
|
* Caller utterances always trigger normal turns, even when a guardian
|
|
314
370
|
* consultation is pending — the consultation is tracked separately.
|
|
315
371
|
*/
|
|
316
|
-
async handleCallerUtterance(
|
|
317
|
-
transcript: string,
|
|
318
|
-
speaker?: PromptSpeakerContext,
|
|
319
|
-
): Promise<void> {
|
|
372
|
+
async handleCallerUtterance(transcript: string): Promise<void> {
|
|
320
373
|
// If the caller speaks while an END_CALL teardown is pending (during the
|
|
321
374
|
// drain wait or the listen window), this is a deferral — the caller is
|
|
322
375
|
// re-engaging after we tried to hang up. Track it so we can cap repeats.
|
|
@@ -343,13 +396,15 @@ export class CallController {
|
|
|
343
396
|
this.currentTurnPromise = null;
|
|
344
397
|
await Promise.race([
|
|
345
398
|
teardownPromise.catch(() => {}),
|
|
346
|
-
new Promise<void>((resolve) =>
|
|
399
|
+
new Promise<void>((resolve) =>
|
|
400
|
+
setTimeout(resolve, BARGE_IN_TEARDOWN_WAIT_MS),
|
|
401
|
+
),
|
|
347
402
|
]);
|
|
348
403
|
}
|
|
349
404
|
|
|
350
405
|
this.state = "processing";
|
|
351
406
|
this.resetSilenceTimer();
|
|
352
|
-
const callerContent =
|
|
407
|
+
const callerContent = transcript;
|
|
353
408
|
const shouldMarkOpeningAck = this.awaitingOpeningAck;
|
|
354
409
|
if (shouldMarkOpeningAck) {
|
|
355
410
|
this.awaitingOpeningAck = false;
|
|
@@ -433,37 +488,35 @@ export class CallController {
|
|
|
433
488
|
}
|
|
434
489
|
|
|
435
490
|
/**
|
|
436
|
-
* Handle a barge-in
|
|
491
|
+
* Handle a barge-in from sustained inbound caller speech.
|
|
437
492
|
*
|
|
438
|
-
*
|
|
439
|
-
*
|
|
440
|
-
*
|
|
441
|
-
*
|
|
442
|
-
*
|
|
493
|
+
* Interrupts the in-flight turn whether the assistant is audibly speaking
|
|
494
|
+
* or still thinking (processing, no audio yet), so a caller can cut in
|
|
495
|
+
* before the assistant starts talking. With no turn in flight the
|
|
496
|
+
* barge-in is ignored. The transport's sustained-speech guard is what
|
|
497
|
+
* keeps a stray frame from cancelling a still-starting turn; the state
|
|
498
|
+
* gate only says whether there is anything to cancel.
|
|
443
499
|
*
|
|
444
|
-
* @param onAccepted Invoked synchronously after the
|
|
445
|
-
*
|
|
446
|
-
*
|
|
447
|
-
*
|
|
448
|
-
*
|
|
449
|
-
* @returns `true` if the barge-in was accepted (
|
|
450
|
-
* `false` if it was ignored (assistant idle
|
|
500
|
+
* @param onAccepted Invoked synchronously after the gate passes but
|
|
501
|
+
* before {@link handleInterrupt} runs. Transports use this to flush
|
|
502
|
+
* queued outbound audio without wiping the end-of-turn mark that
|
|
503
|
+
* handleInterrupt enqueues, and without flushing at all when the
|
|
504
|
+
* barge-in is ignored.
|
|
505
|
+
* @returns `true` if the barge-in was accepted (a turn was in flight),
|
|
506
|
+
* `false` if it was ignored (assistant idle).
|
|
451
507
|
*/
|
|
452
508
|
handleBargeIn(onAccepted?: () => void): boolean {
|
|
453
|
-
if (this.state
|
|
509
|
+
if (this.state === "idle") {
|
|
454
510
|
log.debug(
|
|
455
|
-
{
|
|
456
|
-
|
|
457
|
-
state: this.state,
|
|
458
|
-
},
|
|
459
|
-
"Barge-in ignored — assistant not speaking",
|
|
511
|
+
{ callSessionId: this.callSessionId },
|
|
512
|
+
"Barge-in ignored: no turn in flight",
|
|
460
513
|
);
|
|
461
514
|
return false;
|
|
462
515
|
}
|
|
463
516
|
|
|
464
517
|
log.info(
|
|
465
|
-
{ callSessionId: this.callSessionId },
|
|
466
|
-
"Barge-in accepted
|
|
518
|
+
{ callSessionId: this.callSessionId, state: this.state },
|
|
519
|
+
"Barge-in accepted: interrupting the assistant's turn",
|
|
467
520
|
);
|
|
468
521
|
onAccepted?.();
|
|
469
522
|
this.handleInterrupt();
|
|
@@ -471,11 +524,12 @@ export class CallController {
|
|
|
471
524
|
}
|
|
472
525
|
|
|
473
526
|
/**
|
|
474
|
-
* Handle caller interrupting the assistant's
|
|
527
|
+
* Handle caller interrupting the assistant's turn.
|
|
475
528
|
*
|
|
476
529
|
* This is the hard interrupt path used for explicit teardown and
|
|
477
530
|
* internal abort scenarios. For barge-in from inbound audio, prefer
|
|
478
|
-
* {@link handleBargeIn} which
|
|
531
|
+
* {@link handleBargeIn}, which ignores an idle controller and relies on
|
|
532
|
+
* the transport's sustained-speech guard to vouch for the interruption.
|
|
479
533
|
*/
|
|
480
534
|
handleInterrupt(): void {
|
|
481
535
|
const wasSpeaking = this.state === "speaking";
|
|
@@ -585,22 +639,6 @@ export class CallController {
|
|
|
585
639
|
}
|
|
586
640
|
}
|
|
587
641
|
|
|
588
|
-
private formatCallerUtterance(
|
|
589
|
-
transcript: string,
|
|
590
|
-
speaker?: PromptSpeakerContext,
|
|
591
|
-
): string {
|
|
592
|
-
if (!speaker) {
|
|
593
|
-
return transcript;
|
|
594
|
-
}
|
|
595
|
-
const safeId = speaker.speakerId.replaceAll('"', "'");
|
|
596
|
-
const safeLabel = speaker.speakerLabel.replaceAll('"', "'");
|
|
597
|
-
const confidencePart =
|
|
598
|
-
speaker.speakerConfidence != null
|
|
599
|
-
? ` confidence="${speaker.speakerConfidence.toFixed(2)}"`
|
|
600
|
-
: "";
|
|
601
|
-
return `[SPEAKER id="${safeId}" label="${safeLabel}" source="${speaker.source}"${confidencePart}] ${transcript}`;
|
|
602
|
-
}
|
|
603
|
-
|
|
604
642
|
/**
|
|
605
643
|
* Execute a single voice turn through the conversation pipeline and stream
|
|
606
644
|
* the response back through the relay.
|
|
@@ -617,6 +655,9 @@ export class CallController {
|
|
|
617
655
|
}
|
|
618
656
|
const runVersion = ++this.llmRunVersion;
|
|
619
657
|
const runSignal = this.abortController.signal;
|
|
658
|
+
// Stamped before any pre-bridge work so the bridge's dispatch-timing log
|
|
659
|
+
// attributes the whole turn, TTS provider resolution included.
|
|
660
|
+
const launchedAtMs = Date.now();
|
|
620
661
|
|
|
621
662
|
// Clear silence timer while actively processing. The caller said
|
|
622
663
|
// something (or a turn was triggered), so silence detection should
|
|
@@ -629,19 +670,21 @@ export class CallController {
|
|
|
629
670
|
try {
|
|
630
671
|
// Stay in `processing` through the lock-wait and LLM generation; flip to
|
|
631
672
|
// `speaking` only when real outbound audio/tokens start (see
|
|
632
|
-
// beginSpeaking).
|
|
673
|
+
// beginSpeaking). The transport reads the state to know whether audio
|
|
674
|
+
// is audible; either phase is interruptible by sustained caller speech.
|
|
633
675
|
this.state = "processing";
|
|
634
676
|
|
|
635
|
-
const
|
|
677
|
+
const outcome = await this.streamTtsTokens(
|
|
636
678
|
content,
|
|
637
679
|
runVersion,
|
|
638
680
|
runSignal,
|
|
681
|
+
launchedAtMs,
|
|
639
682
|
);
|
|
640
683
|
if (!this.isCurrentRun(runVersion)) {
|
|
641
684
|
return;
|
|
642
685
|
}
|
|
643
686
|
|
|
644
|
-
await this.handleTurnCompletion(
|
|
687
|
+
await this.handleTurnCompletion(outcome);
|
|
645
688
|
} catch (err: unknown) {
|
|
646
689
|
this.currentTurnHandle = null;
|
|
647
690
|
// Aborted requests are expected (interruptions, rapid utterances)
|
|
@@ -700,15 +743,25 @@ export class CallController {
|
|
|
700
743
|
}
|
|
701
744
|
|
|
702
745
|
/**
|
|
703
|
-
*
|
|
704
|
-
* control markers before they reach the
|
|
705
|
-
*
|
|
746
|
+
* Run one turn through the conversation pipeline and stream its speech to
|
|
747
|
+
* the transport, buffering to strip control markers before they reach the
|
|
748
|
+
* relay.
|
|
749
|
+
*
|
|
750
|
+
* Every turn opens on the front-door leg: a fast, toolless model that
|
|
751
|
+
* either answers outright or hands off with a spoken holding phrase to an
|
|
752
|
+
* escalated leg on the conversation's own model. Both legs stream through
|
|
753
|
+
* the same TTS machinery and share this turn's single end-of-turn signal,
|
|
754
|
+
* so the caller never gets a listening window between the bridge and the
|
|
755
|
+
* answer. Phone turns commit on the STT provider's utterance-boundary
|
|
756
|
+
* final and carry no partial transcript, so the front-door leg never
|
|
757
|
+
* holds: routing is escalate-only.
|
|
706
758
|
*/
|
|
707
759
|
private async streamTtsTokens(
|
|
708
760
|
content: string,
|
|
709
761
|
runVersion: number,
|
|
710
762
|
runSignal: AbortSignal,
|
|
711
|
-
|
|
763
|
+
launchedAtMs: number,
|
|
764
|
+
): Promise<VoiceTurnOutcome> {
|
|
712
765
|
// Resolve the active TTS provider through the global abstraction.
|
|
713
766
|
// The catalog's callMode determines the call path: synthesized-play
|
|
714
767
|
// providers synthesize each speakable segment via the provider API as
|
|
@@ -727,14 +780,18 @@ export class CallController {
|
|
|
727
780
|
// Buffer incoming tokens so we can strip control markers ([ASK_GUARDIAN:...], [END_CALL])
|
|
728
781
|
// before they reach TTS. We hold text whenever an unmatched '[' appears, since it
|
|
729
782
|
// could be the start of a control marker.
|
|
730
|
-
let ttsBuffer = "";
|
|
731
783
|
let fullResponseText = "";
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
784
|
+
const outcome: VoiceTurnOutcome = {
|
|
785
|
+
text: "",
|
|
786
|
+
userMessageId: null,
|
|
787
|
+
assistantMessageId: null,
|
|
788
|
+
};
|
|
789
|
+
// Audio bookkeeping for progress narration: segments handed to synthesis
|
|
790
|
+
// but not yet emitted, text deltas seen (a narration generated while the
|
|
791
|
+
// model was already speaking is stale), and whether the turn is over.
|
|
792
|
+
let segmentsInFlight = 0;
|
|
793
|
+
let deltaEpoch = 0;
|
|
794
|
+
let turnSettled = false;
|
|
738
795
|
|
|
739
796
|
// Synthesized path: text is split at speakable boundaries as it streams
|
|
740
797
|
// and each segment is synthesized while the LLM keeps generating. The
|
|
@@ -770,6 +827,7 @@ export class CallController {
|
|
|
770
827
|
const speakSegmentViaPcmFallback = async (
|
|
771
828
|
failedProviderId: string,
|
|
772
829
|
segment: string,
|
|
830
|
+
language: string | undefined,
|
|
773
831
|
): Promise<void> => {
|
|
774
832
|
if (pcmFallbackProvider === undefined) {
|
|
775
833
|
pcmFallbackProvider =
|
|
@@ -801,6 +859,7 @@ export class CallController {
|
|
|
801
859
|
segment,
|
|
802
860
|
runVersion,
|
|
803
861
|
audioFormat,
|
|
862
|
+
language,
|
|
804
863
|
);
|
|
805
864
|
if (fallbackStatus !== "ok") {
|
|
806
865
|
// The fallback provider is failing too — stop retrying.
|
|
@@ -808,9 +867,12 @@ export class CallController {
|
|
|
808
867
|
}
|
|
809
868
|
};
|
|
810
869
|
|
|
870
|
+
// `language` overrides the turn's synthesis language for a fixed phrase
|
|
871
|
+
// whose text is not in the caller's language; undefined rides the turn's.
|
|
811
872
|
const enqueueSynthesisSegments = (
|
|
812
873
|
ttsProvider: TtsProvider,
|
|
813
874
|
segments: string[],
|
|
875
|
+
language?: string,
|
|
814
876
|
): void => {
|
|
815
877
|
for (const rawSegment of segments) {
|
|
816
878
|
// Sanitized per segment (not per delta) so markdown spanning deltas
|
|
@@ -820,21 +882,23 @@ export class CallController {
|
|
|
820
882
|
continue;
|
|
821
883
|
}
|
|
822
884
|
firstSynthSegmentEnqueued = true;
|
|
885
|
+
segmentsInFlight += 1;
|
|
823
886
|
synthesisChain = synthesisChain.then(async () => {
|
|
824
|
-
if (
|
|
825
|
-
!this.isCurrentRun(runVersion) ||
|
|
826
|
-
synthesisFailure ||
|
|
827
|
-
synthesisCancelled
|
|
828
|
-
) {
|
|
829
|
-
return;
|
|
830
|
-
}
|
|
831
887
|
try {
|
|
888
|
+
if (
|
|
889
|
+
!this.isCurrentRun(runVersion) ||
|
|
890
|
+
synthesisFailure ||
|
|
891
|
+
synthesisCancelled
|
|
892
|
+
) {
|
|
893
|
+
return;
|
|
894
|
+
}
|
|
832
895
|
if (!synthesisFellBack) {
|
|
833
896
|
const status = await this.synthesizeAndStreamAudio(
|
|
834
897
|
ttsProvider,
|
|
835
898
|
segment,
|
|
836
899
|
runVersion,
|
|
837
900
|
audioFormat,
|
|
901
|
+
language,
|
|
838
902
|
);
|
|
839
903
|
if (status === "ok") {
|
|
840
904
|
return;
|
|
@@ -860,9 +924,14 @@ export class CallController {
|
|
|
860
924
|
this.transport.sendTextToken(`${segment} `, false);
|
|
861
925
|
return;
|
|
862
926
|
}
|
|
863
|
-
await speakSegmentViaPcmFallback(ttsProvider.id, segment);
|
|
927
|
+
await speakSegmentViaPcmFallback(ttsProvider.id, segment, language);
|
|
864
928
|
} catch (err) {
|
|
865
929
|
synthesisFailure = { err };
|
|
930
|
+
} finally {
|
|
931
|
+
segmentsInFlight -= 1;
|
|
932
|
+
// The segment's audio has been handed off; the transport's
|
|
933
|
+
// playback tail covers what is still playing.
|
|
934
|
+
cadence.noteAudioSettled();
|
|
866
935
|
}
|
|
867
936
|
});
|
|
868
937
|
}
|
|
@@ -888,150 +957,367 @@ export class CallController {
|
|
|
888
957
|
}
|
|
889
958
|
this.beginSpeakingOnAudioStart(runVersion);
|
|
890
959
|
this.transport.sendTextToken(cleaned, false);
|
|
960
|
+
cadence.noteAudioSettled();
|
|
891
961
|
}
|
|
892
962
|
};
|
|
893
963
|
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
964
|
+
/**
|
|
965
|
+
* Speak a fixed phrase from a localized table (already one complete
|
|
966
|
+
* sentence, no markers) outside the model-text stream. `language` is the
|
|
967
|
+
* hint the phrase must carry: "en" when the table lacked the caller's
|
|
968
|
+
* language, so the English text is not rendered under a ko/ar/ta hint
|
|
969
|
+
* (on the native route that is the transport's system-copy contract);
|
|
970
|
+
* undefined rides the turn's language like model text.
|
|
971
|
+
*/
|
|
972
|
+
const speakFixedPhrase = (
|
|
973
|
+
text: string,
|
|
974
|
+
language: string | undefined,
|
|
975
|
+
): boolean => {
|
|
976
|
+
const cleaned = sanitizeForTts(text).trim();
|
|
977
|
+
if (cleaned.length === 0) {
|
|
978
|
+
return false;
|
|
897
979
|
}
|
|
898
|
-
if (
|
|
899
|
-
|
|
980
|
+
if (synthProvider) {
|
|
981
|
+
enqueueSynthesisSegments(synthProvider, [cleaned], language);
|
|
982
|
+
return true;
|
|
900
983
|
}
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
if (bracketIdx > 0) {
|
|
909
|
-
emitSafeChunk(ttsBuffer.slice(0, bracketIdx));
|
|
910
|
-
ttsBuffer = ttsBuffer.slice(bracketIdx);
|
|
911
|
-
}
|
|
984
|
+
this.beginSpeakingOnAudioStart(runVersion);
|
|
985
|
+
this.transport.sendTextToken(`${cleaned} `, false, {
|
|
986
|
+
systemCopy: language !== undefined,
|
|
987
|
+
});
|
|
988
|
+
cadence.noteAudioSettled();
|
|
989
|
+
return true;
|
|
990
|
+
};
|
|
912
991
|
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
992
|
+
// Spoken progress narration for the turn's dead air (tool loops on the
|
|
993
|
+
// escalated leg). The shared cadence owns the timing; this host supplies
|
|
994
|
+
// the phone's view of audible silence: nothing buffered toward a
|
|
995
|
+
// sentence, no segment mid-synthesis, and the transport's queue empty
|
|
996
|
+
// with its playback-tail estimate expired.
|
|
997
|
+
const callerTranscript =
|
|
998
|
+
content === CALL_OPENING_MARKER ||
|
|
999
|
+
content === CALL_VERIFICATION_COMPLETE_MARKER
|
|
1000
|
+
? ""
|
|
1001
|
+
: content;
|
|
1002
|
+
const cadence = createProgressCadence({
|
|
1003
|
+
config: this.progressConfig,
|
|
1004
|
+
narrator: this.progressNarrator,
|
|
1005
|
+
nextFallbackPhraseIndex: () => this.progressPhraseCounter++,
|
|
1006
|
+
host: {
|
|
1007
|
+
turnId: `${this.callSessionId}#${runVersion}`,
|
|
1008
|
+
launchedAtMs,
|
|
1009
|
+
signal: runSignal,
|
|
1010
|
+
canNarrate: () => this.isCurrentRun(runVersion) && !turnSettled,
|
|
1011
|
+
isAudioIdle: () =>
|
|
1012
|
+
pendingSynthText.length === 0 &&
|
|
1013
|
+
segmentsInFlight === 0 &&
|
|
1014
|
+
(this.transport.isPlaybackIdle?.() ?? true),
|
|
1015
|
+
playbackTailUntilMs: () => this.transport.playbackTailUntilMs?.() ?? 0,
|
|
1016
|
+
transcriptSoFar: () => callerTranscript,
|
|
1017
|
+
language: () => this.resolveSynthesisLanguage(),
|
|
1018
|
+
deltaEpoch: () => deltaEpoch,
|
|
1019
|
+
speak: speakFixedPhrase,
|
|
1020
|
+
},
|
|
1021
|
+
});
|
|
1022
|
+
const settleTurn = (): void => {
|
|
1023
|
+
turnSettled = true;
|
|
1024
|
+
cadence.clear();
|
|
932
1025
|
};
|
|
1026
|
+
runSignal.addEventListener("abort", settleTurn, { once: true });
|
|
1027
|
+
|
|
1028
|
+
// Speech goes out through the shared control-marker holdback: text up to
|
|
1029
|
+
// a possibly-streaming marker flushes, the marker itself is stripped, and
|
|
1030
|
+
// ordinary bracketed text ("[A]", "[note]") never stalls TTS. Marker
|
|
1031
|
+
// detection for actions (END_CALL, ASK_GUARDIAN) reads fullResponseText
|
|
1032
|
+
// separately, after the turn.
|
|
1033
|
+
const flushSafeText = createControlMarkerHoldback((chunk) => {
|
|
1034
|
+
if (!this.isCurrentRun(runVersion)) {
|
|
1035
|
+
return;
|
|
1036
|
+
}
|
|
1037
|
+
emitSafeChunk(chunk);
|
|
1038
|
+
});
|
|
933
1039
|
|
|
934
|
-
//
|
|
935
|
-
|
|
936
|
-
|
|
1040
|
+
// Transcript row ids reported by the bridge. The escalated leg persists
|
|
1041
|
+
// a hidden continuation prompt rather than the caller's words, so only
|
|
1042
|
+
// the front-door leg's user row is the utterance; and a front-door leg
|
|
1043
|
+
// that handed off owns the bridge row, not the answer, so its assistant
|
|
1044
|
+
// row is not the turn's.
|
|
1045
|
+
const legCallbacks = (
|
|
1046
|
+
routingLeg: VoiceRoutingLeg,
|
|
1047
|
+
handedOff: () => boolean,
|
|
1048
|
+
): VoiceTurnCallbacks => ({
|
|
1049
|
+
persisted_user_message_id: (messageId) => {
|
|
1050
|
+
if (!this.isCurrentRun(runVersion) || routingLeg === "escalated") {
|
|
1051
|
+
return;
|
|
1052
|
+
}
|
|
1053
|
+
outcome.userMessageId = messageId;
|
|
1054
|
+
},
|
|
1055
|
+
persisted_assistant_message_id: (messageId) => {
|
|
1056
|
+
if (!this.isCurrentRun(runVersion) || handedOff()) {
|
|
1057
|
+
return;
|
|
1058
|
+
}
|
|
1059
|
+
outcome.assistantMessageId = messageId;
|
|
1060
|
+
},
|
|
1061
|
+
// Tool activity lands in the call event log, the call's own record of
|
|
1062
|
+
// what happened on the line: the call summary and status surfaces
|
|
1063
|
+
// read it, and it is what the caller sat in silence through.
|
|
1064
|
+
tool_use_start: (toolName, detail) => {
|
|
937
1065
|
if (!this.isCurrentRun(runVersion)) {
|
|
938
1066
|
return;
|
|
939
1067
|
}
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
1068
|
+
recordCallEvent(this.callSessionId, "tool_use_started", {
|
|
1069
|
+
toolName,
|
|
1070
|
+
...(detail?.toolUseId !== undefined
|
|
1071
|
+
? { toolUseId: detail.toolUseId }
|
|
1072
|
+
: {}),
|
|
1073
|
+
});
|
|
1074
|
+
cadence.toolStarted(toolName, detail?.toolUseId);
|
|
1075
|
+
},
|
|
1076
|
+
tool_result: (event) => {
|
|
1077
|
+
if (!this.isCurrentRun(runVersion)) {
|
|
1078
|
+
return;
|
|
1079
|
+
}
|
|
1080
|
+
recordCallEvent(this.callSessionId, "tool_use_completed", {
|
|
1081
|
+
toolName: event.toolName,
|
|
1082
|
+
...(event.toolUseId !== undefined
|
|
1083
|
+
? { toolUseId: event.toolUseId }
|
|
1084
|
+
: {}),
|
|
1085
|
+
...(event.isError !== undefined ? { isError: event.isError } : {}),
|
|
1086
|
+
resultPreview: event.resultPreview,
|
|
1087
|
+
});
|
|
1088
|
+
cadence.toolFinished(event);
|
|
1089
|
+
},
|
|
1090
|
+
});
|
|
950
1091
|
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
1092
|
+
// Speak the escalation bridge so the strong-model call has no dead air.
|
|
1093
|
+
// The model's own holding phrase is real assistant speech and stays in
|
|
1094
|
+
// the turn's text, matching the row the bridge's transcript hygiene
|
|
1095
|
+
// keeps for it; the canned fallback is audio-only, matching the row it
|
|
1096
|
+
// deletes.
|
|
1097
|
+
const speakEscalationBridge = (bridge: SpokenEscalationBridge): void => {
|
|
1098
|
+
if (bridge.usesFallback) {
|
|
1099
|
+
speakFixedPhrase(bridge.spokenBridge, bridge.language);
|
|
1100
|
+
return;
|
|
1101
|
+
}
|
|
1102
|
+
fullResponseText += `${bridge.spokenBridge} `;
|
|
1103
|
+
flushSafeText(fullResponseText, { force: true });
|
|
1104
|
+
// Force-synthesize the bridge now. On the synthesized-TTS path text is
|
|
1105
|
+
// held in pendingSynthText until a sentence boundary, so an
|
|
1106
|
+
// unpunctuated bridge would otherwise sit unspoken until the post-leg
|
|
1107
|
+
// drain, leaving the caller in silence during the escalated model's
|
|
1108
|
+
// call: the exact gap the bridge exists to mask.
|
|
1109
|
+
if (synthProvider) {
|
|
1110
|
+
const { segments } = extractSpeakableSegments(pendingSynthText, true);
|
|
1111
|
+
pendingSynthText = "";
|
|
1112
|
+
enqueueSynthesisSegments(synthProvider, segments);
|
|
1113
|
+
}
|
|
1114
|
+
};
|
|
954
1115
|
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
1116
|
+
// The escalated leg, once the front-door leg hands off. Started from
|
|
1117
|
+
// inside the hand-off; awaited by the turn after the front-door leg
|
|
1118
|
+
// settles.
|
|
1119
|
+
let escalatedLeg: Promise<void> | null = null;
|
|
1120
|
+
|
|
1121
|
+
// Run one leg through the session bridge, streaming its deltas through
|
|
1122
|
+
// the shared TTS closures above. Errors cancel and drain this turn's
|
|
1123
|
+
// synthesis before propagating.
|
|
1124
|
+
const runVoiceLeg = async (leg: PhoneVoiceLeg): Promise<void> => {
|
|
1125
|
+
// Reasoning models can inline <think> spans in the content stream when
|
|
1126
|
+
// a profile has not opted into parseThinkTags. Neither the spoken path
|
|
1127
|
+
// nor the post-turn consumers of fullResponseText (transcripts,
|
|
1128
|
+
// assistant_spoke, END_CALL/ASK_GUARDIAN detection) may see them: both
|
|
1129
|
+
// are fed only filtered text. One filter per leg: a span never straddles
|
|
1130
|
+
// the hand-off.
|
|
1131
|
+
const reasoningFilter = createReasoningTagFilter();
|
|
1132
|
+
let legHandle: VoiceTurnHandle | null = null;
|
|
1133
|
+
// Verdict-first: a front-door leg's leading tokens decide the turn's
|
|
1134
|
+
// fate, so its raw stream is a control plane until they classify. The
|
|
1135
|
+
// shared coordinator reads it and sequences the hand-off; only text it
|
|
1136
|
+
// releases is spoken or recorded. Phone has no partial transcripts,
|
|
1137
|
+
// so the hold verdict is never taught.
|
|
1138
|
+
const coordinator: FrontDoorLegCoordinator | null =
|
|
1139
|
+
leg.routingLeg === "front-door"
|
|
1140
|
+
? createFrontDoorLegCoordinator({
|
|
1141
|
+
holdEnabled: false,
|
|
1142
|
+
host: {
|
|
1143
|
+
isLive: () =>
|
|
1144
|
+
this.isCurrentRun(runVersion) && !runSignal.aborted,
|
|
1145
|
+
language: () => this.resolveSynthesisLanguage(),
|
|
1146
|
+
progress: cadence,
|
|
1147
|
+
onAnswerText: (text) => {
|
|
1148
|
+
fullResponseText += text;
|
|
1149
|
+
flushSafeText(fullResponseText);
|
|
1150
|
+
},
|
|
1151
|
+
abortLeg: () => legHandle?.abort(),
|
|
1152
|
+
speakBridge: speakEscalationBridge,
|
|
1153
|
+
startEscalatedLeg: (escalated) => {
|
|
1154
|
+
const started = runVoiceLeg(escalated);
|
|
1155
|
+
// Awaited by the turn; the rejection is observed there.
|
|
1156
|
+
started.catch(() => {});
|
|
1157
|
+
escalatedLeg = started;
|
|
1158
|
+
},
|
|
1159
|
+
},
|
|
1160
|
+
})
|
|
1161
|
+
: null;
|
|
958
1162
|
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
content,
|
|
964
|
-
assistantId: this.assistantId,
|
|
965
|
-
trustContext: this.trustContext ?? undefined,
|
|
966
|
-
isInbound: this.isInbound,
|
|
967
|
-
task: this.task,
|
|
968
|
-
skipDisclosure: this.skipDisclosure,
|
|
969
|
-
onTextDelta,
|
|
970
|
-
onComplete,
|
|
971
|
-
onError,
|
|
972
|
-
signal: runSignal,
|
|
973
|
-
})
|
|
974
|
-
.then((handle) => {
|
|
975
|
-
if (this.isCurrentRun(runVersion)) {
|
|
976
|
-
this.currentTurnHandle = handle;
|
|
977
|
-
} else {
|
|
978
|
-
// Turn was superseded before handle arrived; abort immediately
|
|
979
|
-
handle.abort();
|
|
1163
|
+
const legComplete = new Promise<void>((resolve, reject) => {
|
|
1164
|
+
const ingest = (speakable: string): void => {
|
|
1165
|
+
if (speakable.length > 0) {
|
|
1166
|
+
deltaEpoch += 1;
|
|
980
1167
|
}
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
1168
|
+
if (coordinator !== null) {
|
|
1169
|
+
coordinator.push(speakable);
|
|
1170
|
+
return;
|
|
1171
|
+
}
|
|
1172
|
+
fullResponseText += speakable;
|
|
1173
|
+
flushSafeText(fullResponseText);
|
|
1174
|
+
};
|
|
985
1175
|
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
1176
|
+
const onTextDelta = (text: string): void => {
|
|
1177
|
+
// Once the front-door leg hands off, nothing more from it is
|
|
1178
|
+
// spoken, recorded, or acted on.
|
|
1179
|
+
if (!this.isCurrentRun(runVersion) || coordinator?.handedOff) {
|
|
1180
|
+
return;
|
|
1181
|
+
}
|
|
1182
|
+
ingest(reasoningFilter.push(text));
|
|
1183
|
+
};
|
|
1184
|
+
|
|
1185
|
+
const onComplete = (): void => {
|
|
1186
|
+
// A cancelled leg (barge-in, teardown) never hands off: a dead turn
|
|
1187
|
+
// must not spawn an escalated leg. The bridge reports its own
|
|
1188
|
+
// abort here too, which the hand-off already settled.
|
|
1189
|
+
if (
|
|
1190
|
+
!this.isCurrentRun(runVersion) ||
|
|
1191
|
+
runSignal.aborted ||
|
|
1192
|
+
coordinator?.handedOff
|
|
1193
|
+
) {
|
|
1194
|
+
resolve();
|
|
1195
|
+
return;
|
|
1196
|
+
}
|
|
1197
|
+
// A held "[..." tail that never completed a marker is real text,
|
|
1198
|
+
// released to both consumers before the leg settles.
|
|
1199
|
+
ingest(reasoningFilter.flush());
|
|
1200
|
+
coordinator?.complete();
|
|
992
1201
|
resolve();
|
|
993
|
-
}
|
|
994
|
-
{ once: true },
|
|
995
|
-
);
|
|
996
|
-
});
|
|
1202
|
+
};
|
|
997
1203
|
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1204
|
+
const onError = (message: string): void => {
|
|
1205
|
+
reject(new Error(message));
|
|
1206
|
+
};
|
|
1207
|
+
|
|
1208
|
+
startVoiceTurn({
|
|
1209
|
+
conversationId: this.conversationId,
|
|
1210
|
+
callSessionId: this.callSessionId,
|
|
1211
|
+
// The call session is the phone analogue of a live-voice session:
|
|
1212
|
+
// its id joins the turn's telemetry row to the call, and the entry
|
|
1213
|
+
// names the direction the call came from.
|
|
1214
|
+
voiceTelemetry: {
|
|
1215
|
+
sessionId: this.callSessionId,
|
|
1216
|
+
entry: this.isInbound ? "phone_inbound" : "phone_outbound",
|
|
1217
|
+
},
|
|
1218
|
+
content: leg.content,
|
|
1219
|
+
assistantId: this.assistantId,
|
|
1220
|
+
trustContext: this.trustContext ?? undefined,
|
|
1221
|
+
isInbound: this.isInbound,
|
|
1222
|
+
task: this.task,
|
|
1223
|
+
skipDisclosure: this.skipDisclosure,
|
|
1224
|
+
launchedAtMs,
|
|
1225
|
+
routingLeg: leg.routingLeg,
|
|
1226
|
+
...(leg.spokenEscalationBridge !== undefined
|
|
1227
|
+
? { spokenEscalationBridge: leg.spokenEscalationBridge }
|
|
1228
|
+
: {}),
|
|
1229
|
+
onTextDelta,
|
|
1230
|
+
onComplete,
|
|
1231
|
+
onError,
|
|
1232
|
+
callbacks: legCallbacks(
|
|
1233
|
+
leg.routingLeg,
|
|
1234
|
+
() => coordinator?.handedOff === true,
|
|
1235
|
+
),
|
|
1236
|
+
signal: runSignal,
|
|
1237
|
+
})
|
|
1238
|
+
.then((handle) => {
|
|
1239
|
+
legHandle = handle;
|
|
1240
|
+
if (this.isCurrentRun(runVersion) && !coordinator?.handedOff) {
|
|
1241
|
+
this.currentTurnHandle = handle;
|
|
1242
|
+
} else {
|
|
1243
|
+
// Superseded, or the front-door leg handed off before its
|
|
1244
|
+
// handle arrived: abort immediately.
|
|
1245
|
+
handle.abort();
|
|
1246
|
+
}
|
|
1247
|
+
})
|
|
1248
|
+
.catch((err) => {
|
|
1249
|
+
reject(err);
|
|
1250
|
+
});
|
|
1251
|
+
|
|
1252
|
+
// Defensive: if the turn is aborted (e.g. barge-in) and the event
|
|
1253
|
+
// sink callbacks are never invoked, resolve the promise so it
|
|
1254
|
+
// doesn't hang forever.
|
|
1255
|
+
runSignal.addEventListener(
|
|
1256
|
+
"abort",
|
|
1257
|
+
() => {
|
|
1258
|
+
resolve();
|
|
1259
|
+
},
|
|
1260
|
+
{ once: true },
|
|
1261
|
+
);
|
|
1262
|
+
});
|
|
1263
|
+
|
|
1264
|
+
// Eagerly mark the rejection as handled so runtimes (e.g. bun) don't
|
|
1265
|
+
// flag it as an unhandled rejection when onError fires synchronously
|
|
1266
|
+
// inside the Promise constructor before this await adds its handler.
|
|
1267
|
+
// The await below still re-throws, caught by the outer try-catch.
|
|
1268
|
+
legComplete.catch(() => {});
|
|
1269
|
+
try {
|
|
1270
|
+
await legComplete;
|
|
1271
|
+
} catch (err) {
|
|
1272
|
+
// Cancel and settle this turn's synthesis before the error reaches
|
|
1273
|
+
// the outer handler, so no straggling segment plays the partial
|
|
1274
|
+
// answer over the recovery prompt. While this run is current no
|
|
1275
|
+
// newer run's synthesis can be in flight, so the abort is safe.
|
|
1276
|
+
if (this.isCurrentRun(runVersion)) {
|
|
1277
|
+
synthesisCancelled = true;
|
|
1278
|
+
this.abortActiveSynthesis();
|
|
1279
|
+
}
|
|
1280
|
+
settleTurn();
|
|
1281
|
+
await synthesisChain.catch(() => {});
|
|
1282
|
+
throw err;
|
|
1013
1283
|
}
|
|
1284
|
+
};
|
|
1285
|
+
|
|
1286
|
+
// Superseded mid-stream (barge-in): drain the segment chain (queued
|
|
1287
|
+
// links short-circuit on staleness) so this turn's synthesis fully
|
|
1288
|
+
// settles instead of racing the next turn.
|
|
1289
|
+
const settleSuperseded = async (): Promise<VoiceTurnOutcome> => {
|
|
1290
|
+
settleTurn();
|
|
1014
1291
|
await synthesisChain.catch(() => {});
|
|
1015
|
-
|
|
1016
|
-
|
|
1292
|
+
outcome.text = fullResponseText;
|
|
1293
|
+
return outcome;
|
|
1294
|
+
};
|
|
1295
|
+
|
|
1296
|
+
// Narration speaks into the turn's audible dead air on a cadence wherever
|
|
1297
|
+
// in the turn it occurs; the front-door leg is toolless and quick, so in
|
|
1298
|
+
// practice the escalated leg's tool loops are what it covers.
|
|
1299
|
+
cadence.arm();
|
|
1300
|
+
await runVoiceLeg({ content, routingLeg: "front-door" });
|
|
1017
1301
|
if (!this.isCurrentRun(runVersion)) {
|
|
1018
|
-
|
|
1019
|
-
// links short-circuit on staleness — so this turn's synthesis fully
|
|
1020
|
-
// settles instead of racing the next turn.
|
|
1021
|
-
await synthesisChain.catch(() => {});
|
|
1022
|
-
return fullResponseText;
|
|
1302
|
+
return settleSuperseded();
|
|
1023
1303
|
}
|
|
1024
1304
|
|
|
1025
|
-
//
|
|
1026
|
-
//
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
emitSafeChunk(ttsBuffer);
|
|
1305
|
+
// The front-door model handed off: the coordinator spoke the bridge and
|
|
1306
|
+
// started the escalated leg, which answers for real.
|
|
1307
|
+
if (escalatedLeg !== null) {
|
|
1308
|
+
await escalatedLeg;
|
|
1309
|
+
if (!this.isCurrentRun(runVersion)) {
|
|
1310
|
+
return settleSuperseded();
|
|
1311
|
+
}
|
|
1033
1312
|
}
|
|
1034
1313
|
|
|
1314
|
+
// The turn's legs are done: no narration past this point.
|
|
1315
|
+
settleTurn();
|
|
1316
|
+
|
|
1317
|
+
// Final sweep: a held "[..." tail that never completed a marker is real
|
|
1318
|
+
// text, so the forced flush speaks it instead of dropping it.
|
|
1319
|
+
flushSafeText(fullResponseText, { force: true });
|
|
1320
|
+
|
|
1035
1321
|
// Synthesized path: force-extract whatever never reached a speakable
|
|
1036
1322
|
// boundary, then drain the chain so every segment's audio (or its
|
|
1037
1323
|
// native fallback) is enqueued before the end-of-turn signal. The
|
|
@@ -1052,7 +1338,8 @@ export class CallController {
|
|
|
1052
1338
|
// doesn't inject its end-of-turn marker (or fallback text) into the next
|
|
1053
1339
|
// turn's output stream.
|
|
1054
1340
|
if (!this.isCurrentRun(runVersion)) {
|
|
1055
|
-
|
|
1341
|
+
outcome.text = fullResponseText;
|
|
1342
|
+
return outcome;
|
|
1056
1343
|
}
|
|
1057
1344
|
|
|
1058
1345
|
// Signal end of this turn's speech. An empty token with `last: true`
|
|
@@ -1069,7 +1356,8 @@ export class CallController {
|
|
|
1069
1356
|
this.lastSentWasOpener = false;
|
|
1070
1357
|
}
|
|
1071
1358
|
|
|
1072
|
-
|
|
1359
|
+
outcome.text = fullResponseText;
|
|
1360
|
+
return outcome;
|
|
1073
1361
|
}
|
|
1074
1362
|
|
|
1075
1363
|
/**
|
|
@@ -1093,6 +1381,7 @@ export class CallController {
|
|
|
1093
1381
|
text: string,
|
|
1094
1382
|
runVersion: number,
|
|
1095
1383
|
format: CallAudioFormat = "mp3",
|
|
1384
|
+
languageOverride?: string,
|
|
1096
1385
|
): Promise<SegmentSynthesisStatus> {
|
|
1097
1386
|
let sink: AudioStoreSink | null = null;
|
|
1098
1387
|
let playUrlSent = false;
|
|
@@ -1114,7 +1403,9 @@ export class CallController {
|
|
|
1114
1403
|
|
|
1115
1404
|
this.activeSynthesisAbort = abortController;
|
|
1116
1405
|
|
|
1117
|
-
|
|
1406
|
+
// A fixed phrase whose text is not in the caller's language carries
|
|
1407
|
+
// its own hint; model text rides the turn's resolved language.
|
|
1408
|
+
const language = languageOverride ?? this.resolveSynthesisLanguage();
|
|
1118
1409
|
// A language-known segment may select the synthesizing provider's
|
|
1119
1410
|
// configured per-language voice; no entry keeps the provider default.
|
|
1120
1411
|
const voiceId = resolveTelephonyLanguageVoice(provider.id, language);
|
|
@@ -1202,12 +1493,19 @@ export class CallController {
|
|
|
1202
1493
|
* (ASK_GUARDIAN_APPROVAL / ASK_GUARDIAN), call finalization (END_CALL),
|
|
1203
1494
|
* and normal idle transition.
|
|
1204
1495
|
*/
|
|
1205
|
-
private async handleTurnCompletion(
|
|
1206
|
-
const responseText =
|
|
1496
|
+
private async handleTurnCompletion(outcome: VoiceTurnOutcome): Promise<void> {
|
|
1497
|
+
const responseText = outcome.text;
|
|
1207
1498
|
|
|
1208
|
-
// Record the assistant response event
|
|
1499
|
+
// Record the assistant response event, keyed to the conversation rows it
|
|
1500
|
+
// mirrors when the bridge reported them.
|
|
1209
1501
|
recordCallEvent(this.callSessionId, "assistant_spoke", {
|
|
1210
1502
|
text: responseText,
|
|
1503
|
+
...(outcome.userMessageId !== null
|
|
1504
|
+
? { userMessageId: outcome.userMessageId }
|
|
1505
|
+
: {}),
|
|
1506
|
+
...(outcome.assistantMessageId !== null
|
|
1507
|
+
? { assistantMessageId: outcome.assistantMessageId }
|
|
1508
|
+
: {}),
|
|
1211
1509
|
});
|
|
1212
1510
|
const spokenText = sanitizeForTts(
|
|
1213
1511
|
stripInternalSpeechMarkers(responseText),
|
|
@@ -1612,9 +1910,11 @@ export class CallController {
|
|
|
1612
1910
|
/**
|
|
1613
1911
|
* Flip from the pre-speech `processing` phase to `speaking` at the moment the
|
|
1614
1912
|
* first real outbound audio/token is emitted. Guarded so a superseded or
|
|
1615
|
-
* aborted (idle) turn never (re)enters `speaking
|
|
1616
|
-
*
|
|
1617
|
-
*
|
|
1913
|
+
* aborted (idle) turn never (re)enters `speaking`. The state tells the
|
|
1914
|
+
* transport whether audio is audible (it decides what a barge-in has to
|
|
1915
|
+
* clear and whether an end-of-turn mark is owed); a barge-in itself may
|
|
1916
|
+
* abort a turn in either phase once the transport's sustained-speech
|
|
1917
|
+
* guard has vouched for it.
|
|
1618
1918
|
*/
|
|
1619
1919
|
private beginSpeaking(runVersion: number): void {
|
|
1620
1920
|
if (!this.isCurrentRun(runVersion)) {
|
|
@@ -1833,7 +2133,7 @@ export class CallController {
|
|
|
1833
2133
|
|
|
1834
2134
|
private startDurationTimer(): void {
|
|
1835
2135
|
const maxDurationMs = getMaxCallDurationMs();
|
|
1836
|
-
const warningMs = maxDurationMs -
|
|
2136
|
+
const warningMs = maxDurationMs - DURATION_WARNING_LEAD_MS;
|
|
1837
2137
|
|
|
1838
2138
|
if (warningMs > 0) {
|
|
1839
2139
|
this.durationWarningTimer = setTimeout(() => {
|
|
@@ -1894,7 +2194,7 @@ export class CallController {
|
|
|
1894
2194
|
},
|
|
1895
2195
|
);
|
|
1896
2196
|
}
|
|
1897
|
-
},
|
|
2197
|
+
}, POST_GOODBYE_HANGUP_DELAY_MS);
|
|
1898
2198
|
}, maxDurationMs);
|
|
1899
2199
|
}
|
|
1900
2200
|
|