@vellumai/assistant 0.11.3 → 0.11.4-staging.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +11 -6
- package/docs/architecture/memory.md +11 -0
- package/docs/architecture/turn-actor.md +70 -0
- package/docs/flux-turn-detection-spike.md +243 -0
- package/docs/stt-provider-onboarding.md +3 -1
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/channels.ts +11 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/channels.ts +11 -0
- package/node_modules/@vellumai/gateway-client/src/admission-policy-contract.ts +34 -0
- package/node_modules/@vellumai/gateway-client/src/index.ts +2 -0
- package/node_modules/@vellumai/service-contracts/src/channels.ts +11 -0
- package/openapi.yaml +139 -37
- package/package.json +1 -1
- package/scripts/voice-ttft-spike.ts +3 -3
- package/src/__tests__/app-compiler.test.ts +38 -3
- package/src/__tests__/attachments-store.test.ts +21 -12
- package/src/__tests__/byok-default-profile-ensure.test.ts +17 -0
- package/src/__tests__/call-setup-flow-name-capture.test.ts +0 -1
- package/src/__tests__/call-site-routing-provider.test.ts +1 -1
- package/src/__tests__/channel-availability-routes.test.ts +14 -1
- package/src/__tests__/channel-capabilities-dedupe.test.ts +214 -0
- package/src/__tests__/channel-delivery-store.test.ts +14 -14
- package/src/__tests__/config-loader-backfill.test.ts +3 -3
- package/src/__tests__/config-schema.test.ts +25 -10
- package/src/__tests__/conversation-agent-loop-inference-profile.test.ts +8 -11
- package/src/__tests__/conversation-agent-loop-overflow.test.ts +8 -11
- package/src/__tests__/conversation-agent-loop.test.ts +28 -20
- package/src/__tests__/conversation-attention-store.test.ts +63 -0
- package/src/__tests__/conversation-delete-schedule-cleanup.test.ts +0 -4
- package/src/__tests__/conversation-fork-crud.test.ts +69 -0
- package/src/__tests__/conversation-fork-referential.test.ts +67 -0
- package/src/__tests__/conversation-fork-retrospective.test.ts +24 -0
- package/src/__tests__/conversation-notifiers-provenance.test.ts +59 -0
- package/src/__tests__/conversation-queue.test.ts +177 -4
- package/src/__tests__/conversation-runtime-assembly.test.ts +134 -102
- package/src/__tests__/conversation-runtime-workspace.test.ts +14 -10
- package/src/__tests__/credential-prompt-route.test.ts +7 -10
- package/src/__tests__/custom-profile-ensure.test.ts +5 -1
- package/src/__tests__/discord-access-request-privacy.test.ts +5 -1
- package/src/__tests__/discord-requester-notice-privacy.test.ts +3 -3
- package/src/__tests__/document-append-idempotency.test.ts +233 -0
- package/src/__tests__/edit-propagation.test.ts +0 -7
- package/src/__tests__/helpers/mock-actor-context.ts +49 -0
- package/src/__tests__/helpers/mock-conversation.ts +13 -1
- package/src/__tests__/injector-chain.test.ts +63 -41
- package/src/__tests__/injector-disk-pressure.test.ts +11 -23
- package/src/__tests__/llm-context-resolution.test.ts +73 -1
- package/src/__tests__/llm-schema.test.ts +5 -2
- package/src/__tests__/mcp-list-plugin-servers.test.ts +250 -0
- package/src/__tests__/memory-retrieval-hook.test.ts +6 -5
- package/src/__tests__/messages-read-boundary-guard.test.ts +134 -0
- package/src/__tests__/mtime-cache.test.ts +1 -1
- package/src/__tests__/non-member-access-request.test.ts +0 -20
- package/src/__tests__/outbound-slack-persistence.test.ts +40 -1
- package/src/__tests__/plugin-import-boundary-guard.test.ts +5 -0
- package/src/__tests__/plugin-secret-pattern-contribution.test.ts +1 -1
- package/src/__tests__/post-compaction-reinjection-idempotency.test.ts +14 -7
- package/src/__tests__/provider-commit-message-generator.test.ts +20 -0
- package/src/__tests__/run-conversation-turn-persistence.test.ts +434 -105
- package/src/__tests__/scoped-approval-grants.test.ts +11 -6
- package/src/__tests__/secret-ingress-channel.test.ts +0 -1
- package/src/__tests__/skills.test.ts +32 -0
- package/src/__tests__/slack-edit-ordering-characterization.test.ts +0 -1
- package/src/__tests__/subagent-call-site-routing.test.ts +31 -19
- package/src/__tests__/subagent-spawn-and-await.test.ts +14 -10
- package/src/__tests__/turn-events-store.test.ts +43 -0
- package/src/__tests__/ui-shape-teaching.test.ts +33 -0
- package/src/__tests__/ui-voice-picker-surface.test.ts +128 -0
- package/src/__tests__/user-plugin-loader.test.ts +1 -1
- package/src/__tests__/visible-app-context.test.ts +16 -9
- package/src/__tests__/voice-config-update.test.ts +40 -0
- package/src/__tests__/worker-entrypoint-guards.test.ts +54 -0
- package/src/__tests__/worker-plugin-surface.test.ts +77 -0
- package/src/__tests__/workspace-migration-142-consolidate-voice-front-door.test.ts +158 -0
- package/src/__tests__/workspace-migration-143-repair-deprecated-codex-model-id.test.ts +133 -0
- package/src/__tests__/workspace-migration-144-convert-stranded-subscription-openai-profiles.test.ts +316 -0
- package/src/__tests__/workspace-migration-145-collapse-profile-bindings-to-entries.test.ts +325 -0
- package/src/__tests__/workspace-migration-146-repair-retired-fireworks-deepseek-flash-model-id.test.ts +235 -0
- package/src/acp/__tests__/acp-claude-oauth.test.ts +10 -2
- package/src/acp/__tests__/auth-required.test.ts +161 -0
- package/src/acp/acp-claude-oauth.ts +19 -2
- package/src/acp/agent-process.test.ts +100 -0
- package/src/acp/agent-process.ts +29 -26
- package/src/acp/auth-required.ts +102 -0
- package/src/acp/session-manager.test.ts +119 -0
- package/src/acp/session-manager.ts +68 -2
- package/src/api/events/acp-auth-required.ts +55 -0
- package/src/api/index.ts +7 -0
- package/src/api/surfaces.ts +7 -3
- package/src/apps/app-store.ts +3 -0
- package/src/bundler/package-resolver.ts +2 -30
- package/src/calls/__tests__/voice-session-bridge.test.ts +173 -1
- package/src/calls/__tests__/voice-triage-escalate.test.ts +94 -0
- package/src/calls/call-controller.ts +19 -3
- package/src/calls/call-setup-flow.ts +0 -1
- package/src/calls/media-stream-stt-session.ts +15 -0
- package/src/calls/voice-session-bridge.ts +71 -16
- package/src/calls/voice-triage-escalate.ts +104 -2
- package/src/channels/__tests__/plugin-channel-declarations.test.ts +161 -0
- package/src/channels/config.ts +13 -0
- package/src/channels/plugin-channel-declarations.ts +108 -0
- package/src/channels/types.ts +30 -0
- package/src/cli/AGENTS.md +5 -2
- package/src/cli/commands/credentials.help.ts +2 -2
- package/src/cli/commands/inference-providers.ts +1 -1
- package/src/cli/commands/mcp.help.ts +13 -4
- package/src/cli/commands/mcp.ts +9 -0
- package/src/cli/commands/memory/__tests__/memory-v3.test.ts +128 -5
- package/src/cli/commands/memory/index.help.ts +43 -1
- package/src/cli/commands/memory/memory-v3.ts +64 -0
- package/src/cli/commands/stt.help.ts +27 -2
- package/src/cli/lib/__tests__/upgrade-plugin.test.ts +39 -0
- package/src/cli/lib/bundled-marketplace.json +13 -0
- package/src/cli/lib/upgrade-plugin.ts +42 -0
- package/src/config/__tests__/default-profile-catalog.test.ts +34 -2
- package/src/config/__tests__/default-provider.test.ts +6 -1
- package/src/config/__tests__/profile-materialization.test.ts +75 -19
- package/src/config/bundled-skills/acp/SKILL.md +6 -7
- package/src/config/bundled-skills/document-editor/SKILL.md +2 -2
- package/src/config/bundled-skills/document-editor/TOOLS.json +2 -2
- package/src/config/bundled-skills/media-processing/services/preprocess.ts +14 -4
- package/src/config/bundled-skills/settings/TOOLS.json +3 -3
- package/src/config/bundled-skills/settings/tools/navigate-settings-tab.test.ts +65 -0
- package/src/config/bundled-skills/settings/tools/navigate-settings-tab.ts +7 -1
- package/src/config/bundled-skills/settings/tools/shared.ts +16 -0
- package/src/config/bundled-skills/settings/tools/voice-config-update.ts +19 -1
- package/src/config/bundled-skills/transcribe/tools/transcribe-media.test.ts +22 -1
- package/src/config/bundled-skills/transcribe/tools/transcribe-media.ts +9 -2
- package/src/config/call-site-defaults.ts +4 -5
- package/src/config/default-profile-catalog.ts +83 -12
- package/src/config/default-profile-names.ts +4 -1
- package/src/config/default-provider-resolution.ts +4 -0
- package/src/config/llm-context-resolution.ts +11 -3
- package/src/config/llm-resolver.ts +28 -1
- package/src/config/profile-materialization.ts +70 -22
- package/src/config/schemas/__tests__/live-voice.test.ts +107 -4
- package/src/config/schemas/call-site-catalog.ts +4 -4
- package/src/config/schemas/live-voice.ts +57 -23
- package/src/config/schemas/llm.ts +59 -32
- package/src/config/schemas/mcp.ts +23 -0
- package/src/config/schemas/plugin-updates.ts +6 -2
- package/src/config/schemas/stt.ts +1 -0
- package/src/context/outbound-sanitize.ts +96 -1
- package/src/daemon/__tests__/plugin-mcp-reconcile.test.ts +82 -0
- package/src/daemon/conversation-agent-loop-handlers.ts +15 -10
- package/src/daemon/conversation-agent-loop.ts +17 -6
- package/src/daemon/conversation-messaging.ts +5 -1
- package/src/daemon/conversation-notifiers.ts +9 -1
- package/src/daemon/conversation-process.ts +36 -6
- package/src/daemon/conversation-runtime-assembly.ts +3 -4
- package/src/daemon/conversation-surfaces.ts +27 -5
- package/src/daemon/conversation-tool-setup.ts +1 -2
- package/src/daemon/conversation.ts +48 -0
- package/src/daemon/interactive-turn-sender.ts +59 -0
- package/src/daemon/mcp-reload-service.ts +36 -6
- package/src/daemon/process-message.ts +24 -24
- package/src/daemon/providers-setup.ts +6 -3
- package/src/daemon/trust-context-types.ts +29 -0
- package/src/daemon/wake-conversation-ops.ts +3 -2
- package/src/documents/document-store.ts +138 -5
- package/src/hooks/hook-loader.ts +3 -3
- package/src/hooks/registry.ts +50 -6
- package/src/inbound/__tests__/oauth-callback-url.test.ts +83 -0
- package/src/inbound/oauth-callback-url.ts +61 -0
- package/src/live-voice/__tests__/live-voice-agent-turn.test.ts +1 -104
- package/src/live-voice/__tests__/live-voice-events.test.ts +7 -8
- package/src/live-voice/__tests__/live-voice-flux-turn-end.test.ts +932 -0
- package/src/live-voice/__tests__/live-voice-metrics.test.ts +115 -8
- package/src/live-voice/__tests__/live-voice-photo.test.ts +100 -0
- package/src/live-voice/__tests__/live-voice-progress.test.ts +60 -194
- package/src/live-voice/__tests__/live-voice-stt.test.ts +14 -0
- package/src/live-voice/__tests__/live-voice-triage-escalate.test.ts +29 -0
- package/src/live-voice/__tests__/live-voice-tts-session.test.ts +0 -483
- package/src/live-voice/__tests__/live-voice-vad.test.ts +0 -16
- package/src/live-voice/__tests__/progress-narration.test.ts +214 -0
- package/src/live-voice/live-voice-archive.ts +2 -0
- package/src/live-voice/live-voice-metrics.ts +57 -32
- package/src/live-voice/live-voice-photo.ts +1 -2
- package/src/live-voice/live-voice-session.ts +535 -314
- package/src/live-voice/progress-narration.ts +277 -0
- package/src/live-voice/protocol.ts +21 -1
- package/src/mcp/__tests__/effective-config.test.ts +238 -0
- package/src/mcp/__tests__/mcp-auth-orchestrator.test.ts +0 -1
- package/src/mcp/__tests__/mcp-oauth-client-registration.test.ts +200 -0
- package/src/mcp/__tests__/mcp-oauth-provider.test.ts +9 -9
- package/src/mcp/__tests__/plugin-server-credential-isolation.test.ts +95 -0
- package/src/mcp/client.ts +16 -11
- package/src/mcp/effective-config.ts +113 -0
- package/src/mcp/manager.ts +11 -6
- package/src/mcp/mcp-auth-orchestrator.ts +13 -22
- package/src/mcp/mcp-oauth-provider.ts +205 -240
- package/src/monitoring/__tests__/plugin-auto-update.test.ts +166 -3
- package/src/monitoring/plugin-auto-update.ts +128 -24
- package/src/notifications/signal.ts +1 -0
- package/src/permissions/confirmation-guardian-request.test.ts +15 -11
- package/src/permissions/confirmation-guardian-request.ts +2 -2
- package/src/permissions/question-guardian-request.test.ts +14 -6
- package/src/permissions/question-guardian-request.ts +1 -2
- package/src/persistence/attachments-store.ts +8 -1
- package/src/persistence/bookmark-crud.ts +3 -7
- package/src/persistence/conversation-attention-store.ts +16 -45
- package/src/persistence/conversation-crud.ts +33 -4
- package/src/persistence/conversation-lineage.ts +9 -0
- package/src/persistence/conversation-queries.ts +108 -41
- package/src/persistence/delivery-crud.ts +38 -29
- package/src/persistence/external-conversation-store.ts +32 -4
- package/src/persistence/llm-request-log-store.ts +4 -10
- package/src/persistence/llm-usage-store.ts +8 -3
- package/src/persistence/message-reads.test.ts +197 -0
- package/src/persistence/message-reads.ts +211 -0
- package/src/persistence/migrations/366-chatgpt-subscription-row-identity.test.ts +120 -0
- package/src/persistence/migrations/366-chatgpt-subscription-row-identity.ts +62 -0
- package/src/persistence/real-user-turn-filter.ts +27 -3
- package/src/persistence/steps.ts +9 -0
- package/src/plugin-api/__tests__/oauth-callback-url-export.test.ts +29 -0
- package/src/plugin-api/conversation-turn.ts +168 -5
- package/src/plugin-api/index.ts +21 -5
- package/src/plugin-api/vision-support.test.ts +1 -1
- package/src/plugins/__tests__/mcp-servers.test.ts +371 -0
- package/src/plugins/defaults/memory/AGENTS.md +4 -0
- package/src/plugins/defaults/memory/__tests__/buffer-format.test.ts +204 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-accounting.test.ts +72 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-job.test.ts +4 -1
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-provider-path.test.ts +4 -1
- package/src/plugins/defaults/memory/buffer-format.ts +165 -0
- package/src/plugins/defaults/memory/context-search/sources/conversations.ts +6 -0
- package/src/plugins/defaults/memory/graph/image-ref-utils.ts +3 -0
- package/src/plugins/defaults/memory/graph/tool-handlers.ts +1 -30
- package/src/plugins/defaults/memory/graph-topology/pending-buffer.test.ts +34 -0
- package/src/plugins/defaults/memory/graph-topology/pending-buffer.ts +8 -12
- package/src/plugins/defaults/memory/hooks/post-compact.ts +1 -4
- package/src/plugins/defaults/memory/indexer.ts +3 -1
- package/src/plugins/defaults/memory/memory-retrospective-accounting.ts +19 -7
- package/src/plugins/defaults/memory/src/__tests__/memory-v3-gate-stats.test.ts +281 -0
- package/src/plugins/defaults/memory/src/memory-v3-routes.ts +207 -0
- package/src/plugins/defaults/memory/substrate/__tests__/consolidation-job.test.ts +33 -0
- package/src/plugins/defaults/memory/substrate/__tests__/static-context.test.ts +199 -2
- package/src/plugins/defaults/memory/substrate/consolidation-job.ts +25 -18
- package/src/plugins/defaults/memory/substrate/skill-content.ts +8 -1
- package/src/plugins/defaults/memory/substrate/static-context.ts +160 -4
- package/src/plugins/defaults/memory/substrate/sweep-job.ts +2 -4
- package/src/plugins/defaults/memory/v1/graph/extraction.ts +3 -1
- package/src/plugins/defaults/memory/v3/prune.ts +2 -0
- package/src/plugins/defaults/memory/v3/selection-log-store.ts +2 -0
- package/src/plugins/defaults/memory/worker.ts +6 -3
- package/src/plugins/external-plugin-loader.ts +47 -0
- package/src/plugins/mcp-servers.ts +361 -0
- package/src/plugins/mtime-cache.ts +23 -49
- package/src/plugins/worker-plugin-surface.ts +33 -0
- package/src/providers/__tests__/connection-model-compat.test.ts +1 -1
- package/src/providers/__tests__/dispatch-connection-routing.test.ts +214 -2
- package/src/providers/__tests__/preflight-resolved-config.test.ts +57 -0
- package/src/providers/__tests__/retry-callsite.test.ts +5 -2
- package/src/providers/call-site-routing.ts +30 -3
- package/src/providers/connection-resolution.ts +194 -11
- package/src/providers/inference/auth.ts +6 -0
- package/src/providers/inference/connection-availability.ts +24 -2
- package/src/providers/inference/connections.ts +2 -0
- package/src/providers/model-catalog.ts +3 -3
- package/src/providers/model-intents.ts +28 -8
- package/src/providers/openai/chat-completions-provider.ts +5 -6
- package/src/providers/openai/codex-models.ts +2 -1
- package/src/providers/provider-send-message.ts +32 -3
- package/src/providers/speech-to-text/__tests__/deepgram-flux-frames.test.ts +433 -0
- package/src/providers/speech-to-text/__tests__/deepgram-flux-realtime.test.ts +620 -0
- package/src/providers/speech-to-text/__tests__/provider-catalog.test.ts +34 -0
- package/src/providers/speech-to-text/__tests__/resolve.test.ts +285 -6
- package/src/providers/speech-to-text/deepgram-flux-frames.ts +395 -0
- package/src/providers/speech-to-text/deepgram-flux-realtime.ts +719 -0
- package/src/providers/speech-to-text/provider-catalog.ts +99 -8
- package/src/providers/speech-to-text/resolve.ts +25 -2
- package/src/routes/worker.ts +17 -5
- package/src/runtime/access-request-helper.ts +9 -12
- package/src/runtime/agent-wake.ts +3 -3
- package/src/runtime/pre-first-message-gate.ts +4 -0
- package/src/runtime/routes/__tests__/acp-claude-auth-routes.test.ts +12 -4
- package/src/runtime/routes/__tests__/conversation-list-routes.test.ts +219 -1
- package/src/runtime/routes/__tests__/conversation-query-routes.test.ts +52 -0
- package/src/runtime/routes/__tests__/default-provider-routes.test.ts +61 -0
- package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +44 -0
- package/src/runtime/routes/__tests__/inference-provider-connection-routes.test.ts +102 -1
- package/src/runtime/routes/__tests__/plugins-routes.test.ts +44 -0
- package/src/runtime/routes/__tests__/stt-routes.test.ts +25 -0
- package/src/runtime/routes/__tests__/user-route-dispatcher.test.ts +62 -1
- package/src/runtime/routes/channel-availability-routes.ts +32 -14
- package/src/runtime/routes/channel-route-shared.ts +0 -6
- package/src/runtime/routes/chatgpt-subscription-auth-routes.ts +6 -6
- package/src/runtime/routes/conversation-list-routes.ts +112 -1
- package/src/runtime/routes/conversation-query-routes.ts +40 -27
- package/src/runtime/routes/credential-prompt-routes.ts +4 -7
- package/src/runtime/routes/default-provider-routes.ts +15 -0
- package/src/runtime/routes/inbound-message-handler.ts +17 -41
- package/src/runtime/routes/inbound-stages/acl-enforcement.test.ts +0 -1
- package/src/runtime/routes/inbound-stages/acl-enforcement.ts +0 -9
- package/src/runtime/routes/inbound-stages/admission-policy.ts +1 -17
- package/src/runtime/routes/inbound-stages/bootstrap-intercept.test.ts +0 -1
- package/src/runtime/routes/inbound-stages/bootstrap-intercept.ts +2 -3
- package/src/runtime/routes/inbound-stages/edit-intercept.ts +1 -3
- package/src/runtime/routes/inbound-stages/guardian-reply-intercept.test.ts +0 -1
- package/src/runtime/routes/inbound-stages/guardian-reply-intercept.ts +3 -4
- package/src/runtime/routes/inbound-stages/reaction-intercept.test.ts +0 -1
- package/src/runtime/routes/inbound-stages/reaction-intercept.ts +11 -20
- package/src/runtime/routes/inbound-stages/secret-ingress-check.ts +2 -3
- package/src/runtime/routes/inference-profiles-routes.ts +20 -11
- package/src/runtime/routes/inference-provider-connection-routes.ts +77 -15
- package/src/runtime/routes/log-export-routes.ts +3 -0
- package/src/runtime/routes/mcp-auth-routes.ts +148 -57
- package/src/runtime/routes/plugins-routes.ts +21 -3
- package/src/runtime/routes/stt-routes.ts +31 -25
- package/src/runtime/routes/surface-conversation-resolver.ts +3 -0
- package/src/runtime/routes/user-route-dispatcher.ts +39 -14
- package/src/runtime/routes/user-route-import.ts +108 -0
- package/src/schedule/worker.ts +6 -0
- package/src/security/oauth2.ts +6 -22
- package/src/stt/__tests__/daemon-batch-transcriber.test.ts +22 -0
- package/src/stt/__tests__/types.test.ts +94 -0
- package/src/stt/daemon-batch-transcriber.ts +10 -0
- package/src/stt/stt-stream-session.ts +8 -4
- package/src/stt/types.ts +103 -0
- package/src/subagent/manager.ts +1 -3
- package/src/subagent/types.ts +7 -6
- package/src/tools/acp/spawn.test.ts +97 -0
- package/src/tools/acp/spawn.ts +32 -0
- package/src/tools/document/document-tool.ts +12 -3
- package/src/tools/registry.ts +2 -1
- package/src/tools/ui-surface/surface-shape-docs.ts +11 -0
- package/src/tools/workflows/run-workflow.ts +1 -2
- package/src/tts/__tests__/reasoning-tag-filter.test.ts +78 -0
- package/src/tts/reasoning-tag-filter.ts +89 -0
- package/src/util/think-tag-stream.ts +95 -0
- package/src/workspace/byok-default-profile-ensure.ts +76 -24
- package/src/workspace/custom-profile-ensure.ts +4 -24
- package/src/workspace/migrations/142-consolidate-voice-front-door.ts +70 -0
- package/src/workspace/migrations/143-repair-deprecated-codex-model-id.ts +134 -0
- package/src/workspace/migrations/144-convert-stranded-subscription-openai-profiles.ts +265 -0
- package/src/workspace/migrations/145-collapse-profile-bindings-to-entries.ts +328 -0
- package/src/workspace/migrations/146-repair-retired-fireworks-deepseek-flash-model-id.ts +195 -0
- package/src/workspace/migrations/__tests__/141-stt-english-default-to-multilingual.test.ts +0 -10
- package/src/workspace/migrations/registry.ts +10 -0
- package/src/workspace/provider-commit-message-generator.ts +7 -5
- package/src/live-voice/__tests__/front-decision.test.ts +0 -645
- package/src/live-voice/front-decision.ts +0 -476
|
@@ -1,20 +1,13 @@
|
|
|
1
1
|
import { describe, expect, mock, test } from "bun:test";
|
|
2
2
|
|
|
3
|
-
import { sanitizeForTts } from "../../calls/tts-text-sanitizer.js";
|
|
4
3
|
import type {
|
|
5
4
|
VoiceTurnCallbacks,
|
|
6
5
|
VoiceTurnOptions,
|
|
7
6
|
} from "../../calls/voice-session-bridge.js";
|
|
8
|
-
import { FALLBACK_ESCALATION_BRIDGE } from "../../calls/voice-triage-escalate.js";
|
|
9
|
-
import type { LiveVoiceFrontModelConfig } from "../../config/schemas/live-voice.js";
|
|
10
7
|
import type {
|
|
11
8
|
StreamingTranscriber,
|
|
12
9
|
SttStreamServerEvent,
|
|
13
10
|
} from "../../stt/types.js";
|
|
14
|
-
import type {
|
|
15
|
-
VoiceAckTextInput,
|
|
16
|
-
VoiceFrontDecider,
|
|
17
|
-
} from "../front-decision.js";
|
|
18
11
|
import {
|
|
19
12
|
LiveVoiceSession,
|
|
20
13
|
type LiveVoiceTtsStreamer,
|
|
@@ -93,8 +86,6 @@ function createContext(): {
|
|
|
93
86
|
function createSessionHarness(options: {
|
|
94
87
|
startVoiceTurn: LiveVoiceTurnStarter;
|
|
95
88
|
streamTtsAudio: LiveVoiceTtsStreamer | null;
|
|
96
|
-
frontModelConfig?: Partial<LiveVoiceFrontModelConfig>;
|
|
97
|
-
frontDecider?: VoiceFrontDecider | null;
|
|
98
89
|
}) {
|
|
99
90
|
const transcriber = new MockStreamingTranscriber();
|
|
100
91
|
const { context, frames } = createContext();
|
|
@@ -102,12 +93,6 @@ function createSessionHarness(options: {
|
|
|
102
93
|
resolveTranscriber: mock(async () => transcriber),
|
|
103
94
|
startVoiceTurn: options.startVoiceTurn,
|
|
104
95
|
streamTtsAudio: options.streamTtsAudio,
|
|
105
|
-
...(options.frontModelConfig
|
|
106
|
-
? { frontModelConfig: options.frontModelConfig }
|
|
107
|
-
: {}),
|
|
108
|
-
...(options.frontDecider !== undefined
|
|
109
|
-
? { frontDecider: options.frontDecider }
|
|
110
|
-
: {}),
|
|
111
96
|
createTurnId: () => "live-turn-1",
|
|
112
97
|
});
|
|
113
98
|
|
|
@@ -756,471 +741,3 @@ describe("LiveVoiceSession TTS", () => {
|
|
|
756
741
|
});
|
|
757
742
|
});
|
|
758
743
|
});
|
|
759
|
-
|
|
760
|
-
describe("LiveVoiceSession spoken ack", () => {
|
|
761
|
-
const ACK_TIMEOUT_MS = 40;
|
|
762
|
-
// Every ack is front-model-phrased; the generated text passes through the
|
|
763
|
-
// same TTS sanitizer as regular segments.
|
|
764
|
-
const GENERATED_ACK = "Sure — one moment.";
|
|
765
|
-
const EXPECTED_ACK = sanitizeForTts(GENERATED_ACK).trim();
|
|
766
|
-
|
|
767
|
-
const EXPECTED_BRIDGE = sanitizeForTts(FALLBACK_ESCALATION_BRIDGE).trim();
|
|
768
|
-
|
|
769
|
-
function makeStubFrontDecider(
|
|
770
|
-
generateAckText: VoiceFrontDecider["generateAckText"] = async () =>
|
|
771
|
-
GENERATED_ACK,
|
|
772
|
-
): VoiceFrontDecider {
|
|
773
|
-
return {
|
|
774
|
-
generateAckText,
|
|
775
|
-
generateProgressText: async () => null,
|
|
776
|
-
};
|
|
777
|
-
}
|
|
778
|
-
|
|
779
|
-
function createCapturingTurnStarter(): {
|
|
780
|
-
startVoiceTurn: LiveVoiceTurnStarter;
|
|
781
|
-
getCallbacks: () => VoiceTurnCallbacks | undefined;
|
|
782
|
-
} {
|
|
783
|
-
let callbacks: VoiceTurnCallbacks | undefined;
|
|
784
|
-
const startVoiceTurn = mock(async (options: VoiceTurnOptions) => {
|
|
785
|
-
callbacks = options.callbacks;
|
|
786
|
-
return { turnId: "bridge-turn-1", abort: mock() };
|
|
787
|
-
});
|
|
788
|
-
return { startVoiceTurn, getCallbacks: () => callbacks };
|
|
789
|
-
}
|
|
790
|
-
|
|
791
|
-
function createRecordingTtsStreamer(): {
|
|
792
|
-
streamTtsAudio: LiveVoiceTtsStreamer;
|
|
793
|
-
ttsTexts: string[];
|
|
794
|
-
} {
|
|
795
|
-
const ttsTexts: string[] = [];
|
|
796
|
-
const streamTtsAudio = mock(async (options: LiveVoiceTtsOptions) => {
|
|
797
|
-
ttsTexts.push(options.text);
|
|
798
|
-
options.onAudioChunk(makeTtsChunk(`audio:${options.text}`));
|
|
799
|
-
return makeTtsResult(options.text);
|
|
800
|
-
});
|
|
801
|
-
return { streamTtsAudio, ttsTexts };
|
|
802
|
-
}
|
|
803
|
-
|
|
804
|
-
test("speaks exactly one audio-only ack before agent audio on a slow first delta", async () => {
|
|
805
|
-
const { startVoiceTurn, getCallbacks } = createCapturingTurnStarter();
|
|
806
|
-
const { streamTtsAudio, ttsTexts } = createRecordingTtsStreamer();
|
|
807
|
-
const { frames, session } = createSessionHarness({
|
|
808
|
-
startVoiceTurn,
|
|
809
|
-
streamTtsAudio,
|
|
810
|
-
frontModelConfig: { ackFirstDeltaTimeoutMs: ACK_TIMEOUT_MS },
|
|
811
|
-
frontDecider: makeStubFrontDecider(),
|
|
812
|
-
});
|
|
813
|
-
|
|
814
|
-
await startReleasedTurn(session);
|
|
815
|
-
await waitFor(() =>
|
|
816
|
-
frames.some(
|
|
817
|
-
(frame) =>
|
|
818
|
-
frame.type === "tts_audio" &&
|
|
819
|
-
frame.dataBase64 === b64(`audio:${EXPECTED_ACK}`),
|
|
820
|
-
),
|
|
821
|
-
);
|
|
822
|
-
|
|
823
|
-
// Audio-only: no caption frame carries the ack text, and the turn is
|
|
824
|
-
// still open (no tts_done).
|
|
825
|
-
expect(ttsTexts).toEqual([EXPECTED_ACK]);
|
|
826
|
-
expect(frames.some((frame) => frame.type === "assistant_text_delta")).toBe(
|
|
827
|
-
false,
|
|
828
|
-
);
|
|
829
|
-
expect(frames.some((frame) => frame.type === "tts_done")).toBe(false);
|
|
830
|
-
|
|
831
|
-
getCallbacks()?.assistant_text_delta?.(makeTextDelta("Hello there."));
|
|
832
|
-
getCallbacks()?.message_complete?.(makeMessageComplete());
|
|
833
|
-
await waitFor(() => frames.some((frame) => frame.type === "tts_done"));
|
|
834
|
-
|
|
835
|
-
expect(ttsTexts).toEqual([EXPECTED_ACK, "Hello there."]);
|
|
836
|
-
expect(ttsAudioPayloads(frames)).toEqual([
|
|
837
|
-
b64(`audio:${EXPECTED_ACK}`),
|
|
838
|
-
b64("audio:Hello there."),
|
|
839
|
-
]);
|
|
840
|
-
expect(frames.filter((frame) => frame.type === "tts_done")).toHaveLength(1);
|
|
841
|
-
expect(frames.at(-1)).toMatchObject({
|
|
842
|
-
type: "tts_done",
|
|
843
|
-
turnId: "live-turn-1",
|
|
844
|
-
});
|
|
845
|
-
});
|
|
846
|
-
|
|
847
|
-
test("speaks no ack when the first delta beats the timeout", async () => {
|
|
848
|
-
const { startVoiceTurn, getCallbacks } = createCapturingTurnStarter();
|
|
849
|
-
const { streamTtsAudio, ttsTexts } = createRecordingTtsStreamer();
|
|
850
|
-
const { frames, session } = createSessionHarness({
|
|
851
|
-
startVoiceTurn,
|
|
852
|
-
streamTtsAudio,
|
|
853
|
-
frontModelConfig: { ackFirstDeltaTimeoutMs: ACK_TIMEOUT_MS },
|
|
854
|
-
});
|
|
855
|
-
|
|
856
|
-
await startReleasedTurn(session);
|
|
857
|
-
getCallbacks()?.assistant_text_delta?.(makeTextDelta("Hello there."));
|
|
858
|
-
getCallbacks()?.message_complete?.(makeMessageComplete());
|
|
859
|
-
await waitFor(() => frames.some((frame) => frame.type === "tts_done"));
|
|
860
|
-
await new Promise((resolve) => setTimeout(resolve, ACK_TIMEOUT_MS + 40));
|
|
861
|
-
|
|
862
|
-
expect(ttsTexts).toEqual(["Hello there."]);
|
|
863
|
-
expect(ttsAudioPayloads(frames)).toEqual([b64("audio:Hello there.")]);
|
|
864
|
-
});
|
|
865
|
-
|
|
866
|
-
test("interrupt before the timeout clears the timer and speaks no ack", async () => {
|
|
867
|
-
const { startVoiceTurn } = createCapturingTurnStarter();
|
|
868
|
-
const { streamTtsAudio, ttsTexts } = createRecordingTtsStreamer();
|
|
869
|
-
const { frames, session } = createSessionHarness({
|
|
870
|
-
startVoiceTurn,
|
|
871
|
-
streamTtsAudio,
|
|
872
|
-
frontModelConfig: { ackFirstDeltaTimeoutMs: ACK_TIMEOUT_MS },
|
|
873
|
-
});
|
|
874
|
-
|
|
875
|
-
await startReleasedTurn(session);
|
|
876
|
-
await session.handleClientFrame({ type: "interrupt" });
|
|
877
|
-
await new Promise((resolve) => setTimeout(resolve, ACK_TIMEOUT_MS + 40));
|
|
878
|
-
|
|
879
|
-
expect(ttsTexts).toEqual([]);
|
|
880
|
-
expect(frames.some((frame) => frame.type === "tts_audio")).toBe(false);
|
|
881
|
-
});
|
|
882
|
-
|
|
883
|
-
test("arms no ack without a TTS streamer", async () => {
|
|
884
|
-
const { startVoiceTurn, getCallbacks } = createCapturingTurnStarter();
|
|
885
|
-
const { frames, session } = createSessionHarness({
|
|
886
|
-
startVoiceTurn,
|
|
887
|
-
streamTtsAudio: null,
|
|
888
|
-
frontModelConfig: { ackFirstDeltaTimeoutMs: ACK_TIMEOUT_MS },
|
|
889
|
-
});
|
|
890
|
-
|
|
891
|
-
await startReleasedTurn(session);
|
|
892
|
-
await new Promise((resolve) => setTimeout(resolve, ACK_TIMEOUT_MS + 40));
|
|
893
|
-
expect(frames.some((frame) => frame.type === "tts_audio")).toBe(false);
|
|
894
|
-
|
|
895
|
-
getCallbacks()?.assistant_text_delta?.(makeTextDelta("Hello there."));
|
|
896
|
-
getCallbacks()?.message_complete?.(makeMessageComplete());
|
|
897
|
-
await waitFor(() => frames.some((frame) => frame.type === "tts_done"));
|
|
898
|
-
expect(frames.some((frame) => frame.type === "tts_audio")).toBe(false);
|
|
899
|
-
});
|
|
900
|
-
|
|
901
|
-
test("keeps the eager first-clause flush for the model's first segment after an ack", async () => {
|
|
902
|
-
const { startVoiceTurn, getCallbacks } = createCapturingTurnStarter();
|
|
903
|
-
const { streamTtsAudio, ttsTexts } = createRecordingTtsStreamer();
|
|
904
|
-
const { frames, session } = createSessionHarness({
|
|
905
|
-
startVoiceTurn,
|
|
906
|
-
streamTtsAudio,
|
|
907
|
-
frontModelConfig: { ackFirstDeltaTimeoutMs: ACK_TIMEOUT_MS },
|
|
908
|
-
frontDecider: makeStubFrontDecider(),
|
|
909
|
-
});
|
|
910
|
-
|
|
911
|
-
await startReleasedTurn(session);
|
|
912
|
-
await waitFor(() => ttsTexts.length === 1);
|
|
913
|
-
expect(ttsTexts).toEqual([EXPECTED_ACK]);
|
|
914
|
-
|
|
915
|
-
getCallbacks()?.assistant_text_delta?.(
|
|
916
|
-
makeTextDelta(
|
|
917
|
-
"Sure, I can help with that, and here is more text to say.",
|
|
918
|
-
),
|
|
919
|
-
);
|
|
920
|
-
await waitFor(() => ttsTexts.length === 3);
|
|
921
|
-
|
|
922
|
-
// The ack did not consume the eager first-segment flush: the model's
|
|
923
|
-
// opening clause still splits at the comma instead of buffering to a
|
|
924
|
-
// full sentence.
|
|
925
|
-
expect(ttsTexts).toEqual([
|
|
926
|
-
EXPECTED_ACK,
|
|
927
|
-
"Sure, I can help with that,",
|
|
928
|
-
"and here is more text to say.",
|
|
929
|
-
]);
|
|
930
|
-
|
|
931
|
-
getCallbacks()?.message_complete?.(makeMessageComplete());
|
|
932
|
-
await waitFor(() => frames.some((frame) => frame.type === "tts_done"));
|
|
933
|
-
expect(frames.at(-1)).toMatchObject({
|
|
934
|
-
type: "tts_done",
|
|
935
|
-
turnId: "live-turn-1",
|
|
936
|
-
});
|
|
937
|
-
});
|
|
938
|
-
|
|
939
|
-
test("speaks the front-model-phrased ack", async () => {
|
|
940
|
-
const GENERATED = "Sure — one moment.";
|
|
941
|
-
// The generated text passes through the same TTS sanitizer as the
|
|
942
|
-
// static phrase.
|
|
943
|
-
const EXPECTED_GENERATED = sanitizeForTts(GENERATED).trim();
|
|
944
|
-
const generateAckText = mock(async (ackInput: VoiceAckTextInput) => {
|
|
945
|
-
expect(ackInput.transcriptSoFar).toBe("hello");
|
|
946
|
-
return GENERATED;
|
|
947
|
-
});
|
|
948
|
-
const { startVoiceTurn, getCallbacks } = createCapturingTurnStarter();
|
|
949
|
-
const { streamTtsAudio, ttsTexts } = createRecordingTtsStreamer();
|
|
950
|
-
const { frames, session } = createSessionHarness({
|
|
951
|
-
startVoiceTurn,
|
|
952
|
-
streamTtsAudio,
|
|
953
|
-
frontModelConfig: {
|
|
954
|
-
ackFirstDeltaTimeoutMs: ACK_TIMEOUT_MS,
|
|
955
|
-
},
|
|
956
|
-
frontDecider: makeStubFrontDecider(generateAckText),
|
|
957
|
-
});
|
|
958
|
-
|
|
959
|
-
await startReleasedTurn(session);
|
|
960
|
-
await waitFor(() => ttsTexts.length === 1);
|
|
961
|
-
expect(ttsTexts).toEqual([EXPECTED_GENERATED]);
|
|
962
|
-
expect(generateAckText).toHaveBeenCalledTimes(1);
|
|
963
|
-
|
|
964
|
-
getCallbacks()?.assistant_text_delta?.(makeTextDelta("Hello there."));
|
|
965
|
-
getCallbacks()?.message_complete?.(makeMessageComplete());
|
|
966
|
-
await waitFor(() => frames.some((frame) => frame.type === "tts_done"));
|
|
967
|
-
expect(ttsTexts).toEqual([EXPECTED_GENERATED, "Hello there."]);
|
|
968
|
-
});
|
|
969
|
-
|
|
970
|
-
test("a null generation speaks no ack at all", async () => {
|
|
971
|
-
const generateAckText = mock(async () => null);
|
|
972
|
-
const { startVoiceTurn, getCallbacks } = createCapturingTurnStarter();
|
|
973
|
-
const { streamTtsAudio, ttsTexts } = createRecordingTtsStreamer();
|
|
974
|
-
const { frames, session } = createSessionHarness({
|
|
975
|
-
startVoiceTurn,
|
|
976
|
-
streamTtsAudio,
|
|
977
|
-
frontModelConfig: {
|
|
978
|
-
ackFirstDeltaTimeoutMs: ACK_TIMEOUT_MS,
|
|
979
|
-
},
|
|
980
|
-
frontDecider: makeStubFrontDecider(generateAckText),
|
|
981
|
-
});
|
|
982
|
-
|
|
983
|
-
await startReleasedTurn(session);
|
|
984
|
-
await waitFor(() => generateAckText.mock.calls.length === 1);
|
|
985
|
-
await flushAsyncCallbacks();
|
|
986
|
-
|
|
987
|
-
// There is no canned fallback: an unusable generation leaves the floor
|
|
988
|
-
// silent until the model's own audio arrives.
|
|
989
|
-
expect(ttsTexts).toEqual([]);
|
|
990
|
-
expect(frames.some((frame) => frame.type === "tts_audio")).toBe(false);
|
|
991
|
-
|
|
992
|
-
getCallbacks()?.assistant_text_delta?.(makeTextDelta("Hello there."));
|
|
993
|
-
getCallbacks()?.message_complete?.(makeMessageComplete());
|
|
994
|
-
await waitFor(() => frames.some((frame) => frame.type === "tts_done"));
|
|
995
|
-
expect(ttsTexts).toEqual(["Hello there."]);
|
|
996
|
-
});
|
|
997
|
-
|
|
998
|
-
test("without a decider no ack is spoken", async () => {
|
|
999
|
-
const { startVoiceTurn, getCallbacks } = createCapturingTurnStarter();
|
|
1000
|
-
const { streamTtsAudio, ttsTexts } = createRecordingTtsStreamer();
|
|
1001
|
-
const { frames, session } = createSessionHarness({
|
|
1002
|
-
startVoiceTurn,
|
|
1003
|
-
streamTtsAudio,
|
|
1004
|
-
frontModelConfig: { ackFirstDeltaTimeoutMs: ACK_TIMEOUT_MS },
|
|
1005
|
-
frontDecider: null,
|
|
1006
|
-
});
|
|
1007
|
-
|
|
1008
|
-
await startReleasedTurn(session);
|
|
1009
|
-
await new Promise((resolve) => setTimeout(resolve, ACK_TIMEOUT_MS + 40));
|
|
1010
|
-
expect(ttsTexts).toEqual([]);
|
|
1011
|
-
|
|
1012
|
-
getCallbacks()?.assistant_text_delta?.(makeTextDelta("Hello there."));
|
|
1013
|
-
getCallbacks()?.message_complete?.(makeMessageComplete());
|
|
1014
|
-
await waitFor(() => frames.some((frame) => frame.type === "tts_done"));
|
|
1015
|
-
expect(ttsTexts).toEqual(["Hello there."]);
|
|
1016
|
-
});
|
|
1017
|
-
|
|
1018
|
-
test("a generation resolving around turn completion speaks no ack", async () => {
|
|
1019
|
-
let resolveGeneration!: (text: string | null) => void;
|
|
1020
|
-
const generateAckText = mock(
|
|
1021
|
-
() =>
|
|
1022
|
-
new Promise<string | null>((resolve) => {
|
|
1023
|
-
resolveGeneration = resolve;
|
|
1024
|
-
}),
|
|
1025
|
-
);
|
|
1026
|
-
const { startVoiceTurn, getCallbacks } = createCapturingTurnStarter();
|
|
1027
|
-
const { streamTtsAudio, ttsTexts } = createRecordingTtsStreamer();
|
|
1028
|
-
const { frames, session } = createSessionHarness({
|
|
1029
|
-
startVoiceTurn,
|
|
1030
|
-
streamTtsAudio,
|
|
1031
|
-
frontModelConfig: {
|
|
1032
|
-
ackFirstDeltaTimeoutMs: ACK_TIMEOUT_MS,
|
|
1033
|
-
},
|
|
1034
|
-
frontDecider: makeStubFrontDecider(generateAckText),
|
|
1035
|
-
});
|
|
1036
|
-
|
|
1037
|
-
await startReleasedTurn(session);
|
|
1038
|
-
await waitFor(() => generateAckText.mock.calls.length === 1);
|
|
1039
|
-
|
|
1040
|
-
// A tool-only/no-text turn completes while the ack text is still
|
|
1041
|
-
// generating: the resolving generation must not become the turn's only
|
|
1042
|
-
// audible output (nor enqueue behind the finale).
|
|
1043
|
-
resolveGeneration("Too late to say this.");
|
|
1044
|
-
getCallbacks()?.message_complete?.(makeMessageComplete());
|
|
1045
|
-
await waitFor(() => frames.some((frame) => frame.type === "tts_done"));
|
|
1046
|
-
await flushAsyncCallbacks();
|
|
1047
|
-
|
|
1048
|
-
// No ack synthesized or sent — and since the ack-spoken metric only
|
|
1049
|
-
// records on an actual enqueue, no phantom ack metric either.
|
|
1050
|
-
expect(ttsTexts).toEqual([]);
|
|
1051
|
-
expect(frames.some((frame) => frame.type === "tts_audio")).toBe(false);
|
|
1052
|
-
});
|
|
1053
|
-
|
|
1054
|
-
test("a tool-use ack passes the tool name to the decider", async () => {
|
|
1055
|
-
const GENERATED = "Let me search for that.";
|
|
1056
|
-
const generateAckText = mock(
|
|
1057
|
-
async (_input: VoiceAckTextInput) => GENERATED,
|
|
1058
|
-
);
|
|
1059
|
-
const { startVoiceTurn, getCallbacks } = createCapturingTurnStarter();
|
|
1060
|
-
const { streamTtsAudio, ttsTexts } = createRecordingTtsStreamer();
|
|
1061
|
-
const { frames, session } = createSessionHarness({
|
|
1062
|
-
startVoiceTurn,
|
|
1063
|
-
streamTtsAudio,
|
|
1064
|
-
// Long first-delta budget so only the tool-use trigger can speak.
|
|
1065
|
-
frontModelConfig: { ackFirstDeltaTimeoutMs: 5_000 },
|
|
1066
|
-
frontDecider: makeStubFrontDecider(generateAckText),
|
|
1067
|
-
});
|
|
1068
|
-
|
|
1069
|
-
await startReleasedTurn(session);
|
|
1070
|
-
getCallbacks()?.tool_use_start?.("web_search");
|
|
1071
|
-
await waitFor(() => ttsTexts.length === 1);
|
|
1072
|
-
|
|
1073
|
-
expect(generateAckText).toHaveBeenCalledTimes(1);
|
|
1074
|
-
expect(generateAckText.mock.calls[0]?.[0]).toEqual({
|
|
1075
|
-
transcriptSoFar: "hello",
|
|
1076
|
-
toolName: "web_search",
|
|
1077
|
-
});
|
|
1078
|
-
expect(ttsTexts).toEqual([sanitizeForTts(GENERATED).trim()]);
|
|
1079
|
-
|
|
1080
|
-
getCallbacks()?.assistant_text_delta?.(makeTextDelta("Hello there."));
|
|
1081
|
-
getCallbacks()?.message_complete?.(makeMessageComplete());
|
|
1082
|
-
await waitFor(() => frames.some((frame) => frame.type === "tts_done"));
|
|
1083
|
-
});
|
|
1084
|
-
|
|
1085
|
-
test("a generation resolving after escalation hand-off speaks no ack", async () => {
|
|
1086
|
-
let resolveGeneration!: (text: string | null) => void;
|
|
1087
|
-
const generateAckText = mock(
|
|
1088
|
-
() =>
|
|
1089
|
-
new Promise<string | null>((resolve) => {
|
|
1090
|
-
resolveGeneration = resolve;
|
|
1091
|
-
}),
|
|
1092
|
-
);
|
|
1093
|
-
const { startVoiceTurn, getCallbacks } = createCapturingTurnStarter();
|
|
1094
|
-
const { streamTtsAudio, ttsTexts } = createRecordingTtsStreamer();
|
|
1095
|
-
const { frames, session } = createSessionHarness({
|
|
1096
|
-
startVoiceTurn,
|
|
1097
|
-
streamTtsAudio,
|
|
1098
|
-
frontModelConfig: {
|
|
1099
|
-
ackFirstDeltaTimeoutMs: ACK_TIMEOUT_MS,
|
|
1100
|
-
},
|
|
1101
|
-
frontDecider: makeStubFrontDecider(generateAckText),
|
|
1102
|
-
});
|
|
1103
|
-
|
|
1104
|
-
await startReleasedTurn(session);
|
|
1105
|
-
await waitFor(() => generateAckText.mock.calls.length === 1);
|
|
1106
|
-
|
|
1107
|
-
// The front-door leg escalates with no holding phrase of its own, so the
|
|
1108
|
-
// canned bridge is enqueued — exactly the case where a late-resolving
|
|
1109
|
-
// generation would stack a second filler on top of it.
|
|
1110
|
-
getCallbacks()?.assistant_text_delta?.(makeTextDelta("[1]"));
|
|
1111
|
-
getCallbacks()?.message_complete?.(makeMessageComplete());
|
|
1112
|
-
await waitFor(() => ttsTexts.length === 1);
|
|
1113
|
-
expect(ttsTexts).toEqual([EXPECTED_BRIDGE]);
|
|
1114
|
-
|
|
1115
|
-
resolveGeneration("Stacked filler.");
|
|
1116
|
-
await flushAsyncCallbacks();
|
|
1117
|
-
expect(ttsTexts).toEqual([EXPECTED_BRIDGE]);
|
|
1118
|
-
|
|
1119
|
-
// The escalated leg (whose callbacks the capturing starter now holds)
|
|
1120
|
-
// finishes the turn normally.
|
|
1121
|
-
getCallbacks()?.assistant_text_delta?.(
|
|
1122
|
-
makeTextDelta("Here is the careful answer."),
|
|
1123
|
-
);
|
|
1124
|
-
getCallbacks()?.message_complete?.(makeMessageComplete());
|
|
1125
|
-
await waitFor(() => frames.some((frame) => frame.type === "tts_done"));
|
|
1126
|
-
expect(ttsTexts).toEqual([EXPECTED_BRIDGE, "Here is the careful answer."]);
|
|
1127
|
-
});
|
|
1128
|
-
|
|
1129
|
-
test("a tool started on the escalated leg speaks nothing past the bridge", async () => {
|
|
1130
|
-
const { startVoiceTurn, getCallbacks } = createCapturingTurnStarter();
|
|
1131
|
-
const { streamTtsAudio, ttsTexts } = createRecordingTtsStreamer();
|
|
1132
|
-
const { frames, session } = createSessionHarness({
|
|
1133
|
-
startVoiceTurn,
|
|
1134
|
-
streamTtsAudio,
|
|
1135
|
-
// Long first-delta budget so only the tool-use trigger could speak.
|
|
1136
|
-
frontModelConfig: { ackFirstDeltaTimeoutMs: 5_000 },
|
|
1137
|
-
});
|
|
1138
|
-
|
|
1139
|
-
await startReleasedTurn(session);
|
|
1140
|
-
// A bare hand-off enqueues the canned bridge, which holds the floor.
|
|
1141
|
-
getCallbacks()?.assistant_text_delta?.(makeTextDelta("[1]"));
|
|
1142
|
-
getCallbacks()?.message_complete?.(makeMessageComplete());
|
|
1143
|
-
await waitFor(() => ttsTexts.length === 1);
|
|
1144
|
-
expect(ttsTexts).toEqual([EXPECTED_BRIDGE]);
|
|
1145
|
-
|
|
1146
|
-
// The escalated leg starts a tool before its first delta: no static ack
|
|
1147
|
-
// may stack on the bridge.
|
|
1148
|
-
getCallbacks()?.tool_use_start?.("web_search");
|
|
1149
|
-
await flushAsyncCallbacks();
|
|
1150
|
-
expect(ttsTexts).toEqual([EXPECTED_BRIDGE]);
|
|
1151
|
-
|
|
1152
|
-
getCallbacks()?.assistant_text_delta?.(
|
|
1153
|
-
makeTextDelta("Here is the careful answer."),
|
|
1154
|
-
);
|
|
1155
|
-
getCallbacks()?.message_complete?.(makeMessageComplete());
|
|
1156
|
-
await waitFor(() => frames.some((frame) => frame.type === "tts_done"));
|
|
1157
|
-
expect(ttsTexts).toEqual([EXPECTED_BRIDGE, "Here is the careful answer."]);
|
|
1158
|
-
});
|
|
1159
|
-
|
|
1160
|
-
test("tools started on the escalated leg trigger no ack generation", async () => {
|
|
1161
|
-
const generateAckText = mock(async () => "Never spoken.");
|
|
1162
|
-
const { startVoiceTurn, getCallbacks } = createCapturingTurnStarter();
|
|
1163
|
-
const { streamTtsAudio, ttsTexts } = createRecordingTtsStreamer();
|
|
1164
|
-
const { frames, session } = createSessionHarness({
|
|
1165
|
-
startVoiceTurn,
|
|
1166
|
-
streamTtsAudio,
|
|
1167
|
-
// Long first-delta budget so only the tool-use trigger could generate.
|
|
1168
|
-
frontModelConfig: { ackFirstDeltaTimeoutMs: 5_000 },
|
|
1169
|
-
frontDecider: makeStubFrontDecider(generateAckText),
|
|
1170
|
-
});
|
|
1171
|
-
|
|
1172
|
-
await startReleasedTurn(session);
|
|
1173
|
-
getCallbacks()?.assistant_text_delta?.(makeTextDelta("[1]"));
|
|
1174
|
-
getCallbacks()?.message_complete?.(makeMessageComplete());
|
|
1175
|
-
await waitFor(() => ttsTexts.length === 1);
|
|
1176
|
-
expect(ttsTexts).toEqual([EXPECTED_BRIDGE]);
|
|
1177
|
-
|
|
1178
|
-
// Post-handoff tool starts must not each burn a generation call whose
|
|
1179
|
-
// result is always discarded.
|
|
1180
|
-
getCallbacks()?.tool_use_start?.("web_search");
|
|
1181
|
-
getCallbacks()?.tool_use_start?.("file_read");
|
|
1182
|
-
await flushAsyncCallbacks();
|
|
1183
|
-
expect(generateAckText).not.toHaveBeenCalled();
|
|
1184
|
-
expect(ttsTexts).toEqual([EXPECTED_BRIDGE]);
|
|
1185
|
-
|
|
1186
|
-
getCallbacks()?.assistant_text_delta?.(
|
|
1187
|
-
makeTextDelta("Here is the careful answer."),
|
|
1188
|
-
);
|
|
1189
|
-
getCallbacks()?.message_complete?.(makeMessageComplete());
|
|
1190
|
-
await waitFor(() => frames.some((frame) => frame.type === "tts_done"));
|
|
1191
|
-
expect(ttsTexts).toEqual([EXPECTED_BRIDGE, "Here is the careful answer."]);
|
|
1192
|
-
});
|
|
1193
|
-
|
|
1194
|
-
test("a first delta during generation skips the ack entirely", async () => {
|
|
1195
|
-
let resolveGeneration!: (text: string | null) => void;
|
|
1196
|
-
const generateAckText = mock(
|
|
1197
|
-
() =>
|
|
1198
|
-
new Promise<string | null>((resolve) => {
|
|
1199
|
-
resolveGeneration = resolve;
|
|
1200
|
-
}),
|
|
1201
|
-
);
|
|
1202
|
-
const { startVoiceTurn, getCallbacks } = createCapturingTurnStarter();
|
|
1203
|
-
const { streamTtsAudio, ttsTexts } = createRecordingTtsStreamer();
|
|
1204
|
-
const { frames, session } = createSessionHarness({
|
|
1205
|
-
startVoiceTurn,
|
|
1206
|
-
streamTtsAudio,
|
|
1207
|
-
frontModelConfig: {
|
|
1208
|
-
ackFirstDeltaTimeoutMs: ACK_TIMEOUT_MS,
|
|
1209
|
-
},
|
|
1210
|
-
frontDecider: makeStubFrontDecider(generateAckText),
|
|
1211
|
-
});
|
|
1212
|
-
|
|
1213
|
-
await startReleasedTurn(session);
|
|
1214
|
-
await waitFor(() => generateAckText.mock.calls.length === 1);
|
|
1215
|
-
|
|
1216
|
-
// The brain's first delta lands while the ack text is still generating —
|
|
1217
|
-
// a late generation must not speak over real output.
|
|
1218
|
-
getCallbacks()?.assistant_text_delta?.(makeTextDelta("Hello there."));
|
|
1219
|
-
resolveGeneration("Too late to say this.");
|
|
1220
|
-
getCallbacks()?.message_complete?.(makeMessageComplete());
|
|
1221
|
-
await waitFor(() => frames.some((frame) => frame.type === "tts_done"));
|
|
1222
|
-
|
|
1223
|
-
expect(ttsTexts).toEqual(["Hello there."]);
|
|
1224
|
-
expect(ttsAudioPayloads(frames)).toEqual([b64("audio:Hello there.")]);
|
|
1225
|
-
});
|
|
1226
|
-
});
|
|
@@ -19,7 +19,6 @@ import type {
|
|
|
19
19
|
} from "../../stt/types.js";
|
|
20
20
|
import { __resetRegistryForTesting } from "../../tools/registry.js";
|
|
21
21
|
import { getWorkspaceSkillsDir } from "../../util/platform.js";
|
|
22
|
-
import type { VoiceFrontDecider } from "../front-decision.js";
|
|
23
22
|
import type { LiveVoiceAudioArchiveResult } from "../live-voice-archive.js";
|
|
24
23
|
import {
|
|
25
24
|
CONTINUATION_DELIVERY_CONTENT,
|
|
@@ -182,7 +181,6 @@ function createHarness(options: {
|
|
|
182
181
|
echoBargeInMargin?: number;
|
|
183
182
|
echoEmaHalfLifeMs?: number;
|
|
184
183
|
echoDrainSlackMs?: number;
|
|
185
|
-
frontDecider?: VoiceFrontDecider | null;
|
|
186
184
|
frontModelConfig?: Partial<LiveVoiceFrontModelConfig>;
|
|
187
185
|
emitMetrics?: boolean;
|
|
188
186
|
metricsClock?: () => number;
|
|
@@ -264,9 +262,6 @@ function createHarness(options: {
|
|
|
264
262
|
options.echoBargeInMargin ?? (options.viaFactory ? undefined : 1),
|
|
265
263
|
echoEmaHalfLifeMs: options.echoEmaHalfLifeMs,
|
|
266
264
|
echoDrainSlackMs: options.echoDrainSlackMs,
|
|
267
|
-
...(options.frontDecider !== undefined
|
|
268
|
-
? { frontDecider: options.frontDecider }
|
|
269
|
-
: {}),
|
|
270
265
|
...(options.frontModelConfig
|
|
271
266
|
? { frontModelConfig: options.frontModelConfig }
|
|
272
267
|
: {}),
|
|
@@ -3967,15 +3962,6 @@ describe("LiveVoiceSession unified front-door endpointing", () => {
|
|
|
3967
3962
|
return { startVoiceTurn, calls, discard };
|
|
3968
3963
|
}
|
|
3969
3964
|
|
|
3970
|
-
// A decider that phrases nothing: the front-door leg IS the endpoint
|
|
3971
|
-
// decision, so these tests only need the decider seam to exist.
|
|
3972
|
-
function makeSilentDecider(): VoiceFrontDecider {
|
|
3973
|
-
return {
|
|
3974
|
-
generateAckText: async () => null,
|
|
3975
|
-
generateProgressText: async () => null,
|
|
3976
|
-
};
|
|
3977
|
-
}
|
|
3978
|
-
|
|
3979
3965
|
async function startWithPartial(
|
|
3980
3966
|
session: LiveVoiceSession,
|
|
3981
3967
|
transcribers: MockStreamingTranscriber[],
|
|
@@ -3988,12 +3974,10 @@ describe("LiveVoiceSession unified front-door endpointing", () => {
|
|
|
3988
3974
|
}
|
|
3989
3975
|
|
|
3990
3976
|
test("a chatty answer commits: verdict leg replaces the decider, frames follow commit order", async () => {
|
|
3991
|
-
const decider = makeSilentDecider();
|
|
3992
3977
|
const starter = makeVerdictTurnStarter([["Hey! Not much."]]);
|
|
3993
3978
|
const { frames, session, transcribers } = createHarness({
|
|
3994
3979
|
finals: ["hello world"],
|
|
3995
3980
|
startVoiceTurn: starter.startVoiceTurn,
|
|
3996
|
-
frontDecider: decider,
|
|
3997
3981
|
});
|
|
3998
3982
|
|
|
3999
3983
|
await startWithPartial(session, transcribers);
|