@vellumai/assistant 0.12.3-staging.1 → 0.12.3-staging.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +43 -29
- package/Dockerfile +63 -0
- package/docs/architecture/security.md +10 -5
- package/docs/architecture/turn-actor.md +2 -0
- package/docs/desktop-browser-cli.md +45 -4
- package/docs/live-voice-task-outcomes.md +100 -0
- package/docs/mode-sessions.md +183 -0
- package/docs/voice-input-diagnostics.md +58 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/package.json +2 -1
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/package.json +2 -1
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
- package/node_modules/@vellumai/gateway-client/src/gateway-ipc-contracts.ts +20 -0
- package/node_modules/@vellumai/gateway-client/src/index.ts +4 -1
- package/node_modules/@vellumai/gateway-client/src/outbound-contract.ts +38 -7
- package/node_modules/@vellumai/gateway-client/src/trust-verdict-contract.ts +5 -6
- package/node_modules/@vellumai/service-contracts/package.json +2 -1
- package/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
- package/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
- package/node_modules/@vellumai/slack-text/src/index.ts +4 -8
- package/openapi.yaml +665 -14
- package/package.json +1 -1
- package/scripts/smoke-desktop-browser-cli.ts +10 -26
- package/scripts/smoke-desktop-window-theme.ts +2 -3
- package/src/__tests__/agent-loop-output-hooks.test.ts +150 -1
- package/src/__tests__/agent-loop-provider-error-recording.test.ts +87 -19
- package/src/__tests__/annotate-activity-metadata.test.ts +3 -1
- package/src/__tests__/annotate-risk-options.test.ts +2 -0
- package/src/__tests__/answered-question-persistence.test.ts +2 -0
- package/src/__tests__/anthropic-gateway-shared.test.ts +22 -0
- package/src/__tests__/approval-message-composer.test.ts +1 -118
- package/src/__tests__/approval-routes-http.test.ts +3 -0
- package/src/__tests__/assistant-event-hub-connection-history.test.ts +111 -0
- package/src/__tests__/assistant-event-hub-targeted.test.ts +50 -0
- package/src/__tests__/assistant-event-targeting.test.ts +186 -0
- package/src/__tests__/assistant-stream-state.test.ts +29 -9
- package/src/__tests__/browser-fill-credential.test.ts +11 -22
- package/src/__tests__/btw-routes.test.ts +4 -1
- package/src/__tests__/build-persisted-content.test.ts +1 -1
- package/src/__tests__/call-conversation-messages.test.ts +5 -0
- package/src/__tests__/call-recovery.test.ts +7 -2
- package/src/__tests__/call-routes-http.test.ts +12 -0
- package/src/__tests__/call-store.test.ts +7 -2
- package/src/__tests__/channel-retry-sweep.test.ts +9 -0
- package/src/__tests__/chat-reveal-guard-priming.test.ts +2 -0
- package/src/__tests__/checker.test.ts +11 -3
- package/src/__tests__/client-os-metadata-persistence.test.ts +35 -1
- package/src/__tests__/computer-use-skill-manifest-regression.test.ts +0 -25
- package/src/__tests__/computer-use-tools.test.ts +1 -233
- package/src/__tests__/config-model-image-provider.test.ts +39 -0
- package/src/__tests__/config-schema.test.ts +16 -0
- package/src/__tests__/config-watcher.test.ts +24 -0
- package/src/__tests__/context-search-agent-runner.test.ts +260 -1
- package/src/__tests__/context-search-fanout.test.ts +1 -1
- package/src/__tests__/context-search-types.test.ts +7 -7
- package/src/__tests__/conversation-agent-loop-disk-pressure.test.ts +8 -2
- package/src/__tests__/conversation-agent-loop-fatal-cleanup.test.ts +2 -0
- package/src/__tests__/conversation-agent-loop.test.ts +546 -16
- package/src/__tests__/conversation-app-control-lifecycle.test.ts +53 -1
- package/src/__tests__/conversation-evictor.test.ts +62 -37
- package/src/__tests__/conversation-history-boundary-repair-failure.test.ts +174 -0
- package/src/__tests__/conversation-loop-db-lock-resilience.test.ts +2 -0
- package/src/__tests__/conversation-process-app-control-preactivation.test.ts +2 -0
- package/src/__tests__/conversation-retry-route.test.ts +22 -0
- package/src/__tests__/conversation-routes-disk-view.test.ts +2 -0
- package/src/__tests__/conversation-routes-enabled-plugins.test.ts +2 -0
- package/src/__tests__/conversation-routes-guardian-reply.test.ts +10 -0
- package/src/__tests__/conversation-routes-hidden-queue.test.ts +2 -0
- package/src/__tests__/conversation-routes-slash-commands.test.ts +2 -0
- package/src/__tests__/conversation-runtime-assembly.test.ts +29 -0
- package/src/__tests__/conversation-surface-snapshots.test.ts +543 -0
- package/src/__tests__/conversation-surfaces-action-delivery.test.ts +151 -0
- package/src/__tests__/conversation-surfaces-point-at-budget.test.ts +63 -1
- package/src/__tests__/conversation-tool-setup-attribution.test.ts +23 -0
- package/src/__tests__/conversation-undo.test.ts +61 -0
- package/src/__tests__/credential-execution-client.test.ts +6 -6
- package/src/__tests__/credential-prompt-route.test.ts +35 -3
- package/src/__tests__/cu-unified-flow.test.ts +82 -1
- package/src/__tests__/db-mode-sessions-migration.test.ts +175 -0
- package/src/__tests__/deterministic-verification-control-plane.test.ts +2 -0
- package/src/__tests__/dm-backfill.test.ts +91 -1
- package/src/__tests__/document-create-route.test.ts +88 -0
- package/src/__tests__/document-sync-tags.test.ts +27 -0
- package/src/__tests__/emit-signal-routing-intent.test.ts +142 -2
- package/src/__tests__/events-tail-route.test.ts +25 -0
- package/src/__tests__/evict-conversations-for-reload.test.ts +57 -8
- package/src/__tests__/guardian-action-sweep.test.ts +16 -0
- package/src/__tests__/guardian-dispatch.test.ts +13 -0
- package/src/__tests__/headless-browser-interactions.test.ts +64 -20
- package/src/__tests__/heartbeat-service.test.ts +2 -59
- package/src/__tests__/helpers/mock-conversation.ts +21 -0
- package/src/__tests__/helpers/native-web-search-harness.ts +2 -0
- package/src/__tests__/helpers/verification-sessions-ipc-sim.ts +5 -0
- package/src/__tests__/host-cu-proxy.test.ts +46 -2
- package/src/__tests__/http-user-message-parity.test.ts +2 -0
- package/src/__tests__/image-credentials.test.ts +49 -0
- package/src/__tests__/image-service-dispatcher.test.ts +45 -0
- package/src/__tests__/inference-profile-session-handler.test.ts +1 -1
- package/src/__tests__/interrupt-on-send.test.ts +57 -43
- package/src/__tests__/interrupt-turn-note.test.ts +5 -6
- package/src/__tests__/llm-catalog-parity.test.ts +2 -2
- package/src/__tests__/media-generate-image.test.ts +49 -1
- package/src/__tests__/message-provenance-contact-id.test.ts +69 -0
- package/src/__tests__/migration-export-to-gcs.test.ts +142 -0
- package/src/__tests__/mode-session-history.test.ts +245 -0
- package/src/__tests__/mode-session-recovery.test.ts +222 -0
- package/src/__tests__/mode-session-wire.test.ts +141 -0
- package/src/__tests__/notification-guardian-path.test.ts +7 -0
- package/src/__tests__/notification-telegram-adapter.test.ts +2 -2
- package/src/__tests__/notification-vellum-adapter.test.ts +34 -0
- package/src/__tests__/oauth-provider-serializer.test.ts +12 -0
- package/src/__tests__/oauth-providers-routes.test.ts +1 -0
- package/src/__tests__/outbound-slack-persistence.test.ts +2 -0
- package/src/__tests__/persistence-secret-redaction.test.ts +2 -0
- package/src/__tests__/plugin-api-provider.test.ts +5 -0
- package/src/__tests__/plugin-api-resolve-credential.test.ts +97 -0
- package/src/__tests__/plugin-import-boundary-guard.test.ts +9 -0
- package/src/__tests__/process-message-display-content.test.ts +56 -0
- package/src/__tests__/reaction-persistence.test.ts +5 -0
- package/src/__tests__/request-secret-standalone-channel-gate.test.ts +25 -9
- package/src/__tests__/runtime-events-sse-reconnect.test.ts +1 -1
- package/src/__tests__/scripted-turn-metadata-persistence.test.ts +73 -0
- package/src/__tests__/secret-ingress-http.test.ts +2 -0
- package/src/__tests__/secure-keys.test.ts +43 -1
- package/src/__tests__/send-endpoint-busy.test.ts +163 -1
- package/src/__tests__/server-history-render.test.ts +91 -1
- package/src/__tests__/settings-routes.test.ts +50 -1
- package/src/__tests__/skill-script-runner-sandbox.test.ts +32 -0
- package/src/__tests__/subagent-fork-notifications.test.ts +1 -2
- package/src/__tests__/subagent-manager-notify.test.ts +1 -2
- package/src/__tests__/subagent-notify-parent.test.ts +146 -39
- package/src/__tests__/subagent-notify-stale-rebuild.test.ts +2 -4
- package/src/__tests__/surface-completion-in-flight-snapshot.test.ts +2 -0
- package/src/__tests__/thread-backfill.test.ts +53 -0
- package/src/__tests__/tool-preview-lifecycle.test.ts +2 -0
- package/src/__tests__/tool-result-metadata-plumbing.test.ts +3 -1
- package/src/__tests__/tool-start-timestamp.test.ts +2 -0
- package/src/__tests__/twilio-routes.test.ts +1 -0
- package/src/__tests__/web-search-backend-failure.test.ts +1 -1
- package/src/activation/progress-store.test.ts +12 -7
- package/src/agent/loop.ts +99 -41
- package/src/api/events/assistant-turn-start.ts +3 -0
- package/src/api/events/generation-handoff.ts +2 -0
- package/src/api/events/message-complete.ts +2 -0
- package/src/api/events/notification-conversation-created.ts +14 -16
- package/src/api/events/notification-intent.ts +4 -2
- package/src/api/events/tool-result.ts +85 -0
- package/src/api/events/tool-use-start.ts +3 -0
- package/src/api/events/user-message-echo.ts +3 -0
- package/src/api/index.ts +31 -6
- package/src/api/mode-session.test.ts +131 -0
- package/src/api/mode-session.ts +148 -0
- package/src/api/remember-tool.ts +15 -0
- package/src/api/responses/conversation-message.ts +6 -0
- package/src/approvals/AGENTS.md +4 -2
- package/src/browser/__tests__/operations.test.ts +29 -0
- package/src/browser/operation-meta.ts +2 -1
- package/src/browser/operations.ts +29 -0
- package/src/browser/virtual-desktop-target.ts +2 -8
- package/src/bundler/bundle-signer.ts +1 -3
- package/src/calls/__tests__/barge-in-guard.test.ts +3 -0
- package/src/calls/__tests__/call-controller.test.ts +94 -0
- package/src/calls/__tests__/call-funnel-telemetry.test.ts +204 -0
- package/src/calls/__tests__/voice-control-protocol.test.ts +15 -0
- package/src/calls/__tests__/voice-escalation-judge.test.ts +209 -0
- package/src/calls/__tests__/voice-leg-coordinator.test.ts +168 -0
- package/src/{live-voice/__tests__/live-voice-metrics.test.ts → calls/__tests__/voice-metrics.test.ts} +32 -38
- package/src/calls/__tests__/voice-session-bridge.test.ts +576 -122
- package/src/calls/barge-in-guard.ts +5 -0
- package/src/calls/call-controller.ts +196 -1
- package/src/calls/call-domain.ts +4 -0
- package/src/calls/call-store.ts +87 -2
- package/src/calls/guardian-action-sweep.ts +22 -8
- package/src/calls/media-stream-audio-transcode.ts +1 -3
- package/src/calls/typesafe-noul.ts +156 -0
- package/src/calls/voice-control-protocol.ts +6 -0
- package/src/calls/voice-escalation-judge.ts +145 -0
- package/src/calls/voice-escalation-target.ts +10 -0
- package/src/calls/voice-leg-coordinator.ts +121 -3
- package/src/{live-voice/live-voice-metrics.ts → calls/voice-metrics.ts} +94 -87
- package/src/calls/voice-session-bridge.ts +256 -112
- package/src/calls/voice-triage-escalate.ts +7 -0
- package/src/cli/__tests__/roadmap-cli.test.ts +4 -41
- package/src/cli/commands/__tests__/notifications.test.ts +35 -10
- package/src/cli/commands/browser.help.ts +4 -5
- package/src/cli/commands/browser.ts +0 -1
- package/src/cli/commands/clients.help.ts +46 -0
- package/src/cli/commands/clients.ts +116 -0
- package/src/cli/commands/credentials.help.ts +6 -1
- package/src/cli/commands/notifications.help.ts +16 -2
- package/src/cli/commands/notifications.ts +30 -8
- package/src/cli/commands/roadmap.help.ts +18 -21
- package/src/cli/commands/roadmap.ts +4 -21
- package/src/cli/lib/__tests__/list-installed-plugins.test.ts +5 -2
- package/src/cli/lib/bundled-marketplace.json +84 -34
- package/src/cli/lib/plugin-marketplace.ts +8 -0
- package/src/cli/lib/publish-plugin.ts +1 -2
- package/src/config/__tests__/default-provider.test.ts +3 -1
- package/src/config/__tests__/memory-retrospective-schema.test.ts +14 -1
- package/src/config/bundled-skills/computer-use/SKILL.md +1 -1
- package/src/config/bundled-skills/computer-use/TOOLS.json +89 -1
- package/src/config/bundled-skills/image-studio/SKILL.md +17 -2
- package/src/config/bundled-skills/image-studio/tools/media-generate-image.ts +7 -11
- package/src/config/bundled-skills/sequences/TOOLS.json +1 -5
- package/src/config/call-site-defaults.ts +9 -0
- package/src/config/feature-flag-registry.json +16 -0
- package/src/config/profile-text-generation.test.ts +2 -2
- package/src/config/schemas/__tests__/tools.test.ts +31 -0
- package/src/config/schemas/call-site-catalog.ts +14 -0
- package/src/config/schemas/channels.ts +0 -18
- package/src/config/schemas/llm.ts +3 -1
- package/src/config/schemas/mcp.ts +2 -1
- package/src/config/schemas/memory-retrospective.ts +11 -2
- package/src/config/schemas/services.ts +15 -6
- package/src/config/schemas/tools.ts +10 -0
- package/src/config/session-groups-gate.ts +21 -0
- package/src/config/skills.ts +2 -6
- package/src/conversations/__tests__/message-consolidation.test.ts +149 -0
- package/src/conversations/message-consolidation.ts +102 -5
- package/src/credential-execution/ces-connect.test.ts +69 -0
- package/src/credential-execution/ces-connect.ts +115 -0
- package/src/credential-execution/ces-runtime.ts +55 -93
- package/src/credential-execution/executable-discovery.ts +1 -11
- package/src/daemon/__tests__/conversation-surfaces-launch.test.ts +10 -1
- package/src/daemon/__tests__/inference-profile-notification.test.ts +2 -0
- package/src/daemon/browser-mode-session.test.ts +402 -0
- package/src/daemon/browser-mode-session.ts +186 -0
- package/src/daemon/computer-use-mode-session.test.ts +210 -0
- package/src/daemon/computer-use-mode-session.ts +105 -0
- package/src/daemon/config-watcher.ts +10 -0
- package/src/daemon/conversation-agent-loop-handlers.ts +58 -10
- package/src/daemon/conversation-agent-loop.ts +219 -25
- package/src/daemon/conversation-evictor.ts +5 -24
- package/src/daemon/conversation-history.ts +10 -0
- package/src/daemon/conversation-interrupt-repair.ts +4 -43
- package/src/daemon/conversation-interrupt.ts +7 -11
- package/src/daemon/conversation-messaging.ts +58 -5
- package/src/daemon/conversation-mode-session.test.ts +1667 -0
- package/src/daemon/conversation-mode-session.ts +1008 -0
- package/src/daemon/conversation-process.ts +5 -0
- package/src/daemon/conversation-runtime-assembly.ts +10 -20
- package/src/daemon/conversation-skill-tools.ts +1 -0
- package/src/daemon/conversation-store.ts +4 -4
- package/src/daemon/conversation-surface-snapshots.ts +135 -0
- package/src/daemon/conversation-surfaces.ts +113 -5
- package/src/daemon/conversation-tool-setup.ts +8 -3
- package/src/daemon/conversation.ts +143 -41
- package/src/daemon/handlers/config-model.ts +7 -5
- package/src/daemon/handlers/shared.ts +55 -16
- package/src/daemon/host-cu-proxy.ts +51 -22
- package/src/daemon/lifecycle.ts +30 -13
- package/src/daemon/message-protocol.ts +0 -1
- package/src/daemon/message-provenance.ts +28 -4
- package/src/daemon/mode-session-startup-recovery.ts +23 -0
- package/src/daemon/mode-session-tracking.ts +54 -0
- package/src/daemon/process-message.ts +13 -4
- package/src/daemon/virtual-desktop-context.ts +21 -0
- package/src/desktop/__tests__/fake-desktop.ts +1 -0
- package/src/desktop/desktop-accessibility-script.ts +88 -0
- package/src/desktop/desktop-accessibility.test.ts +160 -0
- package/src/desktop/desktop-accessibility.ts +169 -0
- package/src/desktop/desktop-automation-lease.test.ts +76 -68
- package/src/desktop/desktop-automation-lease.ts +41 -50
- package/src/desktop/desktop-browser-endpoint.ts +4 -0
- package/src/desktop/desktop-computer-use-routing.test.ts +330 -0
- package/src/desktop/desktop-computer-use.test.ts +449 -0
- package/src/desktop/desktop-computer-use.ts +502 -0
- package/src/desktop/desktop-dependencies.ts +47 -293
- package/src/desktop/desktop-display.ts +2 -2
- package/src/desktop/desktop-panel-config.test.ts +13 -3
- package/src/desktop/desktop-panel-config.ts +109 -15
- package/src/desktop/desktop-session-manager.test.ts +69 -21
- package/src/desktop/desktop-session-manager.ts +76 -22
- package/src/desktop/desktop-stream-bridge.test.ts +11 -122
- package/src/desktop/desktop-stream-bridge.ts +6 -8
- package/src/desktop/desktop-window-theme.ts +5 -25
- package/src/desktop/virtual-desktop-feature.ts +52 -2
- package/src/desktop/virtual-desktop-platform.test.ts +5 -9
- package/src/documents/document-store.ts +23 -0
- package/src/heartbeat/__tests__/heartbeat-service.test.ts +22 -26
- package/src/heartbeat/heartbeat-service.ts +5 -33
- package/src/ipc/__tests__/browser-ipc.test.ts +215 -6
- package/src/ipc/assistant-server.ts +2 -0
- package/src/ipc/routes/__tests__/channel-reply-ipc-routes.test.ts +111 -0
- package/src/ipc/routes/channel-reply-ipc-routes.ts +73 -0
- package/src/live-voice/__tests__/continuation-judge.test.ts +151 -0
- package/src/live-voice/__tests__/live-activity-reporter.test.ts +154 -15
- package/src/live-voice/__tests__/live-voice-flux-turn-end.test.ts +743 -31
- package/src/live-voice/__tests__/live-voice-look-follow-up.test.ts +191 -10
- package/src/live-voice/__tests__/live-voice-session-groups-gate.test.ts +87 -0
- package/src/live-voice/__tests__/live-voice-session-manager.test.ts +5 -1
- package/src/live-voice/__tests__/live-voice-sight-frame-echo.test.ts +41 -0
- package/src/live-voice/__tests__/live-voice-sight-frame.test.ts +1017 -9
- package/src/live-voice/__tests__/live-voice-triage-escalate.test.ts +224 -1
- package/src/live-voice/__tests__/live-voice-tts-session.test.ts +101 -0
- package/src/live-voice/__tests__/live-voice-tts.test.ts +44 -0
- package/src/live-voice/__tests__/live-voice-vad.test.ts +939 -75
- package/src/live-voice/__tests__/microphone-idle-gate.test.ts +87 -0
- package/src/live-voice/__tests__/session-controls.test.ts +23 -0
- package/src/live-voice/__tests__/task-outcomes.test.ts +117 -0
- package/src/live-voice/__tests__/voice-input-diagnostics.test.ts +117 -0
- package/src/live-voice/camera-mode-session.test.ts +265 -0
- package/src/live-voice/camera-mode-session.ts +277 -0
- package/src/live-voice/continuation-judge.ts +123 -0
- package/src/live-voice/live-activity-reporter.ts +115 -6
- package/src/live-voice/live-voice-manager.ts +36 -1
- package/src/live-voice/live-voice-photo.ts +88 -7
- package/src/live-voice/live-voice-session-manager.ts +15 -0
- package/src/live-voice/live-voice-session.ts +1443 -747
- package/src/live-voice/live-voice-tts.ts +22 -4
- package/src/live-voice/microphone-idle-gate.ts +56 -0
- package/src/live-voice/protocol.ts +171 -4
- package/src/live-voice/session-controls.ts +11 -3
- package/src/live-voice/task-outcomes.ts +131 -0
- package/src/live-voice/voice-input-diagnostics.ts +117 -0
- package/src/mcp/__tests__/manager-tool-caps.test.ts +63 -18
- package/src/mcp/__tests__/tool-caps.test.ts +28 -1
- package/src/mcp/manager.ts +3 -0
- package/src/mcp/tool-caps.ts +12 -4
- package/src/media/__tests__/image-models.test.ts +58 -0
- package/src/media/image-credentials.ts +5 -0
- package/src/media/image-models.ts +52 -0
- package/src/media/image-service.ts +8 -0
- package/src/media/openrouter-image-service.test.ts +133 -0
- package/src/media/openrouter-image-service.ts +153 -0
- package/src/media/types.ts +1 -1
- package/src/messaging/providers/__tests__/transport-dispatch.test.ts +65 -11
- package/src/messaging/providers/discord/send.ts +4 -17
- package/src/messaging/providers/discord/transport.ts +2 -2
- package/src/messaging/providers/outlook/client.ts +16 -22
- package/src/messaging/providers/telegram-bot/send.test.ts +236 -1
- package/src/messaging/providers/telegram-bot/send.ts +10 -7
- package/src/messaging/providers/telegram-bot/transport.ts +3 -10
- package/src/messaging/providers/whatsapp/send.test.ts +169 -4
- package/src/monitoring/__tests__/control.test.ts +17 -5
- package/src/monitoring/control.ts +15 -5
- package/src/notifications/AGENTS.md +1 -1
- package/src/notifications/README.md +83 -99
- package/src/notifications/__tests__/broadcaster.test.ts +1 -1
- package/src/notifications/__tests__/channel-allowlist.test.ts +39 -0
- package/src/notifications/__tests__/copy-composer.test.ts +1 -1
- package/src/notifications/__tests__/decision-engine.test.ts +79 -2
- package/src/notifications/__tests__/deterministic-checks.test.ts +10 -10
- package/src/notifications/__tests__/home-feed-side-effect.test.ts +93 -4
- package/src/notifications/__tests__/notification-utils.test.ts +18 -0
- package/src/notifications/adapters/macos.ts +37 -46
- package/src/notifications/broadcaster.ts +5 -6
- package/src/notifications/channel-allowlist.ts +44 -0
- package/src/notifications/copy-composer.ts +0 -23
- package/src/notifications/decision-engine.ts +24 -14
- package/src/notifications/deterministic-checks.ts +1 -1
- package/src/notifications/emit-signal.ts +123 -37
- package/src/notifications/home-feed-side-effect.ts +9 -7
- package/src/notifications/notification-utils.ts +19 -0
- package/src/notifications/signal.ts +4 -19
- package/src/notifications/types.ts +15 -0
- package/src/oauth/connection-resolver.ts +1 -2
- package/src/oauth/oauth-store.ts +1 -3
- package/src/oauth/provider-serializer.ts +7 -0
- package/src/oauth/seed-providers.ts +30 -0
- package/src/onboarding/onboarding-events-store.ts +99 -12
- package/src/persistence/__tests__/client-connection-events-store.test.ts +322 -0
- package/src/persistence/client-connection-events-store.ts +377 -0
- package/src/persistence/conversation-crud.ts +55 -6
- package/src/persistence/conversation-mode-sessions.test.ts +424 -0
- package/src/persistence/conversation-mode-sessions.ts +460 -0
- package/src/persistence/conversation-queries.ts +6 -19
- package/src/persistence/llm-usage-store.ts +3 -9
- package/src/persistence/message-metadata.ts +25 -0
- package/src/persistence/migrations/381-create-conversation-mode-sessions.ts +41 -0
- package/src/persistence/migrations/382-create-client-connection-events.ts +40 -0
- package/src/persistence/migrations/__tests__/382-create-client-connection-events.test.ts +56 -0
- package/src/persistence/schema/conversations.ts +67 -0
- package/src/persistence/schema/infrastructure.ts +29 -0
- package/src/persistence/steps.ts +17 -0
- package/src/plugin-api/__tests__/plugin-name-env.test.ts +249 -0
- package/src/plugin-api/effective-context-window.ts +33 -0
- package/src/plugin-api/index.ts +24 -5
- package/src/plugin-api/plugin-name-env.ts +125 -0
- package/src/plugin-api/resolve-credential.ts +79 -20
- package/src/plugin-api/vision-support.test.ts +22 -0
- package/src/plugin-api/vision-support.ts +17 -5
- package/src/plugins/__tests__/active-plugin-env.test.ts +136 -0
- package/src/plugins/active-plugin-env.ts +51 -0
- package/src/plugins/defaults/index.ts +24 -0
- package/src/plugins/defaults/memory/__tests__/archive-removal-framing.test.ts +64 -0
- package/src/plugins/defaults/memory/__tests__/worker-control.test.ts +19 -6
- package/src/plugins/defaults/memory/context-search/agent-protocol.ts +9 -12
- package/src/plugins/defaults/memory/context-search/agent-runner.ts +78 -22
- package/src/plugins/defaults/memory/context-search/format.ts +51 -7
- package/src/plugins/defaults/memory/context-search/limits.ts +10 -12
- package/src/plugins/defaults/memory/context-search/recall-input.ts +38 -0
- package/src/plugins/defaults/memory/context-search/search.ts +8 -14
- package/src/plugins/defaults/memory/context-search/sources/conversations.ts +1 -0
- package/src/plugins/defaults/memory/context-search/types.ts +1 -11
- package/src/plugins/defaults/memory/graph/__tests__/handle-remember-v2.test.ts +11 -4
- package/src/plugins/defaults/memory/graph/tool-handlers.ts +12 -21
- package/src/plugins/defaults/memory/graph/tools.ts +35 -64
- package/src/plugins/defaults/memory/src/memory-eval-routes.ts +2 -3
- package/src/plugins/defaults/memory/src/memory-item-routes.test.ts +5 -0
- package/src/plugins/defaults/memory/src/memory-item-routes.ts +2 -1
- package/src/plugins/defaults/memory/substrate/prompts/consolidation.ts +2 -2
- package/src/plugins/defaults/memory/tools.test.ts +124 -9
- package/src/plugins/defaults/memory/tools.ts +38 -11
- package/src/plugins/defaults/memory/v3/__tests__/pool-log-store.test.ts +165 -27
- package/src/plugins/defaults/memory/v3/__tests__/pool-select.test.ts +626 -27
- package/src/plugins/defaults/memory/v3/__tests__/shadow-plugin.test.ts +46 -27
- package/src/plugins/defaults/memory/v3/orchestrate.ts +78 -38
- package/src/plugins/defaults/memory/v3/pool-log-store.ts +65 -79
- package/src/plugins/defaults/memory/v3/pool-select.test.ts +14 -5
- package/src/plugins/defaults/memory/v3/pool-select.ts +683 -44
- package/src/plugins/defaults/memory/worker-control.ts +15 -5
- package/src/plugins/defaults/task-progress/__tests__/registration.test.ts +70 -0
- package/src/plugins/defaults/task-progress/__tests__/task-progress-context.test.ts +380 -0
- package/src/plugins/defaults/task-progress/hooks/post-compact.ts +19 -0
- package/src/plugins/defaults/task-progress/hooks/user-prompt-submit.ts +24 -0
- package/src/plugins/defaults/task-progress/package.json +14 -0
- package/src/plugins/defaults/task-progress/src/apply-task-progress-context.ts +93 -0
- package/src/plugins/defaults/task-progress/src/format-task-progress-context.ts +67 -0
- package/src/providers/__tests__/context-overflow-error.test.ts +10 -0
- package/src/providers/__tests__/search-provider-catalog.test.ts +18 -0
- package/src/providers/anthropic/client.ts +10 -1
- package/src/providers/anthropic-gateway-shared.ts +6 -0
- package/src/providers/fetch-provider-catalog.ts +11 -0
- package/src/providers/gemini/client.ts +25 -8
- package/src/providers/inference/adapter-factory.ts +1 -1
- package/src/providers/jev/client.test.ts +10 -0
- package/src/providers/jev/client.ts +20 -4
- package/src/providers/model-catalog.ts +4 -3
- package/src/providers/openai/__tests__/chat-completions-extra-body.test.ts +94 -1
- package/src/providers/openai/__tests__/chat-completions-provider-reasoning.test.ts +106 -1
- package/src/providers/openai/__tests__/chat-template-flatten.test.ts +24 -0
- package/src/providers/openai/__tests__/google-thought-signature.test.ts +1 -2
- package/src/providers/openai/chat-completions-provider.ts +100 -29
- package/src/providers/openai/google-thought-signature.ts +0 -6
- package/src/providers/openai/responses-provider.ts +23 -1
- package/src/providers/ratelimit.ts +1 -3
- package/src/providers/search-provider-catalog.ts +24 -0
- package/src/providers/speech-to-text/__tests__/deepgram-flux-frames.test.ts +24 -0
- package/src/providers/speech-to-text/__tests__/deepgram-flux-realtime.test.ts +7 -1
- package/src/providers/speech-to-text/__tests__/vellum-managed-realtime.test.ts +50 -1
- package/src/providers/speech-to-text/deepgram-flux-frames.ts +11 -0
- package/src/providers/speech-to-text/vellum-managed-flux-realtime.ts +13 -0
- package/src/providers/types.ts +5 -0
- package/src/routes/control.ts +7 -5
- package/src/runtime/AGENTS.md +4 -2
- package/src/runtime/actor-trust-resolver.ts +3 -3
- package/src/runtime/agent-wake.ts +1 -1
- package/src/runtime/approval-message-composer.ts +3 -69
- package/src/runtime/assistant-event-hub.ts +73 -75
- package/src/runtime/assistant-event-publish-options.ts +7 -0
- package/src/runtime/assistant-event-targeting.ts +79 -0
- package/src/runtime/assistant-stream-state.ts +13 -100
- package/src/runtime/background-job-runner.ts +5 -5
- package/src/runtime/channel-retry-sweep.ts +1 -0
- package/src/runtime/guardian-action-message-composer.ts +8 -161
- package/src/runtime/guardian-reply-router.ts +28 -32
- package/src/runtime/http-types.ts +6 -0
- package/src/runtime/message-composer-types.ts +3 -61
- package/src/runtime/migrations/__tests__/vbundle-import-parity.test.ts +67 -1
- package/src/runtime/migrations/vbundle-builder.ts +24 -1
- package/src/runtime/migrations/vbundle-import-analyzer.ts +13 -4
- package/src/runtime/migrations/vbundle-import-policy.ts +10 -0
- package/src/runtime/migrations/vbundle-importer.ts +4 -1
- package/src/runtime/migrations/vbundle-streaming-importer.ts +4 -1
- package/src/runtime/migrations/vbundle-validator.ts +1 -0
- package/src/runtime/pending-interactions-mode-session.test.ts +62 -0
- package/src/runtime/pending-interactions.ts +29 -0
- package/src/runtime/routes/__tests__/client-history-routes.test.ts +104 -0
- package/src/runtime/routes/__tests__/image-generation-routes.test.ts +97 -0
- package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +1 -1
- package/src/runtime/routes/__tests__/roadmap-routes.test.ts +29 -6
- package/src/runtime/routes/browser-context.ts +31 -11
- package/src/runtime/routes/browser-routes.ts +43 -4
- package/src/runtime/routes/btw-routes.ts +9 -4
- package/src/runtime/routes/client-routes.ts +108 -0
- package/src/runtime/routes/conversation-management-routes.ts +4 -1
- package/src/runtime/routes/conversation-routes.ts +113 -27
- package/src/runtime/routes/credential-prompt-routes.ts +4 -3
- package/src/runtime/routes/credential-routes.ts +1 -2
- package/src/runtime/routes/desktop-setup-routes.test.ts +9 -18
- package/src/runtime/routes/desktop-setup-routes.ts +4 -6
- package/src/runtime/routes/documents-routes.ts +51 -0
- package/src/runtime/routes/email-routes.ts +3 -7
- package/src/runtime/routes/events-routes.ts +75 -54
- package/src/runtime/routes/identity-routes.ts +1 -5
- package/src/runtime/routes/image-generation-routes.ts +8 -14
- package/src/runtime/routes/inbound-message-handler.ts +35 -38
- package/src/runtime/routes/inbound-stages/backfill-sender-provenance.test.ts +168 -0
- package/src/runtime/routes/inbound-stages/backfill-sender-provenance.ts +96 -0
- package/src/runtime/routes/inbound-stages/background-dispatch.test.ts +38 -9
- package/src/runtime/routes/inbound-stages/background-dispatch.ts +2 -5
- package/src/runtime/routes/inbound-stages/reaction-intercept.ts +2 -0
- package/src/runtime/routes/llm-context-normalization.ts +1 -7
- package/src/runtime/routes/migration-routes.ts +73 -2
- package/src/runtime/routes/notification-routes.ts +10 -0
- package/src/runtime/routes/oauth-providers.ts +7 -0
- package/src/runtime/routes/plugins-routes.ts +6 -0
- package/src/runtime/routes/roadmap-routes.ts +47 -8
- package/src/runtime/routes/secret-routes.ts +1 -1
- package/src/runtime/routes/settings-routes.ts +10 -4
- package/src/runtime/routes/watcher-routes.ts +1 -3
- package/src/runtime/trust-class.test.ts +7 -6
- package/src/runtime/trust-class.ts +9 -17
- package/src/schedule/__tests__/worker-control.test.ts +36 -5
- package/src/schedule/worker-control.ts +15 -5
- package/src/security/secure-keys.ts +53 -103
- package/src/skills/clawhub.ts +3 -8
- package/src/skills/managed-store.ts +1 -5
- package/src/stt/types.ts +4 -0
- package/src/subagent/notify.ts +54 -34
- package/src/subagent/parent-notification.ts +6 -0
- package/src/telemetry/__tests__/phone-call-funnel.test.ts +72 -0
- package/src/telemetry/phone-call-funnel.ts +144 -0
- package/src/tools/__tests__/tool-schema-root-combinator-guard.test.ts +81 -0
- package/src/tools/ask-question/ask-question-tool.ts +6 -4
- package/src/tools/browser/browser-execution.ts +22 -59
- package/src/tools/browser/cdp-client/__tests__/cdp-dom-helpers.test.ts +561 -39
- package/src/tools/browser/cdp-client/cdp-dom-helpers.ts +435 -39
- package/src/tools/browser/cdp-client/cdp-inspect-client.ts +1 -8
- package/src/tools/client-os.ts +15 -0
- package/src/tools/computer-use/skill-proxy-bridge.ts +1 -1
- package/src/tools/computer-use/target.ts +42 -0
- package/src/tools/document/document-tool.ts +13 -18
- package/src/tools/execution-target.ts +14 -16
- package/src/tools/host-terminal/host-shell.ts +4 -1
- package/src/tools/network/__tests__/web-fetch-tinyfish.test.ts +250 -0
- package/src/tools/network/__tests__/web-search.test.ts +203 -0
- package/src/tools/network/web-fetch.ts +274 -7
- package/src/tools/network/web-search.ts +468 -6
- package/src/tools/permission-checker.ts +3 -3
- package/src/tools/policy-context.ts +3 -1
- package/src/tools/skills/load.ts +2 -1
- package/src/tools/skills/sandbox-runner.ts +11 -2
- package/src/tools/skills/skill-script-runner.ts +7 -0
- package/src/tools/skills/skill-tool-factory.ts +15 -1
- package/src/tools/terminal/__tests__/safe-env.test.ts +11 -0
- package/src/tools/terminal/safe-env.ts +4 -0
- package/src/tools/terminal/shell.ts +2 -0
- package/src/tools/tool-approval-handler.ts +5 -1
- package/src/tools/tool-types.ts +3 -3
- package/src/tools/types.ts +8 -2
- package/src/util/__tests__/worker-process-command.test.ts +5 -4
- package/src/util/abort-reasons.ts +18 -26
- package/src/util/canonicalize-identity.test.ts +21 -0
- package/src/util/canonicalize-identity.ts +16 -0
- package/src/util/errors.ts +13 -3
- package/src/util/provider-error-patterns.ts +3 -0
- package/src/util/worker-process.ts +77 -35
- package/src/workspace/migrations/157-add-desktop-files-launcher.ts +94 -0
- package/src/workspace/migrations/__tests__/156-extract-workspace-mcp-json.test.ts +4 -10
- package/src/workspace/migrations/__tests__/157-add-desktop-files-launcher.test.ts +113 -0
- package/src/workspace/migrations/registry.ts +2 -0
- package/src/__tests__/computer-use-window-schema.test.ts +0 -24
- package/src/api/events/heartbeat-alert.ts +0 -20
- package/src/daemon/message-types/web-activity.ts +0 -65
- package/src/desktop/desktop-dependencies.test.ts +0 -131
- package/src/tools/computer-use/definitions.ts +0 -607
|
@@ -3,6 +3,8 @@ import { join } from "node:path";
|
|
|
3
3
|
import { afterAll, beforeAll, describe, expect, mock, test } from "bun:test";
|
|
4
4
|
|
|
5
5
|
import type { TurnDetectorConfig } from "../../calls/media-turn-detector.js";
|
|
6
|
+
import type { VoiceProgressNarrator } from "../../calls/progress-narration.js";
|
|
7
|
+
import { TASK_UPDATE_SILENT_MARKER } from "../../calls/voice-control-protocol.js";
|
|
6
8
|
import type {
|
|
7
9
|
VoiceTurnCallbacks,
|
|
8
10
|
VoiceTurnOptions,
|
|
@@ -13,14 +15,15 @@ import {
|
|
|
13
15
|
saveRawConfig,
|
|
14
16
|
} from "../../config/loader.js";
|
|
15
17
|
import type { VoiceFrontModelConfig } from "../../config/schemas/voice.js";
|
|
18
|
+
import type {
|
|
19
|
+
ConversationModeSessionCoordinator,
|
|
20
|
+
ModeSessionSourceHandle,
|
|
21
|
+
} from "../../daemon/conversation-mode-session.js";
|
|
22
|
+
import { RiskLevel } from "../../permissions/types.js";
|
|
16
23
|
import type {
|
|
17
24
|
StreamingTranscriber,
|
|
18
25
|
SttStreamServerEvent,
|
|
19
26
|
} from "../../stt/types.js";
|
|
20
|
-
import {
|
|
21
|
-
computerUseKeyTool,
|
|
22
|
-
computerUseObserveTool,
|
|
23
|
-
} from "../../tools/computer-use/definitions.js";
|
|
24
27
|
import {
|
|
25
28
|
__resetRegistryForTesting,
|
|
26
29
|
registerSkillTools,
|
|
@@ -28,6 +31,7 @@ import {
|
|
|
28
31
|
} from "../../tools/registry.js";
|
|
29
32
|
import { finalizeTool } from "../../tools/tool-defaults.js";
|
|
30
33
|
import { getWorkspaceSkillsDir } from "../../util/platform.js";
|
|
34
|
+
import type { LiveVoiceContinuationJudge } from "../continuation-judge.js";
|
|
31
35
|
import type { LiveVoiceContinuationLabeler } from "../continuation-label.js";
|
|
32
36
|
import type { LiveVoiceAudioArchiveResult } from "../live-voice-archive.js";
|
|
33
37
|
import {
|
|
@@ -36,6 +40,7 @@ import {
|
|
|
36
40
|
type LiveVoiceBackgroundContinuationSpawner,
|
|
37
41
|
LiveVoiceSession,
|
|
38
42
|
type LiveVoiceSessionAudioArchiver,
|
|
43
|
+
type LiveVoiceSessionOptions,
|
|
39
44
|
type LiveVoiceTtsStreamer,
|
|
40
45
|
type LiveVoiceTurnStarter,
|
|
41
46
|
} from "../live-voice-session.js";
|
|
@@ -58,7 +63,12 @@ import {
|
|
|
58
63
|
// `mock.module` is process-global in Bun and leaks into sibling files that run
|
|
59
64
|
// in the same process — run this file on its own (`bun test <path>`).
|
|
60
65
|
const injectMessageIntoParentMock = mock(
|
|
61
|
-
(
|
|
66
|
+
(
|
|
67
|
+
_parentConversationId: string,
|
|
68
|
+
_message: string,
|
|
69
|
+
_metadata?: Record<string, unknown>,
|
|
70
|
+
_opts?: { cronRunId?: string | null; bypassLiveVoice?: boolean },
|
|
71
|
+
) => {},
|
|
62
72
|
);
|
|
63
73
|
mock.module("../../subagent/notify.js", () => ({
|
|
64
74
|
injectMessageIntoParent: injectMessageIntoParentMock,
|
|
@@ -208,12 +218,17 @@ function createHarness(options: {
|
|
|
208
218
|
viaFactory?: boolean;
|
|
209
219
|
spawnBackgroundContinuation?: LiveVoiceBackgroundContinuationSpawner;
|
|
210
220
|
labelBackgroundContinuation?: LiveVoiceContinuationLabeler;
|
|
221
|
+
judgeBackgroundContinuation?: LiveVoiceContinuationJudge;
|
|
211
222
|
getTurnTeardown?: (conversationId: string) => Promise<void> | undefined;
|
|
212
223
|
detachTeardownSettleTimeoutMs?: number;
|
|
213
224
|
continuationAnnounceSilenceMs?: number;
|
|
225
|
+
progressNarrator?: VoiceProgressNarrator;
|
|
214
226
|
foregroundTaskResumeSilenceMs?: number;
|
|
215
227
|
foregroundTaskMaxSuspendedMs?: number;
|
|
216
228
|
foregroundTaskMaxInterveningTurns?: number;
|
|
229
|
+
acquireModeSessionResidency?: NonNullable<
|
|
230
|
+
LiveVoiceSessionOptions["acquireModeSessionResidency"]
|
|
231
|
+
>;
|
|
217
232
|
}) {
|
|
218
233
|
const sequencer = createLiveVoiceServerFrameSequencer();
|
|
219
234
|
const frames: LiveVoiceServerFrame[] = [];
|
|
@@ -259,6 +274,7 @@ function createHarness(options: {
|
|
|
259
274
|
options.startVoiceTurn ??
|
|
260
275
|
mock(async () => ({ turnId: "bridge-turn", abort: mock() })),
|
|
261
276
|
streamTtsAudio: options.streamTtsAudio ?? null,
|
|
277
|
+
progressNarrator: options.progressNarrator,
|
|
262
278
|
archiveAudio: options.archiveAudio ?? null,
|
|
263
279
|
emitMetrics: options.emitMetrics ?? false,
|
|
264
280
|
...(options.metricsClock ? { metricsClock: options.metricsClock } : {}),
|
|
@@ -288,6 +304,9 @@ function createHarness(options: {
|
|
|
288
304
|
...(options.spawnBackgroundContinuation
|
|
289
305
|
? { spawnBackgroundContinuation: options.spawnBackgroundContinuation }
|
|
290
306
|
: {}),
|
|
307
|
+
...(options.judgeBackgroundContinuation
|
|
308
|
+
? { judgeBackgroundContinuation: options.judgeBackgroundContinuation }
|
|
309
|
+
: {}),
|
|
291
310
|
...(options.labelBackgroundContinuation
|
|
292
311
|
? { labelBackgroundContinuation: options.labelBackgroundContinuation }
|
|
293
312
|
: {}),
|
|
@@ -314,6 +333,9 @@ function createHarness(options: {
|
|
|
314
333
|
options.foregroundTaskMaxInterveningTurns,
|
|
315
334
|
}
|
|
316
335
|
: {}),
|
|
336
|
+
...(options.acquireModeSessionResidency
|
|
337
|
+
? { acquireModeSessionResidency: options.acquireModeSessionResidency }
|
|
338
|
+
: {}),
|
|
317
339
|
};
|
|
318
340
|
const session = options.viaFactory
|
|
319
341
|
? createLiveVoiceSession(context, {
|
|
@@ -514,6 +536,7 @@ async function startForegroundTaskBargeInScenario(options?: {
|
|
|
514
536
|
skillExecuteInput?: Record<string, unknown>;
|
|
515
537
|
skillExecuteAllowedToolNames?: ReadonlySet<string>;
|
|
516
538
|
startHostTool?: boolean;
|
|
539
|
+
judgeBackgroundContinuation?: LiveVoiceContinuationJudge;
|
|
517
540
|
}): Promise<{
|
|
518
541
|
calls: VoiceTurnOptions[];
|
|
519
542
|
frames: LiveVoiceServerFrame[];
|
|
@@ -538,6 +561,9 @@ async function startForegroundTaskBargeInScenario(options?: {
|
|
|
538
561
|
return makeTtsResult("assistant audio");
|
|
539
562
|
}),
|
|
540
563
|
spawnBackgroundContinuation,
|
|
564
|
+
...(options?.judgeBackgroundContinuation
|
|
565
|
+
? { judgeBackgroundContinuation: options.judgeBackgroundContinuation }
|
|
566
|
+
: {}),
|
|
541
567
|
foregroundTaskResumeSilenceMs: options?.foregroundTaskResumeSilenceMs ?? 20,
|
|
542
568
|
...(options?.foregroundTaskMaxInterveningTurns !== undefined
|
|
543
569
|
? {
|
|
@@ -576,17 +602,550 @@ async function startForegroundTaskBargeInScenario(options?: {
|
|
|
576
602
|
return { calls, frames, session, spawnBackgroundContinuation };
|
|
577
603
|
}
|
|
578
604
|
|
|
605
|
+
describe("LiveVoiceSession subagent outcomes", () => {
|
|
606
|
+
function outcome(subagentId: string, status = "completed") {
|
|
607
|
+
return {
|
|
608
|
+
taskId: subagentId,
|
|
609
|
+
message: `Task ${subagentId} ${status}`,
|
|
610
|
+
metadata: {
|
|
611
|
+
subagentNotification: { subagentId, label: subagentId, status },
|
|
612
|
+
},
|
|
613
|
+
cronRunId: "run-123",
|
|
614
|
+
};
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
function immediateTts(): LiveVoiceTtsStreamer {
|
|
618
|
+
return async (options) => {
|
|
619
|
+
options.onAudioChunk(makeTtsChunk("assistant audio"));
|
|
620
|
+
return makeTtsResult("assistant audio");
|
|
621
|
+
};
|
|
622
|
+
}
|
|
623
|
+
|
|
624
|
+
test("speaks independent outcomes through hidden tool-capable turns", async () => {
|
|
625
|
+
const { calls, startVoiceTurn } = makeAutoCompletingTurnStarter([
|
|
626
|
+
"The comparison is ready.",
|
|
627
|
+
"The export failed. It needs a connection.",
|
|
628
|
+
]);
|
|
629
|
+
const { session, frames } = createHarness({
|
|
630
|
+
startVoiceTurn,
|
|
631
|
+
streamTtsAudio: immediateTts(),
|
|
632
|
+
continuationAnnounceSilenceMs: 20,
|
|
633
|
+
});
|
|
634
|
+
const before = injectMessageIntoParentMock.mock.calls.length;
|
|
635
|
+
await session.start();
|
|
636
|
+
try {
|
|
637
|
+
session.receiveSubagentNotification(outcome("task-1"));
|
|
638
|
+
session.receiveSubagentNotification(outcome("task-2", "failed"));
|
|
639
|
+
await waitFor(() => countType(frames, "tts_done") === 2);
|
|
640
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
641
|
+
expect(calls.map((call) => call.subagentNotification?.taskId)).toEqual([
|
|
642
|
+
"task-1",
|
|
643
|
+
"task-2",
|
|
644
|
+
]);
|
|
645
|
+
for (const call of calls) {
|
|
646
|
+
expect(call.hiddenSyntheticPrompt).toBe(true);
|
|
647
|
+
expect(call.directEscalated).toBe(true);
|
|
648
|
+
expect(call.subagentNotification?.cronRunId).toBe("run-123");
|
|
649
|
+
expect(call.voiceControlPrompt).toContain(
|
|
650
|
+
"internal background task update",
|
|
651
|
+
);
|
|
652
|
+
}
|
|
653
|
+
expect(countType(frames, "tts_audio")).toBeGreaterThanOrEqual(2);
|
|
654
|
+
} finally {
|
|
655
|
+
await session.close("client_end");
|
|
656
|
+
}
|
|
657
|
+
expect(injectMessageIntoParentMock.mock.calls.length).toBe(before);
|
|
658
|
+
});
|
|
659
|
+
|
|
660
|
+
test("silently consumes a redundant update and still speaks the next useful result", async () => {
|
|
661
|
+
const { calls, startVoiceTurn } = makeAutoCompletingTurnStarter([
|
|
662
|
+
TASK_UPDATE_SILENT_MARKER,
|
|
663
|
+
"The export is ready.",
|
|
664
|
+
]);
|
|
665
|
+
const { session, frames } = createHarness({
|
|
666
|
+
startVoiceTurn,
|
|
667
|
+
streamTtsAudio: immediateTts(),
|
|
668
|
+
continuationAnnounceSilenceMs: 10,
|
|
669
|
+
});
|
|
670
|
+
const before = injectMessageIntoParentMock.mock.calls.length;
|
|
671
|
+
await session.start();
|
|
672
|
+
try {
|
|
673
|
+
session.receiveSubagentNotification(outcome("task-1", "running"));
|
|
674
|
+
await waitFor(() => countType(frames, "tts_done") === 1);
|
|
675
|
+
expect(countType(frames, "tts_audio")).toBe(0);
|
|
676
|
+
expect(countType(frames, "assistant_text_delta")).toBe(0);
|
|
677
|
+
session.receiveSubagentNotification(outcome("task-2"));
|
|
678
|
+
await waitFor(() => countType(frames, "tts_done") === 2);
|
|
679
|
+
await new Promise((resolve) => setTimeout(resolve, 30));
|
|
680
|
+
expect(calls).toHaveLength(2);
|
|
681
|
+
expect(countType(frames, "tts_audio")).toBe(1);
|
|
682
|
+
} finally {
|
|
683
|
+
await session.close("client_end");
|
|
684
|
+
}
|
|
685
|
+
expect(injectMessageIntoParentMock.mock.calls.length).toBe(before);
|
|
686
|
+
});
|
|
687
|
+
|
|
688
|
+
test("an interrupted silent decision leaves the update pending", async () => {
|
|
689
|
+
let callbacks: VoiceTurnCallbacks | undefined;
|
|
690
|
+
const { session } = createHarness({
|
|
691
|
+
startVoiceTurn: async (turn) => {
|
|
692
|
+
callbacks = turn.callbacks;
|
|
693
|
+
return { turnId: "turn-1", abort: mock() };
|
|
694
|
+
},
|
|
695
|
+
streamTtsAudio: immediateTts(),
|
|
696
|
+
continuationAnnounceSilenceMs: 10,
|
|
697
|
+
});
|
|
698
|
+
const first = outcome("task-1");
|
|
699
|
+
const before = injectMessageIntoParentMock.mock.calls.length;
|
|
700
|
+
await session.start();
|
|
701
|
+
session.receiveSubagentNotification(first);
|
|
702
|
+
await waitFor(() => callbacks !== undefined);
|
|
703
|
+
callbacks?.assistant_text_delta?.(makeTextDelta(TASK_UPDATE_SILENT_MARKER));
|
|
704
|
+
await session.handleClientFrame({ type: "interrupt" });
|
|
705
|
+
callbacks?.message_complete?.(makeMessageComplete());
|
|
706
|
+
await session.close("client_end");
|
|
707
|
+
expect(injectMessageIntoParentMock.mock.calls.slice(before)).toEqual([
|
|
708
|
+
[
|
|
709
|
+
"conversation-123",
|
|
710
|
+
first.message,
|
|
711
|
+
first.metadata,
|
|
712
|
+
{ cronRunId: "run-123", bypassLiveVoice: true },
|
|
713
|
+
],
|
|
714
|
+
]);
|
|
715
|
+
});
|
|
716
|
+
|
|
717
|
+
test("waits through user speech and the current reply before announcing", async () => {
|
|
718
|
+
const calls: VoiceTurnOptions[] = [];
|
|
719
|
+
const { session } = createHarness({
|
|
720
|
+
finals: ["explain this setting"],
|
|
721
|
+
continuationAnnounceSilenceMs: 10,
|
|
722
|
+
streamTtsAudio: immediateTts(),
|
|
723
|
+
startVoiceTurn: async (turn) => {
|
|
724
|
+
calls.push(turn);
|
|
725
|
+
return { turnId: `turn-${calls.length}`, abort: mock() };
|
|
726
|
+
},
|
|
727
|
+
});
|
|
728
|
+
await session.start();
|
|
729
|
+
try {
|
|
730
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
731
|
+
session.receiveSubagentNotification(outcome("task-1"));
|
|
732
|
+
await waitFor(() => calls.length === 1);
|
|
733
|
+
expect(calls[0]?.content).toBe("explain this setting");
|
|
734
|
+
await new Promise((resolve) => setTimeout(resolve, 40));
|
|
735
|
+
expect(calls).toHaveLength(1);
|
|
736
|
+
calls[0]?.callbacks?.assistant_text_delta?.(
|
|
737
|
+
makeTextDelta("This is the setting."),
|
|
738
|
+
);
|
|
739
|
+
calls[0]?.callbacks?.message_complete?.(makeMessageComplete());
|
|
740
|
+
await waitFor(() => calls.length === 2);
|
|
741
|
+
expect(calls[1]?.subagentNotification?.taskId).toBe("task-1");
|
|
742
|
+
} finally {
|
|
743
|
+
await session.close("client_end");
|
|
744
|
+
}
|
|
745
|
+
});
|
|
746
|
+
|
|
747
|
+
test.each(["what does this setting mean", ""])(
|
|
748
|
+
"a barge-in (%j) retains both outcomes without spawning another worker",
|
|
749
|
+
async (utterance) => {
|
|
750
|
+
const calls: VoiceTurnOptions[] = [];
|
|
751
|
+
const spawnBackgroundContinuation = mock(
|
|
752
|
+
async () => "unexpected continuation",
|
|
753
|
+
);
|
|
754
|
+
const { session, frames } = createHarness({
|
|
755
|
+
finals: [utterance],
|
|
756
|
+
continuationAnnounceSilenceMs: 10,
|
|
757
|
+
streamTtsAudio: immediateTts(),
|
|
758
|
+
spawnBackgroundContinuation,
|
|
759
|
+
startVoiceTurn: async (turn) => {
|
|
760
|
+
calls.push(turn);
|
|
761
|
+
if (calls.length > 1) {
|
|
762
|
+
turn.callbacks?.assistant_text_delta?.(
|
|
763
|
+
makeTextDelta("Here is the answer."),
|
|
764
|
+
);
|
|
765
|
+
turn.callbacks?.message_complete?.(makeMessageComplete());
|
|
766
|
+
}
|
|
767
|
+
return { turnId: `turn-${calls.length}`, abort: mock() };
|
|
768
|
+
},
|
|
769
|
+
});
|
|
770
|
+
await session.start();
|
|
771
|
+
try {
|
|
772
|
+
session.receiveSubagentNotification(outcome("task-1"));
|
|
773
|
+
session.receiveSubagentNotification(outcome("task-2"));
|
|
774
|
+
await waitFor(() => calls.length === 1);
|
|
775
|
+
await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
|
|
776
|
+
await waitFor(
|
|
777
|
+
() => countType(frames, "tts_done") === (utterance ? 3 : 2),
|
|
778
|
+
);
|
|
779
|
+
expect(calls.map((call) => call.subagentNotification?.taskId)).toEqual([
|
|
780
|
+
"task-1",
|
|
781
|
+
...(utterance ? [undefined] : []),
|
|
782
|
+
"task-1",
|
|
783
|
+
"task-2",
|
|
784
|
+
]);
|
|
785
|
+
expect(spawnBackgroundContinuation).not.toHaveBeenCalled();
|
|
786
|
+
} finally {
|
|
787
|
+
await session.close("client_end");
|
|
788
|
+
}
|
|
789
|
+
},
|
|
790
|
+
);
|
|
791
|
+
|
|
792
|
+
test("a completion replacing interrupted findings carries the delivery receipt into its retry", async () => {
|
|
793
|
+
const calls: VoiceTurnOptions[] = [];
|
|
794
|
+
const { session, frames } = createHarness({
|
|
795
|
+
finals: ["tell me about birds"],
|
|
796
|
+
continuationAnnounceSilenceMs: 10,
|
|
797
|
+
streamTtsAudio: immediateTts(),
|
|
798
|
+
startVoiceTurn: async (turn) => {
|
|
799
|
+
calls.push(turn);
|
|
800
|
+
return { turnId: `turn-${calls.length}`, abort: mock() };
|
|
801
|
+
},
|
|
802
|
+
});
|
|
803
|
+
await session.start();
|
|
804
|
+
try {
|
|
805
|
+
session.receiveSubagentNotification(outcome("task-1", "running"));
|
|
806
|
+
await waitFor(() => calls.length === 1);
|
|
807
|
+
calls[0]?.callbacks?.assistant_text_delta?.(
|
|
808
|
+
makeTextDelta("Here are all the generated findings."),
|
|
809
|
+
);
|
|
810
|
+
await waitFor(() => countType(frames, "tts_audio") > 0);
|
|
811
|
+
await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
|
|
812
|
+
await waitFor(() => calls.length === 2);
|
|
813
|
+
session.receiveSubagentNotification(outcome("task-1", "completed"));
|
|
814
|
+
calls[1]?.callbacks?.assistant_text_delta?.(
|
|
815
|
+
makeTextDelta("Birds have feathers."),
|
|
816
|
+
);
|
|
817
|
+
calls[1]?.callbacks?.message_complete?.(makeMessageComplete());
|
|
818
|
+
await waitFor(() => calls.length === 3);
|
|
819
|
+
expect(calls[2]?.subagentNotification?.message).toBe(
|
|
820
|
+
"Task task-1 completed",
|
|
821
|
+
);
|
|
822
|
+
expect(calls[2]?.voiceControlPrompt).toContain(
|
|
823
|
+
"interrupted before playback completed",
|
|
824
|
+
);
|
|
825
|
+
expect(calls[2]?.voiceControlPrompt).toContain("Birds have feathers.");
|
|
826
|
+
expect(calls[2]?.voiceControlPrompt).not.toContain(
|
|
827
|
+
"Here are all the generated findings.",
|
|
828
|
+
);
|
|
829
|
+
} finally {
|
|
830
|
+
await session.close("client_end");
|
|
831
|
+
}
|
|
832
|
+
});
|
|
833
|
+
|
|
834
|
+
test("hang-up returns every undelivered outcome with its metadata and attribution", async () => {
|
|
835
|
+
const { session } = createHarness({ continuationAnnounceSilenceMs: 1_000 });
|
|
836
|
+
await session.start();
|
|
837
|
+
const first = outcome("task-1");
|
|
838
|
+
const second = outcome("task-2", "running");
|
|
839
|
+
const before = injectMessageIntoParentMock.mock.calls.length;
|
|
840
|
+
session.receiveSubagentNotification(first);
|
|
841
|
+
session.receiveSubagentNotification(second);
|
|
842
|
+
await session.close("client_end");
|
|
843
|
+
await session.close("client_end");
|
|
844
|
+
expect(injectMessageIntoParentMock.mock.calls.slice(before)).toEqual([
|
|
845
|
+
[
|
|
846
|
+
"conversation-123",
|
|
847
|
+
first.message,
|
|
848
|
+
first.metadata,
|
|
849
|
+
{ cronRunId: "run-123", bypassLiveVoice: true },
|
|
850
|
+
],
|
|
851
|
+
[
|
|
852
|
+
"conversation-123",
|
|
853
|
+
second.message,
|
|
854
|
+
second.metadata,
|
|
855
|
+
{ cronRunId: "run-123", bypassLiveVoice: true },
|
|
856
|
+
],
|
|
857
|
+
]);
|
|
858
|
+
expect(session.receiveSubagentNotification(outcome("task-3"))).toBe(false);
|
|
859
|
+
});
|
|
860
|
+
|
|
861
|
+
test("hang-up hands back results only after the voice turn's queue-clearing abort settles", async () => {
|
|
862
|
+
let finishTeardown!: () => void;
|
|
863
|
+
const teardown = new Promise<void>((resolve) => {
|
|
864
|
+
finishTeardown = resolve;
|
|
865
|
+
});
|
|
866
|
+
const abort = mock(() => {});
|
|
867
|
+
const startVoiceTurn = mock(async () => ({ turnId: "turn-1", abort }));
|
|
868
|
+
const { session } = createHarness({
|
|
869
|
+
startVoiceTurn,
|
|
870
|
+
getTurnTeardown: () => teardown,
|
|
871
|
+
continuationAnnounceSilenceMs: 10,
|
|
872
|
+
});
|
|
873
|
+
await session.start();
|
|
874
|
+
const first = outcome("task-1");
|
|
875
|
+
session.receiveSubagentNotification(first);
|
|
876
|
+
await waitFor(() => startVoiceTurn.mock.calls.length === 1);
|
|
877
|
+
const before = injectMessageIntoParentMock.mock.calls.length;
|
|
878
|
+
const closing = session.close("client_end");
|
|
879
|
+
const duringClose = outcome("task-2");
|
|
880
|
+
expect(session.receiveSubagentNotification(duringClose)).toBe(true);
|
|
881
|
+
await waitFor(() => abort.mock.calls.length > 0);
|
|
882
|
+
const duringTeardown = outcome("task-3");
|
|
883
|
+
expect(session.receiveSubagentNotification(duringTeardown)).toBe(true);
|
|
884
|
+
expect(injectMessageIntoParentMock.mock.calls.length).toBe(before);
|
|
885
|
+
finishTeardown();
|
|
886
|
+
await closing;
|
|
887
|
+
expect(injectMessageIntoParentMock.mock.calls.slice(before)).toEqual(
|
|
888
|
+
[first, duringClose, duringTeardown].map((notification) => [
|
|
889
|
+
"conversation-123",
|
|
890
|
+
notification.message,
|
|
891
|
+
notification.metadata,
|
|
892
|
+
{ cronRunId: "run-123", bypassLiveVoice: true },
|
|
893
|
+
]),
|
|
894
|
+
);
|
|
895
|
+
});
|
|
896
|
+
|
|
897
|
+
test.each(["resolved", "rejected"])(
|
|
898
|
+
"hang-up waits for pending startup (%s) before handing back results",
|
|
899
|
+
async (mode) => {
|
|
900
|
+
let finishStartup!: () => void;
|
|
901
|
+
const startup = new Promise<void>((resolve) => {
|
|
902
|
+
finishStartup = resolve;
|
|
903
|
+
});
|
|
904
|
+
let finishTeardown!: () => void;
|
|
905
|
+
const teardown = new Promise<void>((resolve) => {
|
|
906
|
+
finishTeardown = resolve;
|
|
907
|
+
});
|
|
908
|
+
let registeredTeardown: Promise<void> | undefined;
|
|
909
|
+
let turn: VoiceTurnOptions | undefined;
|
|
910
|
+
const abort = mock(() => {});
|
|
911
|
+
const { session } = createHarness({
|
|
912
|
+
startVoiceTurn: async (options) => {
|
|
913
|
+
turn = options;
|
|
914
|
+
await startup;
|
|
915
|
+
if (mode === "rejected") {
|
|
916
|
+
throw new Error("startup unavailable");
|
|
917
|
+
}
|
|
918
|
+
registeredTeardown = teardown;
|
|
919
|
+
return { turnId: "turn-1", abort };
|
|
920
|
+
},
|
|
921
|
+
getTurnTeardown: () => registeredTeardown,
|
|
922
|
+
continuationAnnounceSilenceMs: 10,
|
|
923
|
+
});
|
|
924
|
+
await session.start();
|
|
925
|
+
const first = outcome("task-1");
|
|
926
|
+
const before = injectMessageIntoParentMock.mock.calls.length;
|
|
927
|
+
session.receiveSubagentNotification(first);
|
|
928
|
+
await waitFor(() => turn !== undefined);
|
|
929
|
+
const closing = session.close("client_end");
|
|
930
|
+
await waitFor(() => turn?.signal?.aborted === true);
|
|
931
|
+
expect(registeredTeardown).toBeUndefined();
|
|
932
|
+
expect(injectMessageIntoParentMock.mock.calls.length).toBe(before);
|
|
933
|
+
finishStartup();
|
|
934
|
+
if (mode === "resolved") {
|
|
935
|
+
await waitFor(() => abort.mock.calls.length > 0);
|
|
936
|
+
expect(injectMessageIntoParentMock.mock.calls.length).toBe(before);
|
|
937
|
+
}
|
|
938
|
+
finishTeardown();
|
|
939
|
+
await closing;
|
|
940
|
+
expect(injectMessageIntoParentMock.mock.calls.slice(before)).toEqual([
|
|
941
|
+
[
|
|
942
|
+
"conversation-123",
|
|
943
|
+
first.message,
|
|
944
|
+
first.metadata,
|
|
945
|
+
{ cronRunId: "run-123", bypassLiveVoice: true },
|
|
946
|
+
],
|
|
947
|
+
]);
|
|
948
|
+
},
|
|
949
|
+
);
|
|
950
|
+
|
|
951
|
+
test("a failed announcement waits for a user turn before retrying", async () => {
|
|
952
|
+
const calls: VoiceTurnOptions[] = [];
|
|
953
|
+
const { session, frames } = createHarness({
|
|
954
|
+
finals: ["are you still there"],
|
|
955
|
+
continuationAnnounceSilenceMs: 10,
|
|
956
|
+
streamTtsAudio: immediateTts(),
|
|
957
|
+
startVoiceTurn: async (turn) => {
|
|
958
|
+
calls.push(turn);
|
|
959
|
+
if (calls.length === 1) {
|
|
960
|
+
throw new Error("conversation busy");
|
|
961
|
+
}
|
|
962
|
+
turn.callbacks?.assistant_text_delta?.(
|
|
963
|
+
makeTextDelta("Here is the answer."),
|
|
964
|
+
);
|
|
965
|
+
turn.callbacks?.message_complete?.(makeMessageComplete());
|
|
966
|
+
return { turnId: `turn-${calls.length}`, abort: mock() };
|
|
967
|
+
},
|
|
968
|
+
});
|
|
969
|
+
await session.start();
|
|
970
|
+
try {
|
|
971
|
+
session.receiveSubagentNotification(outcome("task-1"));
|
|
972
|
+
await waitFor(() => calls.length === 1);
|
|
973
|
+
await new Promise((resolve) => setTimeout(resolve, 60));
|
|
974
|
+
expect(calls).toHaveLength(1);
|
|
975
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
976
|
+
await waitFor(() => countType(frames, "tts_done") === 2);
|
|
977
|
+
expect(calls.map((call) => call.subagentNotification?.taskId)).toEqual([
|
|
978
|
+
"task-1",
|
|
979
|
+
undefined,
|
|
980
|
+
"task-1",
|
|
981
|
+
]);
|
|
982
|
+
} finally {
|
|
983
|
+
await session.close("client_end");
|
|
984
|
+
}
|
|
985
|
+
});
|
|
986
|
+
|
|
987
|
+
test.each(["throws", "empty", "unspeakable"])(
|
|
988
|
+
"speech without delivered audio (%s) keeps the result for hang-up delivery",
|
|
989
|
+
async (mode) => {
|
|
990
|
+
const { startVoiceTurn, calls } = makeAutoCompletingTurnStarter([
|
|
991
|
+
mode === "unspeakable" ? "" : "The result is ready.",
|
|
992
|
+
]);
|
|
993
|
+
const { session, frames } = createHarness({
|
|
994
|
+
startVoiceTurn,
|
|
995
|
+
continuationAnnounceSilenceMs: 10,
|
|
996
|
+
streamTtsAudio: async () => {
|
|
997
|
+
if (mode === "throws") {
|
|
998
|
+
throw new Error("speech unavailable");
|
|
999
|
+
}
|
|
1000
|
+
return { ...makeTtsResult(""), chunks: 0 };
|
|
1001
|
+
},
|
|
1002
|
+
});
|
|
1003
|
+
await session.start();
|
|
1004
|
+
const first = outcome("task-1");
|
|
1005
|
+
const before = injectMessageIntoParentMock.mock.calls.length;
|
|
1006
|
+
session.receiveSubagentNotification(first);
|
|
1007
|
+
await waitFor(() => countType(frames, "tts_done") === 1);
|
|
1008
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
1009
|
+
expect(calls).toHaveLength(1);
|
|
1010
|
+
await session.close("client_end");
|
|
1011
|
+
expect(injectMessageIntoParentMock.mock.calls.slice(before)).toEqual([
|
|
1012
|
+
[
|
|
1013
|
+
"conversation-123",
|
|
1014
|
+
first.message,
|
|
1015
|
+
first.metadata,
|
|
1016
|
+
{ cronRunId: "run-123", bypassLiveVoice: true },
|
|
1017
|
+
],
|
|
1018
|
+
]);
|
|
1019
|
+
},
|
|
1020
|
+
);
|
|
1021
|
+
|
|
1022
|
+
test("progress audio alone does not acknowledge the outcome", async () => {
|
|
1023
|
+
const calls: VoiceTurnOptions[] = [];
|
|
1024
|
+
const { session, frames } = createHarness({
|
|
1025
|
+
startVoiceTurn: async (turn) => {
|
|
1026
|
+
calls.push(turn);
|
|
1027
|
+
return { turnId: "turn-1", abort: mock() };
|
|
1028
|
+
},
|
|
1029
|
+
streamTtsAudio: immediateTts(),
|
|
1030
|
+
continuationAnnounceSilenceMs: 10,
|
|
1031
|
+
frontModelConfig: {
|
|
1032
|
+
progress: {
|
|
1033
|
+
enabled: true,
|
|
1034
|
+
opsThreshold: 3,
|
|
1035
|
+
idleIntervalMs: 40,
|
|
1036
|
+
maxSilenceMs: 40,
|
|
1037
|
+
longOpMs: 15_000,
|
|
1038
|
+
minGapMs: 60_000,
|
|
1039
|
+
generationTimeoutMs: 1_500,
|
|
1040
|
+
},
|
|
1041
|
+
},
|
|
1042
|
+
progressNarrator: { generateProgressText: async () => "Still working." },
|
|
1043
|
+
});
|
|
1044
|
+
const first = outcome("task-1");
|
|
1045
|
+
const before = injectMessageIntoParentMock.mock.calls.length;
|
|
1046
|
+
await session.start();
|
|
1047
|
+
try {
|
|
1048
|
+
session.receiveSubagentNotification(first);
|
|
1049
|
+
await waitFor(() => countType(frames, "tts_audio") === 1);
|
|
1050
|
+
calls[0]?.callbacks?.message_complete?.(makeMessageComplete());
|
|
1051
|
+
await waitFor(() => countType(frames, "tts_done") === 1);
|
|
1052
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
1053
|
+
expect(calls).toHaveLength(1);
|
|
1054
|
+
} finally {
|
|
1055
|
+
await session.close("client_end");
|
|
1056
|
+
}
|
|
1057
|
+
expect(injectMessageIntoParentMock.mock.calls.slice(before)).toEqual([
|
|
1058
|
+
[
|
|
1059
|
+
"conversation-123",
|
|
1060
|
+
first.message,
|
|
1061
|
+
first.metadata,
|
|
1062
|
+
{ cronRunId: "run-123", bypassLiveVoice: true },
|
|
1063
|
+
],
|
|
1064
|
+
]);
|
|
1065
|
+
});
|
|
1066
|
+
|
|
1067
|
+
test("speech during the playback tail retries the outcome after answering the user", async () => {
|
|
1068
|
+
const { startVoiceTurn, calls } = makeAutoCompletingTurnStarter([
|
|
1069
|
+
"The result is ready.",
|
|
1070
|
+
"Here is the setting.",
|
|
1071
|
+
"The result is ready.",
|
|
1072
|
+
]);
|
|
1073
|
+
const audio = "x".repeat(28_800);
|
|
1074
|
+
const { session, frames } = createHarness({
|
|
1075
|
+
finals: ["explain this setting"],
|
|
1076
|
+
startVoiceTurn,
|
|
1077
|
+
continuationAnnounceSilenceMs: 10,
|
|
1078
|
+
streamTtsAudio: async (options) => {
|
|
1079
|
+
options.onAudioChunk(makeTtsChunk(audio));
|
|
1080
|
+
return makeTtsResult(audio);
|
|
1081
|
+
},
|
|
1082
|
+
});
|
|
1083
|
+
await session.start();
|
|
1084
|
+
try {
|
|
1085
|
+
session.receiveSubagentNotification(outcome("task-1"));
|
|
1086
|
+
await waitFor(() => countType(frames, "tts_done") === 1);
|
|
1087
|
+
await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
|
|
1088
|
+
await waitFor(() => countType(frames, "tts_done") === 3);
|
|
1089
|
+
expect(calls.map((call) => call.subagentNotification?.taskId)).toEqual([
|
|
1090
|
+
"task-1",
|
|
1091
|
+
undefined,
|
|
1092
|
+
"task-1",
|
|
1093
|
+
]);
|
|
1094
|
+
} finally {
|
|
1095
|
+
await session.close("client_end");
|
|
1096
|
+
}
|
|
1097
|
+
});
|
|
1098
|
+
|
|
1099
|
+
test("hang-up during playback preserves a generated but unheard result", async () => {
|
|
1100
|
+
const { startVoiceTurn } = makeAutoCompletingTurnStarter([
|
|
1101
|
+
"The comparison is ready.",
|
|
1102
|
+
]);
|
|
1103
|
+
const audio = "x".repeat(28_800);
|
|
1104
|
+
const { session, frames } = createHarness({
|
|
1105
|
+
startVoiceTurn,
|
|
1106
|
+
continuationAnnounceSilenceMs: 10,
|
|
1107
|
+
streamTtsAudio: async (options) => {
|
|
1108
|
+
options.onAudioChunk(makeTtsChunk(audio));
|
|
1109
|
+
return makeTtsResult(audio);
|
|
1110
|
+
},
|
|
1111
|
+
});
|
|
1112
|
+
await session.start();
|
|
1113
|
+
const first = outcome("task-1");
|
|
1114
|
+
const before = injectMessageIntoParentMock.mock.calls.length;
|
|
1115
|
+
session.receiveSubagentNotification(first);
|
|
1116
|
+
await waitFor(() => countType(frames, "tts_done") === 1);
|
|
1117
|
+
await session.close("client_end");
|
|
1118
|
+
expect(injectMessageIntoParentMock.mock.calls.slice(before)).toEqual([
|
|
1119
|
+
[
|
|
1120
|
+
"conversation-123",
|
|
1121
|
+
first.message,
|
|
1122
|
+
first.metadata,
|
|
1123
|
+
{ cronRunId: "run-123", bypassLiveVoice: true },
|
|
1124
|
+
],
|
|
1125
|
+
]);
|
|
1126
|
+
});
|
|
1127
|
+
});
|
|
1128
|
+
|
|
579
1129
|
describe("LiveVoiceSession server VAD", () => {
|
|
580
1130
|
// The foreground-wins classification consults the tool registry's owner map
|
|
581
1131
|
// (a tool is only provably read-only when it is the trusted built-in);
|
|
582
1132
|
// register the core baseline so built-in names resolve like in the daemon.
|
|
583
1133
|
beforeAll(() => {
|
|
584
1134
|
__resetRegistryForTesting();
|
|
585
|
-
registerSkillTools(
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
1135
|
+
registerSkillTools(
|
|
1136
|
+
"computer-use-test",
|
|
1137
|
+
["computer_use_key", "computer_use_observe", "read_file"].map((name) =>
|
|
1138
|
+
finalizeTool({
|
|
1139
|
+
name,
|
|
1140
|
+
description: name,
|
|
1141
|
+
category: "computer-use",
|
|
1142
|
+
defaultRiskLevel: RiskLevel.Low,
|
|
1143
|
+
executionTarget: "host",
|
|
1144
|
+
input_schema: { type: "object", properties: {} },
|
|
1145
|
+
execute: async () => ({ content: "ok", isError: false }),
|
|
1146
|
+
}),
|
|
1147
|
+
),
|
|
1148
|
+
);
|
|
590
1149
|
});
|
|
591
1150
|
afterAll(() => unregisterSkillTools("computer-use-test"));
|
|
592
1151
|
|
|
@@ -739,6 +1298,85 @@ describe("LiveVoiceSession server VAD", () => {
|
|
|
739
1298
|
await waitFor(() => foregroundTaskStateOf(session) === null);
|
|
740
1299
|
});
|
|
741
1300
|
|
|
1301
|
+
function scriptedJudge(verdict: Promise<boolean> | boolean) {
|
|
1302
|
+
const judged: Array<{
|
|
1303
|
+
interruptedRequest: string;
|
|
1304
|
+
interruption: string | null;
|
|
1305
|
+
}> = [];
|
|
1306
|
+
const judge: LiveVoiceContinuationJudge = async (args) => {
|
|
1307
|
+
const interruption = await args.interruption;
|
|
1308
|
+
judged.push({
|
|
1309
|
+
interruptedRequest: args.interruptedRequest,
|
|
1310
|
+
interruption,
|
|
1311
|
+
});
|
|
1312
|
+
const keep = await verdict;
|
|
1313
|
+
return keep
|
|
1314
|
+
? { keep: true, outcome: "keep", noul: 0.9, latencyMs: 1 }
|
|
1315
|
+
: { keep: false, outcome: "drop", noul: 0.1, latencyMs: 1 };
|
|
1316
|
+
};
|
|
1317
|
+
return { judge, judged };
|
|
1318
|
+
}
|
|
1319
|
+
|
|
1320
|
+
async function answerInterruption(calls: VoiceTurnOptions[]) {
|
|
1321
|
+
const answer = calls[2];
|
|
1322
|
+
answer?.callbacks?.assistant_text_delta?.(makeTextDelta("Okay."));
|
|
1323
|
+
answer?.callbacks?.message_complete?.(makeMessageComplete());
|
|
1324
|
+
}
|
|
1325
|
+
|
|
1326
|
+
test("a suspended task the caller called off is cleared, not resumed", async () => {
|
|
1327
|
+
const { judge, judged } = scriptedJudge(false);
|
|
1328
|
+
const { calls, session } = await startForegroundTaskBargeInScenario({
|
|
1329
|
+
startHostTool: false,
|
|
1330
|
+
judgeBackgroundContinuation: judge,
|
|
1331
|
+
});
|
|
1332
|
+
|
|
1333
|
+
await answerInterruption(calls);
|
|
1334
|
+
await waitFor(() => judged.length === 1);
|
|
1335
|
+
expect(judged[0]).toEqual({
|
|
1336
|
+
interruptedRequest: "change the title",
|
|
1337
|
+
interruption: "what title is there now",
|
|
1338
|
+
});
|
|
1339
|
+
await waitFor(() => foregroundTaskStateOf(session) === null);
|
|
1340
|
+
// Well past the resume silence: no hidden resume turn was launched.
|
|
1341
|
+
await new Promise((resolve) => setTimeout(resolve, 80));
|
|
1342
|
+
expect(calls).toHaveLength(3);
|
|
1343
|
+
});
|
|
1344
|
+
|
|
1345
|
+
test("a suspended task the caller still wants resumes as before", async () => {
|
|
1346
|
+
const { judge } = scriptedJudge(true);
|
|
1347
|
+
const { calls } = await startForegroundTaskBargeInScenario({
|
|
1348
|
+
startHostTool: false,
|
|
1349
|
+
judgeBackgroundContinuation: judge,
|
|
1350
|
+
});
|
|
1351
|
+
|
|
1352
|
+
await answerInterruption(calls);
|
|
1353
|
+
await waitFor(() => calls.length === 4);
|
|
1354
|
+
expect(calls[3]).toMatchObject({ hiddenSyntheticPrompt: true });
|
|
1355
|
+
expect(calls[3]?.content).toContain("change the title");
|
|
1356
|
+
});
|
|
1357
|
+
|
|
1358
|
+
test("the resume waits for a verdict still in flight", async () => {
|
|
1359
|
+
let decide!: (keep: boolean) => void;
|
|
1360
|
+
const { judge } = scriptedJudge(
|
|
1361
|
+
new Promise<boolean>((resolve) => {
|
|
1362
|
+
decide = resolve;
|
|
1363
|
+
}),
|
|
1364
|
+
);
|
|
1365
|
+
const { calls, session } = await startForegroundTaskBargeInScenario({
|
|
1366
|
+
startHostTool: false,
|
|
1367
|
+
judgeBackgroundContinuation: judge,
|
|
1368
|
+
});
|
|
1369
|
+
|
|
1370
|
+
await answerInterruption(calls);
|
|
1371
|
+
await new Promise((resolve) => setTimeout(resolve, 80));
|
|
1372
|
+
expect(calls).toHaveLength(3);
|
|
1373
|
+
|
|
1374
|
+
decide(false);
|
|
1375
|
+
await waitFor(() => foregroundTaskStateOf(session) === null);
|
|
1376
|
+
await new Promise((resolve) => setTimeout(resolve, 40));
|
|
1377
|
+
expect(calls).toHaveLength(3);
|
|
1378
|
+
});
|
|
1379
|
+
|
|
742
1380
|
test("a new barged-in task becomes the resume anchor", async () => {
|
|
743
1381
|
const calls: VoiceTurnOptions[] = [];
|
|
744
1382
|
const startVoiceTurn = mock(async (turn: VoiceTurnOptions) => {
|
|
@@ -1489,6 +2127,133 @@ describe("LiveVoiceSession server VAD", () => {
|
|
|
1489
2127
|
expect(spawnArgs?.objective).toContain("first question");
|
|
1490
2128
|
});
|
|
1491
2129
|
|
|
2130
|
+
async function bargeInWithJudge(keep: boolean) {
|
|
2131
|
+
const judged: Array<{
|
|
2132
|
+
interruptedRequest: string;
|
|
2133
|
+
interruption: string | null;
|
|
2134
|
+
}> = [];
|
|
2135
|
+
const judgeBackgroundContinuation: LiveVoiceContinuationJudge = async (
|
|
2136
|
+
args,
|
|
2137
|
+
) => {
|
|
2138
|
+
const interruption = await args.interruption;
|
|
2139
|
+
judged.push({
|
|
2140
|
+
interruptedRequest: args.interruptedRequest,
|
|
2141
|
+
interruption,
|
|
2142
|
+
});
|
|
2143
|
+
return keep
|
|
2144
|
+
? { keep: true, outcome: "keep", noul: 0.9, latencyMs: 1 }
|
|
2145
|
+
: { keep: false, outcome: "drop", noul: 0.1, latencyMs: 1 };
|
|
2146
|
+
};
|
|
2147
|
+
const spawnBackgroundContinuation = mock(
|
|
2148
|
+
async (_args: {
|
|
2149
|
+
parentConversationId: string;
|
|
2150
|
+
objective: string;
|
|
2151
|
+
label: string;
|
|
2152
|
+
signal: AbortSignal;
|
|
2153
|
+
}): Promise<string> => "",
|
|
2154
|
+
);
|
|
2155
|
+
const calls: VoiceTurnOptions[] = [];
|
|
2156
|
+
const startVoiceTurn = mock(async (options: VoiceTurnOptions) => {
|
|
2157
|
+
calls.push(options);
|
|
2158
|
+
return { turnId: `bridge-turn-${calls.length}`, abort: mock() };
|
|
2159
|
+
});
|
|
2160
|
+
const streamTtsAudio = mock(async (options: LiveVoiceTtsOptions) => {
|
|
2161
|
+
options.onAudioChunk(makeTtsChunk("assistant audio"));
|
|
2162
|
+
return makeTtsResult("assistant audio");
|
|
2163
|
+
});
|
|
2164
|
+
const { frames, session } = createHarness({
|
|
2165
|
+
finals: ["first question", "never mind that"],
|
|
2166
|
+
startVoiceTurn,
|
|
2167
|
+
streamTtsAudio,
|
|
2168
|
+
spawnBackgroundContinuation,
|
|
2169
|
+
judgeBackgroundContinuation,
|
|
2170
|
+
});
|
|
2171
|
+
|
|
2172
|
+
await session.start();
|
|
2173
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
2174
|
+
await waitFor(() => frames.some((frame) => frame.type === "thinking"));
|
|
2175
|
+
await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
|
|
2176
|
+
await waitFor(() => judged.length === 1);
|
|
2177
|
+
await flushAsyncCallbacks();
|
|
2178
|
+
return { judged, spawnBackgroundContinuation, calls };
|
|
2179
|
+
}
|
|
2180
|
+
|
|
2181
|
+
test("the continuation judge sees the interrupting words and can drop the continuation", async () => {
|
|
2182
|
+
const { judged, spawnBackgroundContinuation, calls } =
|
|
2183
|
+
await bargeInWithJudge(false);
|
|
2184
|
+
|
|
2185
|
+
expect(judged).toEqual([
|
|
2186
|
+
{ interruptedRequest: "first question", interruption: "never mind that" },
|
|
2187
|
+
]);
|
|
2188
|
+
expect(spawnBackgroundContinuation).not.toHaveBeenCalled();
|
|
2189
|
+
// The foreground turn still carries the interrupted request.
|
|
2190
|
+
const followUp = calls.find((c) => c.content === "never mind that");
|
|
2191
|
+
expect(followUp?.voiceControlPrompt).toContain("first question");
|
|
2192
|
+
});
|
|
2193
|
+
|
|
2194
|
+
test("a speculative interruption reaches the judge only once its turn commits", async () => {
|
|
2195
|
+
const judged: Array<string | null> = [];
|
|
2196
|
+
const judgeBackgroundContinuation: LiveVoiceContinuationJudge = async (
|
|
2197
|
+
args,
|
|
2198
|
+
) => {
|
|
2199
|
+
judged.push(await args.interruption);
|
|
2200
|
+
return { keep: false, outcome: "drop", noul: 0.1, latencyMs: 1 };
|
|
2201
|
+
};
|
|
2202
|
+
let openGate!: () => void;
|
|
2203
|
+
const gate = new Promise<void>((resolve) => {
|
|
2204
|
+
openGate = resolve;
|
|
2205
|
+
});
|
|
2206
|
+
const calls: VoiceTurnOptions[] = [];
|
|
2207
|
+
const startVoiceTurn = mock(async (options: VoiceTurnOptions) => {
|
|
2208
|
+
calls.push(options);
|
|
2209
|
+
if (options.content !== "first question") {
|
|
2210
|
+
// The leg's first word is its verdict: nothing commits until then.
|
|
2211
|
+
void gate.then(() => {
|
|
2212
|
+
options.callbacks?.assistant_text_delta?.(
|
|
2213
|
+
makeTextDelta("Sure thing."),
|
|
2214
|
+
);
|
|
2215
|
+
options.callbacks?.message_complete?.(makeMessageComplete());
|
|
2216
|
+
});
|
|
2217
|
+
}
|
|
2218
|
+
return { turnId: `bridge-turn-${calls.length}`, abort: mock() };
|
|
2219
|
+
});
|
|
2220
|
+
const { frames, session, transcribers } = createHarness({
|
|
2221
|
+
finals: ["first question", "never mind that"],
|
|
2222
|
+
startVoiceTurn,
|
|
2223
|
+
streamTtsAudio: makeImmediateTts(),
|
|
2224
|
+
spawnBackgroundContinuation: mock(async () => ""),
|
|
2225
|
+
judgeBackgroundContinuation,
|
|
2226
|
+
});
|
|
2227
|
+
|
|
2228
|
+
await session.start();
|
|
2229
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
2230
|
+
await waitFor(() => frames.some((frame) => frame.type === "thinking"));
|
|
2231
|
+
await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
|
|
2232
|
+
await waitFor(() => transcribers.length === 2);
|
|
2233
|
+
transcribers[1]?.emit({ type: "partial", text: "never mind that" });
|
|
2234
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
2235
|
+
await waitFor(() => calls.some((c) => c.content === "never mind that"));
|
|
2236
|
+
expect(
|
|
2237
|
+
calls.find((c) => c.content === "never mind that")?.unifiedVerdict,
|
|
2238
|
+
).toBe(true);
|
|
2239
|
+
|
|
2240
|
+
await flushAsyncCallbacks();
|
|
2241
|
+
expect(judged).toEqual([]);
|
|
2242
|
+
|
|
2243
|
+
openGate();
|
|
2244
|
+
await waitFor(() => judged.length === 1);
|
|
2245
|
+
expect(judged).toEqual(["never mind that"]);
|
|
2246
|
+
});
|
|
2247
|
+
|
|
2248
|
+
test("a keep verdict spawns the continuation as before", async () => {
|
|
2249
|
+
const { spawnBackgroundContinuation } = await bargeInWithJudge(true);
|
|
2250
|
+
|
|
2251
|
+
await waitFor(() => spawnBackgroundContinuation.mock.calls.length === 1);
|
|
2252
|
+
expect(spawnBackgroundContinuation.mock.calls[0]?.[0]?.objective).toContain(
|
|
2253
|
+
"first question",
|
|
2254
|
+
);
|
|
2255
|
+
});
|
|
2256
|
+
|
|
1492
2257
|
test("the continuation carries the label the model phrased", async () => {
|
|
1493
2258
|
const labelCalls: Array<{
|
|
1494
2259
|
parentConversationId: string;
|
|
@@ -1884,7 +2649,7 @@ describe("LiveVoiceSession server VAD", () => {
|
|
|
1884
2649
|
return { startVoiceTurn, calls };
|
|
1885
2650
|
}
|
|
1886
2651
|
|
|
1887
|
-
test("a completed continuation
|
|
2652
|
+
test("a completed continuation remains pending across unrelated user turns", async () => {
|
|
1888
2653
|
// Control the continuation's resolution so we know exactly when its result
|
|
1889
2654
|
// is stashed relative to the turns we inspect.
|
|
1890
2655
|
let resolveContinuation: ((result: string) => void) | undefined;
|
|
@@ -1942,14 +2707,14 @@ describe("LiveVoiceSession server VAD", () => {
|
|
|
1942
2707
|
expect(resurfaced?.voiceControlPrompt).toContain("THE_RESULT");
|
|
1943
2708
|
expect(resurfaced?.voiceControlPrompt).toContain("background");
|
|
1944
2709
|
|
|
1945
|
-
//
|
|
2710
|
+
// Context exposure is not delivery, so a later turn still has the result.
|
|
1946
2711
|
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
1947
2712
|
await waitFor(() => calls.some((c) => c.content === "fourth question"));
|
|
1948
2713
|
const later = calls.find((c) => c.content === "fourth question");
|
|
1949
|
-
expect(later?.voiceControlPrompt).
|
|
2714
|
+
expect(later?.voiceControlPrompt).toContain("THE_RESULT");
|
|
1950
2715
|
});
|
|
1951
2716
|
|
|
1952
|
-
test("a client interrupt
|
|
2717
|
+
test("a client interrupt defers delivery without dropping a finished result", async () => {
|
|
1953
2718
|
let resolveContinuation: ((result: string) => void) | undefined;
|
|
1954
2719
|
const spawnBackgroundContinuation = mock(
|
|
1955
2720
|
(_args: {
|
|
@@ -1994,15 +2759,15 @@ describe("LiveVoiceSession server VAD", () => {
|
|
|
1994
2759
|
resolveContinuation?.("THE_RESULT");
|
|
1995
2760
|
await flushAsyncCallbacks();
|
|
1996
2761
|
|
|
1997
|
-
//
|
|
2762
|
+
// Stopping speech defers delivery until the next user turn.
|
|
1998
2763
|
await session.handleClientFrame({ type: "interrupt" });
|
|
1999
2764
|
await flushAsyncCallbacks();
|
|
2000
2765
|
|
|
2001
|
-
//
|
|
2766
|
+
// Finished work remains available to the next turn.
|
|
2002
2767
|
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
2003
2768
|
await waitFor(() => calls.some((c) => c.content === "third question"));
|
|
2004
2769
|
const later = calls.find((c) => c.content === "third question");
|
|
2005
|
-
expect(later?.voiceControlPrompt).
|
|
2770
|
+
expect(later?.voiceControlPrompt).toContain("THE_RESULT");
|
|
2006
2771
|
});
|
|
2007
2772
|
|
|
2008
2773
|
test("an empty continuation result adds no context to the next turn", async () => {
|
|
@@ -2195,7 +2960,7 @@ describe("LiveVoiceSession server VAD", () => {
|
|
|
2195
2960
|
resolvers[1]?.("");
|
|
2196
2961
|
});
|
|
2197
2962
|
|
|
2198
|
-
test("a new barge-in
|
|
2963
|
+
test("a new barge-in preserves an already-finished continuation", async () => {
|
|
2199
2964
|
const resolvers: Array<(result: string) => void> = [];
|
|
2200
2965
|
const spawnBackgroundContinuation = mock(
|
|
2201
2966
|
(_args: {
|
|
@@ -2248,15 +3013,14 @@ describe("LiveVoiceSession server VAD", () => {
|
|
|
2248
3013
|
resolvers[0]?.("A_RESULT");
|
|
2249
3014
|
await flushAsyncCallbacks();
|
|
2250
3015
|
|
|
2251
|
-
// Barge #2
|
|
2252
|
-
// (and detach the newer continuation).
|
|
3016
|
+
// Barge #2 starts a new continuation while the finished result stays pending.
|
|
2253
3017
|
await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
|
|
2254
3018
|
await waitFor(() => calls.some((c) => c.content === "third question"));
|
|
2255
3019
|
await waitFor(() => resolvers.length === 2);
|
|
2256
3020
|
|
|
2257
|
-
// The
|
|
3021
|
+
// The follow-up still has the completed result as context.
|
|
2258
3022
|
const followUp = calls.find((c) => c.content === "third question");
|
|
2259
|
-
expect(followUp?.voiceControlPrompt).
|
|
3023
|
+
expect(followUp?.voiceControlPrompt).toContain("A_RESULT");
|
|
2260
3024
|
|
|
2261
3025
|
// Cleanup the still-pending newer continuation.
|
|
2262
3026
|
resolvers[1]?.("");
|
|
@@ -2494,6 +3258,45 @@ describe("LiveVoiceSession server VAD", () => {
|
|
|
2494
3258
|
expect(announcement?.voiceControlPrompt).toContain("first question");
|
|
2495
3259
|
});
|
|
2496
3260
|
|
|
3261
|
+
test("continuations and explicit workers share delivery and finish independently", async () => {
|
|
3262
|
+
const continuation = makeControlledContinuation();
|
|
3263
|
+
const { startVoiceTurn, calls } = makeResurfaceTurnStarter();
|
|
3264
|
+
const { frames, session } = createHarness({
|
|
3265
|
+
finals: ["first question", ""],
|
|
3266
|
+
startVoiceTurn,
|
|
3267
|
+
streamTtsAudio: makeImmediateTts(),
|
|
3268
|
+
spawnBackgroundContinuation: continuation.spawnBackgroundContinuation,
|
|
3269
|
+
continuationAnnounceSilenceMs: 20,
|
|
3270
|
+
});
|
|
3271
|
+
await session.start();
|
|
3272
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
3273
|
+
await waitFor(() => calls.length === 1);
|
|
3274
|
+
await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
|
|
3275
|
+
await waitFor(
|
|
3276
|
+
() => continuation.spawnBackgroundContinuation.mock.calls.length === 1,
|
|
3277
|
+
);
|
|
3278
|
+
await waitFor(() => countType(frames, "utterance_discarded") === 1);
|
|
3279
|
+
continuation.finish("CONTINUATION_RESULT");
|
|
3280
|
+
session.receiveSubagentNotification({
|
|
3281
|
+
taskId: "task-1",
|
|
3282
|
+
message: "WORKER_RESULT",
|
|
3283
|
+
metadata: {},
|
|
3284
|
+
});
|
|
3285
|
+
await waitFor(() => countType(frames, "tts_done") === 2);
|
|
3286
|
+
expect(
|
|
3287
|
+
calls.filter((call) => call.content === "WORKER_RESULT"),
|
|
3288
|
+
).toHaveLength(1);
|
|
3289
|
+
expect(announcementCount(calls)).toBe(1);
|
|
3290
|
+
expect(announcementOf(calls)?.voiceControlPrompt).toContain(
|
|
3291
|
+
"CONTINUATION_RESULT",
|
|
3292
|
+
);
|
|
3293
|
+
expect(announcementOf(calls)?.directEscalated).toBe(true);
|
|
3294
|
+
await new Promise((resolve) => setTimeout(resolve, 30));
|
|
3295
|
+
const before = injectMessageIntoParentMock.mock.calls.length;
|
|
3296
|
+
await session.close("client_end");
|
|
3297
|
+
expect(injectMessageIntoParentMock.mock.calls.length).toBe(before);
|
|
3298
|
+
});
|
|
3299
|
+
|
|
2497
3300
|
// Deepgram Flux sends interim updates through silence, each an empty
|
|
2498
3301
|
// partial. A call nobody is talking on is still idle.
|
|
2499
3302
|
test("an empty partial on an idle call does not hold the announcement back", async () => {
|
|
@@ -2565,7 +3368,7 @@ describe("LiveVoiceSession server VAD", () => {
|
|
|
2565
3368
|
expect(later?.voiceControlPrompt).not.toContain("THE_RESULT");
|
|
2566
3369
|
});
|
|
2567
3370
|
|
|
2568
|
-
test("
|
|
3371
|
+
test("an unrelated user reply does not consume the queued continuation", async () => {
|
|
2569
3372
|
const continuation = makeControlledContinuation();
|
|
2570
3373
|
const { startVoiceTurn, calls } = makeResurfaceTurnStarter();
|
|
2571
3374
|
const { frames, session } = createHarness({
|
|
@@ -2574,7 +3377,7 @@ describe("LiveVoiceSession server VAD", () => {
|
|
|
2574
3377
|
streamTtsAudio: makeImmediateTts(),
|
|
2575
3378
|
spawnBackgroundContinuation: continuation.spawnBackgroundContinuation,
|
|
2576
3379
|
// Long enough that the user always speaks first.
|
|
2577
|
-
continuationAnnounceSilenceMs:
|
|
3380
|
+
continuationAnnounceSilenceMs: 250,
|
|
2578
3381
|
});
|
|
2579
3382
|
|
|
2580
3383
|
await session.start();
|
|
@@ -2591,14 +3394,16 @@ describe("LiveVoiceSession server VAD", () => {
|
|
|
2591
3394
|
continuation.finish("THE_RESULT");
|
|
2592
3395
|
await flushAsyncCallbacks();
|
|
2593
3396
|
|
|
2594
|
-
// The
|
|
2595
|
-
// and the announcement is dropped rather than repeating it after.
|
|
3397
|
+
// The unrelated reply can see the result, but only says "ok".
|
|
2596
3398
|
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
2597
3399
|
await waitFor(() => calls.some((c) => c.content === "third question"));
|
|
2598
3400
|
const spoken = calls.find((c) => c.content === "third question");
|
|
2599
3401
|
expect(spoken?.voiceControlPrompt).toContain("THE_RESULT");
|
|
2600
|
-
await
|
|
2601
|
-
expect(announcementOf(calls)).
|
|
3402
|
+
await waitFor(() => announcementCount(calls) === 1);
|
|
3403
|
+
expect(announcementOf(calls)?.voiceControlPrompt).toContain("THE_RESULT");
|
|
3404
|
+
expect(announcementOf(calls)?.voiceControlPrompt).toContain(
|
|
3405
|
+
'completed audio playback for this task was: "ok"',
|
|
3406
|
+
);
|
|
2602
3407
|
});
|
|
2603
3408
|
|
|
2604
3409
|
test("an active turn at fire time announces after the turn becomes idle", async () => {
|
|
@@ -2821,52 +3626,42 @@ describe("LiveVoiceSession server VAD", () => {
|
|
|
2821
3626
|
expect(later?.voiceControlPrompt).toContain("THE_RESULT");
|
|
2822
3627
|
});
|
|
2823
3628
|
|
|
2824
|
-
test("an announcement whose launch throws
|
|
3629
|
+
test("an announcement whose launch throws retains its result", async () => {
|
|
2825
3630
|
const { startVoiceTurn, calls } = makeResurfaceTurnStarter();
|
|
2826
3631
|
const { session } = createHarness({
|
|
2827
3632
|
finals: ["stashed question"],
|
|
2828
3633
|
startVoiceTurn,
|
|
2829
3634
|
continuationAnnounceSilenceMs: 20,
|
|
2830
3635
|
});
|
|
2831
|
-
|
|
2832
3636
|
await session.start();
|
|
2833
|
-
await flushAsyncCallbacks();
|
|
2834
|
-
|
|
2835
3637
|
const internals = session as unknown as {
|
|
2836
|
-
|
|
2837
|
-
|
|
2838
|
-
|
|
3638
|
+
receiveTaskOutcome: (
|
|
3639
|
+
outcome: import("../task-outcomes.js").VoiceTaskOutcome,
|
|
3640
|
+
) => boolean;
|
|
2839
3641
|
launchAssistantTurn: (
|
|
2840
3642
|
utterance: unknown,
|
|
2841
3643
|
content: string,
|
|
2842
3644
|
opts?: unknown,
|
|
2843
3645
|
) => Promise<boolean>;
|
|
2844
3646
|
};
|
|
2845
|
-
// The announcement's launch REJECTS rather than returning false — a frame
|
|
2846
|
-
// write failing ahead of the leg's own error handling. The `started ===
|
|
2847
|
-
// false` re-stash never runs on that path, so only the catch can save the
|
|
2848
|
-
// answer.
|
|
2849
3647
|
const launch = internals.launchAssistantTurn.bind(session);
|
|
3648
|
+
const failed = mock(() => {
|
|
3649
|
+
throw new Error("thinking frame write failed");
|
|
3650
|
+
});
|
|
2850
3651
|
internals.launchAssistantTurn = async (utterance, content, opts) => {
|
|
2851
3652
|
if (content === CONTINUATION_DELIVERY_CONTENT) {
|
|
2852
|
-
|
|
3653
|
+
return failed();
|
|
2853
3654
|
}
|
|
2854
3655
|
return launch(utterance, content, opts);
|
|
2855
3656
|
};
|
|
2856
|
-
internals.
|
|
2857
|
-
|
|
2858
|
-
|
|
2859
|
-
|
|
2860
|
-
|
|
2861
|
-
|
|
2862
|
-
|
|
2863
|
-
// The attempt runs (the queue empties) and throws.
|
|
2864
|
-
await waitFor(() => internals.pendingAnnouncement === null);
|
|
2865
|
-
await flushAsyncCallbacks();
|
|
3657
|
+
internals.receiveTaskOutcome({
|
|
3658
|
+
taskId: "continuation-1",
|
|
3659
|
+
message: "THE_RESULT",
|
|
3660
|
+
metadata: {},
|
|
3661
|
+
source: "continuation",
|
|
3662
|
+
});
|
|
3663
|
+
await waitFor(() => failed.mock.calls.length === 1);
|
|
2866
3664
|
expect(announcementOf(calls)).toBeUndefined();
|
|
2867
|
-
expect(internals.pendingContinuationResult).toBe("THE_RESULT");
|
|
2868
|
-
|
|
2869
|
-
// ...so the user's next turn still carries it.
|
|
2870
3665
|
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
2871
3666
|
await waitFor(() => calls.some((c) => c.content === "stashed question"));
|
|
2872
3667
|
expect(
|
|
@@ -2952,7 +3747,7 @@ describe("LiveVoiceSession server VAD", () => {
|
|
|
2952
3747
|
expect(message).toContain("first question");
|
|
2953
3748
|
});
|
|
2954
3749
|
|
|
2955
|
-
test("a
|
|
3750
|
+
test("a speculative user reply preserves the pending announcement", async () => {
|
|
2956
3751
|
const continuation = makeControlledContinuation();
|
|
2957
3752
|
const { startVoiceTurn, calls } = makeResurfaceTurnStarter();
|
|
2958
3753
|
const { frames, session, transcribers } = createHarness({
|
|
@@ -2995,22 +3790,66 @@ describe("LiveVoiceSession server VAD", () => {
|
|
|
2995
3790
|
expect(speculative?.unifiedVerdict).toBe(true);
|
|
2996
3791
|
expect(speculative?.voiceControlPrompt).toContain("THE_RESULT");
|
|
2997
3792
|
|
|
2998
|
-
//
|
|
2999
|
-
|
|
3000
|
-
await new Promise((resolve) => setTimeout(resolve, 600));
|
|
3001
|
-
expect(announcementCount(calls)).toBe(0);
|
|
3793
|
+
// Completing an unrelated speculative reply does not acknowledge delivery.
|
|
3794
|
+
await waitFor(() => announcementCount(calls) === 1);
|
|
3002
3795
|
});
|
|
3003
3796
|
|
|
3004
3797
|
test("a held speculative turn hands the stashed answer back for the replay", async () => {
|
|
3005
3798
|
const continuation = makeControlledContinuation();
|
|
3006
3799
|
const calls: VoiceTurnOptions[] = [];
|
|
3007
|
-
const
|
|
3800
|
+
const claimedTurnIds: string[] = [];
|
|
3801
|
+
const releasedTurnIds = new Set<string>();
|
|
3802
|
+
let activation = 0;
|
|
3803
|
+
let activeSource: ModeSessionSourceHandle | undefined;
|
|
3804
|
+
const modeSessions = {
|
|
3805
|
+
activateSource: (input: {
|
|
3806
|
+
sourceId: string;
|
|
3807
|
+
generation: number;
|
|
3808
|
+
mode: "ambient" | "live_vision";
|
|
3809
|
+
}) => {
|
|
3810
|
+
activation += 1;
|
|
3811
|
+
activeSource = {
|
|
3812
|
+
sourceId: input.sourceId,
|
|
3813
|
+
generation: input.generation,
|
|
3814
|
+
activation,
|
|
3815
|
+
id: `camera-session-${activation}`,
|
|
3816
|
+
mode: input.mode,
|
|
3817
|
+
};
|
|
3818
|
+
return activeSource;
|
|
3819
|
+
},
|
|
3820
|
+
claimTurn: (turnId: string, source: ModeSessionSourceHandle) => {
|
|
3821
|
+
if (source !== activeSource) {
|
|
3822
|
+
return undefined;
|
|
3823
|
+
}
|
|
3824
|
+
claimedTurnIds.push(turnId);
|
|
3825
|
+
return { id: source.id, mode: source.mode };
|
|
3826
|
+
},
|
|
3827
|
+
recordActivity: () => true,
|
|
3828
|
+
releaseTurn: (turnId: string) => {
|
|
3829
|
+
releasedTurnIds.add(turnId);
|
|
3830
|
+
return true;
|
|
3831
|
+
},
|
|
3832
|
+
retireSource: (source: ModeSessionSourceHandle) => {
|
|
3833
|
+
if (source !== activeSource) {
|
|
3834
|
+
return false;
|
|
3835
|
+
}
|
|
3836
|
+
activeSource = undefined;
|
|
3837
|
+
return true;
|
|
3838
|
+
},
|
|
3839
|
+
trackPersistedRow: () => {},
|
|
3840
|
+
} as unknown as ConversationModeSessionCoordinator;
|
|
3008
3841
|
// The first turn stays thinking so the barge-in lands on it. Every
|
|
3009
3842
|
// speculative leg afterwards answers with the hold token while it is
|
|
3010
3843
|
// offered one (`unifiedVerdict`), and answers for real on the replay.
|
|
3011
3844
|
const startVoiceTurn: LiveVoiceTurnStarter = async (options) => {
|
|
3012
3845
|
calls.push(options);
|
|
3013
3846
|
const index = calls.length;
|
|
3847
|
+
const discard = mock(async () => {
|
|
3848
|
+
const requestId = options.preacceptedModeSession?.requestId;
|
|
3849
|
+
if (requestId) {
|
|
3850
|
+
modeSessions.releaseTurn(requestId);
|
|
3851
|
+
}
|
|
3852
|
+
});
|
|
3014
3853
|
if (options.content !== "first question") {
|
|
3015
3854
|
const reply = options.unifiedVerdict === true ? "[0]" : "Sure thing.";
|
|
3016
3855
|
setTimeout(() => {
|
|
@@ -3031,9 +3870,19 @@ describe("LiveVoiceSession server VAD", () => {
|
|
|
3031
3870
|
// The announcement must not fire during the hold window — the replay
|
|
3032
3871
|
// turn is the delivery this test is about.
|
|
3033
3872
|
continuationAnnounceSilenceMs: 5_000,
|
|
3873
|
+
acquireModeSessionResidency: async () => ({
|
|
3874
|
+
coordinator: modeSessions,
|
|
3875
|
+
release: () => {},
|
|
3876
|
+
}),
|
|
3034
3877
|
});
|
|
3035
3878
|
|
|
3036
3879
|
await session.start();
|
|
3880
|
+
await session.handleClientFrame({
|
|
3881
|
+
type: "sight_start",
|
|
3882
|
+
cameraEpoch: 1,
|
|
3883
|
+
source: "live",
|
|
3884
|
+
});
|
|
3885
|
+
expect(claimedTurnIds).toHaveLength(1);
|
|
3037
3886
|
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
3038
3887
|
await waitFor(() => frames.some((frame) => frame.type === "thinking"));
|
|
3039
3888
|
await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
|
|
@@ -3052,7 +3901,26 @@ describe("LiveVoiceSession server VAD", () => {
|
|
|
3052
3901
|
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
3053
3902
|
|
|
3054
3903
|
// The speculative leg holds: it is rolled back, taking nothing with it.
|
|
3055
|
-
await waitFor(() =>
|
|
3904
|
+
await waitFor(() =>
|
|
3905
|
+
calls.some(
|
|
3906
|
+
(call) =>
|
|
3907
|
+
call.content === "third question" && call.unifiedVerdict === true,
|
|
3908
|
+
),
|
|
3909
|
+
);
|
|
3910
|
+
const speculative = calls.find(
|
|
3911
|
+
(call) =>
|
|
3912
|
+
call.content === "third question" && call.unifiedVerdict === true,
|
|
3913
|
+
);
|
|
3914
|
+
expect(speculative?.preacceptedModeSession).toBeDefined();
|
|
3915
|
+
await waitFor(() =>
|
|
3916
|
+
releasedTurnIds.has(speculative?.preacceptedModeSession?.requestId ?? ""),
|
|
3917
|
+
);
|
|
3918
|
+
const primaryIndex = claimedTurnIds.indexOf(
|
|
3919
|
+
speculative!.preacceptedModeSession!.requestId,
|
|
3920
|
+
);
|
|
3921
|
+
expect(primaryIndex).toBeGreaterThanOrEqual(0);
|
|
3922
|
+
expect(releasedTurnIds.has(claimedTurnIds[primaryIndex + 1]!)).toBe(true);
|
|
3923
|
+
expect(releasedTurnIds.has(claimedTurnIds[primaryIndex + 2]!)).toBe(true);
|
|
3056
3924
|
|
|
3057
3925
|
// The extension replays the boundary and the replay leg answers. It is a
|
|
3058
3926
|
// different dispatch, so it only carries the answer if the rollback gave
|
|
@@ -3063,6 +3931,7 @@ describe("LiveVoiceSession server VAD", () => {
|
|
|
3063
3931
|
const replay = calls.filter((c) => c.content === "third question")[1];
|
|
3064
3932
|
expect(replay?.unifiedVerdict).toBeUndefined();
|
|
3065
3933
|
expect(replay?.voiceControlPrompt).toContain("THE_RESULT");
|
|
3934
|
+
await session.close("client_end");
|
|
3066
3935
|
});
|
|
3067
3936
|
|
|
3068
3937
|
test("an announcement waits out the client's queued playback", async () => {
|
|
@@ -3126,24 +3995,19 @@ describe("LiveVoiceSession server VAD", () => {
|
|
|
3126
3995
|
// every transcript-derived idle signal still reads "nobody is speaking".
|
|
3127
3996
|
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
3128
3997
|
|
|
3129
|
-
// A manual session reaches the announcement path only through machinery a
|
|
3130
|
-
// server_vad barge-in owns, so the state a finished continuation leaves
|
|
3131
|
-
// behind is staged directly: the stashed answer plus its queued
|
|
3132
|
-
// announcement.
|
|
3133
3998
|
const internals = session as unknown as {
|
|
3134
|
-
|
|
3135
|
-
|
|
3136
|
-
|
|
3999
|
+
receiveTaskOutcome: (
|
|
4000
|
+
outcome: import("../task-outcomes.js").VoiceTaskOutcome,
|
|
4001
|
+
) => boolean;
|
|
3137
4002
|
};
|
|
3138
|
-
internals.
|
|
3139
|
-
|
|
3140
|
-
|
|
3141
|
-
|
|
3142
|
-
|
|
3143
|
-
|
|
4003
|
+
internals.receiveTaskOutcome({
|
|
4004
|
+
taskId: "continuation-1",
|
|
4005
|
+
message: "THE_RESULT",
|
|
4006
|
+
metadata: {},
|
|
4007
|
+
source: "continuation",
|
|
4008
|
+
});
|
|
3144
4009
|
|
|
3145
|
-
// The
|
|
3146
|
-
// flight, so the session never speaks over the in-progress utterance.
|
|
4010
|
+
// The floor stays blocked while captured speech is awaiting release.
|
|
3147
4011
|
await new Promise((resolve) => setTimeout(resolve, 200));
|
|
3148
4012
|
expect(announcementOf(calls)).toBeUndefined();
|
|
3149
4013
|
|