@vellumai/assistant 0.10.12 → 0.11.0-staging.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +1 -1
- package/ARCHITECTURE.md +4 -5
- package/README.md +1 -1
- package/docs/architecture/memory.md +60 -23
- package/docs/error-handling.md +3 -3
- package/docs/guardian-request-flow.md +112 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/package.json +1 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/__tests__/channels.test.ts +9 -1
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +449 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/channels.ts +1 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/secret-detection.ts +468 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@slack/types/dist/events/app.d.ts +28 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@slack/types/dist/events/app.d.ts.map +1 -1
- package/node_modules/@vellumai/gateway-client/node_modules/@slack/types/dist/events/index.d.ts +2 -2
- package/node_modules/@vellumai/gateway-client/node_modules/@slack/types/dist/events/index.d.ts.map +1 -1
- package/node_modules/@vellumai/gateway-client/node_modules/@slack/types/dist/events/index.js.map +1 -1
- package/node_modules/@vellumai/gateway-client/node_modules/@slack/types/dist/events/message.d.ts +2 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@slack/types/dist/events/message.d.ts.map +1 -1
- package/node_modules/@vellumai/gateway-client/node_modules/@slack/types/package.json +1 -1
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/package.json +1 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/__tests__/channels.test.ts +9 -1
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +449 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/channels.ts +1 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/secret-detection.ts +468 -0
- package/node_modules/@vellumai/gateway-client/package.json +1 -1
- package/node_modules/@vellumai/gateway-client/src/admission-policy-contract.ts +6 -0
- package/node_modules/@vellumai/gateway-client/src/inbound-contract.ts +34 -0
- package/node_modules/@vellumai/service-contracts/package.json +1 -0
- package/node_modules/@vellumai/service-contracts/src/__tests__/channels.test.ts +9 -1
- package/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +449 -0
- package/node_modules/@vellumai/service-contracts/src/channels.ts +1 -0
- package/node_modules/@vellumai/service-contracts/src/secret-detection.ts +468 -0
- package/openapi.yaml +264 -14
- package/package.json +2 -2
- package/src/__tests__/acp-session.test.ts +3 -3
- package/src/__tests__/agent-loop-mutable-latest-user-message.test.ts +3 -0
- package/src/__tests__/anthropic-provider.test.ts +40 -0
- package/src/__tests__/app-conversation-ids-backfill.test.ts +33 -2
- package/src/__tests__/app-conversation-ids.test.ts +204 -0
- package/src/__tests__/approval-cascade.test.ts +5 -5
- package/src/__tests__/approval-routes-http.test.ts +3 -3
- package/src/__tests__/ask-question-resolver.test.ts +207 -0
- package/src/__tests__/assistant-event-hub-self-exclusion.test.ts +16 -16
- package/src/__tests__/assistant-event-hub-targeted.test.ts +13 -11
- package/src/__tests__/assistant-event-hub.test.ts +13 -11
- package/src/__tests__/assistant-event.test.ts +12 -9
- package/src/__tests__/assistant-feature-flag-guard.test.ts +3 -3
- package/src/__tests__/assistant-feature-flag-guardrails.test.ts +1 -1
- package/src/__tests__/assistant-id-boundary-guard.test.ts +0 -58
- package/src/__tests__/assistant-stream-state.test.ts +17 -7
- package/src/__tests__/avatar-identity-sync.test.ts +3 -3
- package/src/__tests__/background-workers-disk-pressure.test.ts +1 -0
- package/src/__tests__/call-conversation-messages.test.ts +134 -18
- package/src/__tests__/call-start-guardian-guard.test.ts +27 -0
- package/src/__tests__/channel-approvals.test.ts +0 -27
- package/src/__tests__/channel-guardian.test.ts +85 -512
- package/src/__tests__/channel-readiness-slack-remote.test.ts +151 -3
- package/src/__tests__/channel-retry-sweep.test.ts +88 -1
- package/src/__tests__/channel-setup-panel-ack.test.ts +12 -9
- package/src/__tests__/chat-reveal-guard-priming.test.ts +31 -34
- package/src/__tests__/cli-memory-v2-reembed-skills.test.ts +2 -2
- package/src/__tests__/compaction-events.test.ts +8 -8
- package/src/__tests__/config-loader-platform-defaults.test.ts +54 -28
- package/src/__tests__/config-model-image-provider.test.ts +20 -0
- package/src/__tests__/config-schema-memory-substrate.test.ts +306 -0
- package/src/__tests__/config-schema.test.ts +14 -2
- package/src/__tests__/config-set-platform-guard.test.ts +11 -12
- package/src/__tests__/config-sounds-sync.test.ts +4 -4
- package/src/__tests__/confirmation-request-guardian-bridge.test.ts +5 -0
- package/src/__tests__/context-search-memory-source.test.ts +1 -1
- package/src/__tests__/context-search-memory-v2-source.test.ts +6 -6
- package/src/__tests__/context-token-estimator.test.ts +54 -1
- package/src/__tests__/conversation-agent-loop-disk-pressure.test.ts +12 -3
- package/src/__tests__/conversation-agent-loop-fatal-cleanup.test.ts +13 -4
- package/src/__tests__/conversation-agent-loop-handlers-max-tokens.test.ts +4 -4
- package/src/__tests__/conversation-agent-loop-inference-profile.test.ts +10 -1
- package/src/__tests__/conversation-agent-loop-overflow.test.ts +22 -13
- package/src/__tests__/conversation-agent-loop.test.ts +39 -30
- package/src/__tests__/conversation-confirmation-signals.test.ts +14 -14
- package/src/__tests__/conversation-evictor.test.ts +49 -0
- package/src/__tests__/conversation-fork-crud.test.ts +1 -1
- package/src/__tests__/conversation-group-tools.test.ts +279 -0
- package/src/__tests__/conversation-history-web-search.test.ts +1 -1
- package/src/__tests__/conversation-key-store-disk-view.test.ts +5 -5
- package/src/__tests__/conversation-lineage.test.ts +235 -0
- package/src/__tests__/conversation-loop-db-lock-resilience.test.ts +5 -5
- package/src/__tests__/conversation-message-sync-tags.test.ts +4 -4
- package/src/__tests__/conversation-queue.test.ts +81 -81
- package/src/__tests__/conversation-routes-enabled-plugins.test.ts +8 -1
- package/src/__tests__/conversation-routes-hidden-queue.test.ts +8 -1
- package/src/__tests__/conversation-routes-slash-commands.test.ts +10 -1
- package/src/__tests__/conversation-runtime-assembly.test.ts +29 -2
- package/src/__tests__/conversation-slash-queue.test.ts +11 -11
- package/src/__tests__/conversation-slash-unknown.test.ts +2 -2
- package/src/__tests__/conversation-speed-override.test.ts +2 -2
- package/src/__tests__/conversation-summarize-route.test.ts +9 -0
- package/src/__tests__/conversation-surfaces-action-delivery.test.ts +18 -16
- package/src/__tests__/conversation-surfaces-activation-emit.test.ts +17 -17
- package/src/__tests__/conversation-surfaces-app-open.test.ts +7 -4
- package/src/__tests__/conversation-surfaces-data-persist.test.ts +7 -7
- package/src/__tests__/conversation-surfaces-standalone-payloads.test.ts +9 -9
- package/src/__tests__/conversation-surfaces-standalone.test.ts +12 -12
- package/src/__tests__/conversation-surfaces-state-update.test.ts +8 -5
- package/src/__tests__/conversation-surfaces-table-action.test.ts +6 -3
- package/src/__tests__/conversation-surfaces-task-progress.test.ts +46 -46
- package/src/__tests__/conversation-sync-tags.test.ts +5 -5
- package/src/__tests__/conversation-wait-for-idle.test.ts +2 -2
- package/src/__tests__/conversation-workspace-cache-state.test.ts +0 -4
- package/src/__tests__/conversation-workspace-injection.test.ts +0 -3
- package/src/__tests__/conversation-workspace-tool-tracking.test.ts +0 -3
- package/src/__tests__/copy-composer-tc-templates.test.ts +3 -131
- package/src/__tests__/credential-security-invariants.test.ts +1 -0
- package/src/__tests__/daemon-assistant-events.test.ts +14 -15
- package/src/__tests__/db-test-helpers.ts +13 -27
- package/src/__tests__/disk-pressure-routes.test.ts +12 -3
- package/src/__tests__/document-tool-security.test.ts +95 -0
- package/src/__tests__/drain-kick-guard.test.ts +232 -0
- package/src/__tests__/dynamic-skill-background-guard.test.ts +19 -6
- package/src/__tests__/emit-event-signal.test.ts +5 -6
- package/src/__tests__/encrypted-store-test-helpers.ts +10 -20
- package/src/__tests__/events-tail-route.test.ts +6 -4
- package/src/__tests__/feature-flag-test-helpers.ts +10 -20
- package/src/__tests__/file-edit-tool.test.ts +24 -4
- package/src/__tests__/file-list-tool.test.ts +2 -1
- package/src/__tests__/file-read-tool.test.ts +62 -10
- package/src/__tests__/file-write-tool.test.ts +24 -5
- package/src/__tests__/followup-tools.test.ts +58 -0
- package/src/__tests__/gateway-threshold-reader-mock.ts +99 -0
- package/src/__tests__/graph-extraction-event-date.test.ts +1 -1
- package/src/__tests__/guardian-question-mode.test.ts +42 -3
- package/src/__tests__/guardian-reply-router-answer-mode.test.ts +283 -0
- package/src/__tests__/guardian-vellum-destination-pinning.test.ts +221 -0
- package/src/__tests__/heartbeat-disk-pressure.test.ts +1 -0
- package/src/__tests__/heartbeat-service.test.ts +1 -0
- package/src/__tests__/helpers/native-web-search-harness.ts +10 -10
- package/src/__tests__/host-browser-proxy.test.ts +1 -1
- package/src/__tests__/host-file-edit-tool.test.ts +3 -3
- package/src/__tests__/host-file-read-tool.test.ts +2 -2
- package/src/__tests__/host-file-write-tool.test.ts +7 -3
- package/src/__tests__/host-shell-tool.test.ts +17 -5
- package/src/__tests__/host-transfer-pending-interactions.test.ts +8 -8
- package/src/__tests__/image-credentials.test.ts +65 -8
- package/src/__tests__/injection-block.test.ts +4 -4
- package/src/__tests__/injector-chain.test.ts +1 -1
- package/src/__tests__/injector-pkb-v2-silenced.test.ts +2 -2
- package/src/__tests__/injector-registry-order-guard.test.ts +4 -9
- package/src/__tests__/injector-v3-suppression.test.ts +6 -4
- package/src/__tests__/lifecycle-memory-v2-seed.test.ts +27 -28
- package/src/__tests__/list-messages-page-latest.test.ts +2 -2
- package/src/__tests__/llm-usage-store.test.ts +67 -4
- package/src/__tests__/log-redact.test.ts +18 -0
- package/src/__tests__/managed-profile-guard.test.ts +27 -0
- package/src/__tests__/managed-skill-lifecycle.test.ts +2 -2
- package/src/__tests__/media-generate-image.test.ts +62 -11
- package/src/__tests__/memory-identity-context-parity.test.ts +1 -1
- package/src/__tests__/memory-jobs-worker-lanes.test.ts +9 -1
- package/src/__tests__/memory-retrieval-hook.test.ts +4 -4
- package/src/__tests__/notification-decision-strategy.test.ts +115 -11
- package/src/__tests__/notification-deep-link.test.ts +12 -12
- package/src/__tests__/notification-vellum-adapter.test.ts +8 -8
- package/src/__tests__/outbound-slack-persistence.test.ts +8 -8
- package/src/__tests__/path-policy.test.ts +315 -0
- package/src/__tests__/pending-interactions-resolved-event.test.ts +6 -6
- package/src/__tests__/persistence-layering-guard.test.ts +2 -2
- package/src/__tests__/pkb-autoinject.test.ts +1 -1
- package/src/__tests__/platform-bash-auto-approve.test.ts +1 -0
- package/src/__tests__/platform-callback-registration.test.ts +10 -10
- package/src/__tests__/plugin-event-hub-facade.test.ts +22 -15
- package/src/__tests__/plugin-import-boundary-guard.test.ts +15 -15
- package/src/__tests__/processing-flag-persist-failure.test.ts +0 -4
- package/src/__tests__/provider-platform-proxy-integration.test.ts +0 -1
- package/src/__tests__/rebuild-index-graph-nodes.test.ts +1 -1
- package/src/__tests__/request-secret-standalone-channel-gate.test.ts +5 -5
- package/src/__tests__/require-fresh-approval.test.ts +38 -38
- package/src/__tests__/resolve-app-id.test.ts +32 -0
- package/src/__tests__/run-due-schedules.test.ts +72 -1
- package/src/__tests__/runtime-events-sse-parity.test.ts +11 -10
- package/src/__tests__/runtime-events-sse.test.ts +1 -1
- package/src/__tests__/schedule-routes.test.ts +44 -5
- package/src/__tests__/schedule-store.test.ts +3 -3
- package/src/__tests__/schedule-tools.test.ts +176 -0
- package/src/__tests__/secret-prompt-log-hygiene.test.ts +3 -3
- package/src/__tests__/secret-prompter-channel-fallback.test.ts +3 -3
- package/src/__tests__/secret-response-routing.test.ts +3 -3
- package/src/__tests__/secret-routes-platform-proxy.test.ts +4 -7
- package/src/__tests__/secret-scanner.test.ts +36 -0
- package/src/__tests__/send-endpoint-busy.test.ts +7 -7
- package/src/__tests__/slack-required-scopes-mirror.test.ts +81 -0
- package/src/__tests__/starter-task-flow.test.ts +26 -3
- package/src/__tests__/subagent-call-site-routing.test.ts +2 -2
- package/src/__tests__/subagent-disposal.test.ts +40 -11
- package/src/__tests__/subagent-fork-notifications.test.ts +4 -4
- package/src/__tests__/subagent-fork-prompt-role.test.ts +3 -3
- package/src/__tests__/subagent-fork-spawn.test.ts +4 -4
- package/src/__tests__/subagent-manager-notify.test.ts +73 -14
- package/src/__tests__/subagent-notify-parent.test.ts +48 -4
- package/src/__tests__/subagent-spawn-and-await.test.ts +62 -14
- package/src/__tests__/subagent-tools.test.ts +64 -0
- package/src/__tests__/sync-message-contract.test.ts +3 -3
- package/src/__tests__/terminal-tools.test.ts +12 -3
- package/src/__tests__/test-preload.ts +9 -0
- package/src/__tests__/tool-approval-handler.test.ts +587 -43
- package/src/__tests__/tool-execution-pipeline.benchmark.test.ts +7 -4
- package/src/__tests__/tool-executor.test.ts +116 -0
- package/src/__tests__/tool-grant-request-escalation.test.ts +5 -0
- package/src/__tests__/tool-preview-lifecycle.test.ts +32 -29
- package/src/__tests__/tool-result-metadata-plumbing.test.ts +5 -5
- package/src/__tests__/tool-side-effects-slack-dm.test.ts +1 -0
- package/src/__tests__/tool-start-timestamp.test.ts +4 -4
- package/src/__tests__/trusted-contact-lifecycle-notifications.test.ts +9 -22
- package/src/__tests__/ui-choice-copy-surfaces.test.ts +13 -13
- package/src/__tests__/ui-work-result-surface.test.ts +3 -3
- package/src/__tests__/voice-scoped-grant-consumer.test.ts +6 -6
- package/src/__tests__/voice-session-bridge.test.ts +44 -44
- package/src/__tests__/workspace-migration-134-image-generation-mode-to-provider.test.ts +284 -0
- package/src/__tests__/workspace-policy.test.ts +482 -2
- package/src/acp/__tests__/client-handler.test.ts +3 -3
- package/src/acp/__tests__/session-manager-persistence.test.ts +10 -10
- package/src/acp/__tests__/session-manager-resume.test.ts +7 -7
- package/src/acp/client-handler.ts +3 -3
- package/src/acp/session-manager.ts +8 -8
- package/src/api/events/subagent-event.ts +2 -2
- package/src/api/responses/memory-v2-activation-log.ts +2 -2
- package/src/api/responses/subagent-detail.ts +13 -0
- package/src/approvals/AGENTS.md +2 -0
- package/src/approvals/guardian-request-resolvers.ts +118 -24
- package/src/apps/app-store.ts +95 -4
- package/src/calls/__tests__/voice-control-protocol.test.ts +57 -0
- package/src/calls/__tests__/voice-session-bridge.test.ts +229 -7
- package/src/calls/__tests__/voice-triage-escalate.test.ts +3 -4
- package/src/calls/call-controller.ts +27 -27
- package/src/calls/call-conversation-messages.ts +58 -18
- package/src/calls/voice-control-protocol.ts +56 -0
- package/src/calls/voice-session-bridge.ts +178 -65
- package/src/calls/voice-triage-escalate.ts +2 -3
- package/src/channels/config.ts +8 -0
- package/src/channels/types.ts +1 -0
- package/src/cli/commands/config.ts +49 -13
- package/src/cli/commands/memory/index.help.ts +4 -2
- package/src/cli/commands/memory/memory-retrospective.ts +5 -0
- package/src/cli/commands/webhooks.help.ts +9 -4
- package/src/cli/lib/bundled-marketplace.json +12 -0
- package/src/cli.ts +2 -109
- package/src/config/__tests__/deployment-context-defaults.test.ts +9 -0
- package/src/config/__tests__/substrate-twin-shadowing.test.ts +191 -0
- package/src/config/assistant-feature-flags.ts +1 -1
- package/src/config/bundled-skills/conversation-groups/SKILL.md +29 -0
- package/src/config/bundled-skills/conversation-groups/TOOLS.json +58 -0
- package/src/config/bundled-skills/conversation-groups/tools/group-create.ts +12 -0
- package/src/config/bundled-skills/conversation-groups/tools/group-list.ts +12 -0
- package/src/config/bundled-skills/conversation-groups/tools/move-to-group.ts +12 -0
- package/src/config/bundled-skills/image-studio/tools/media-generate-image.ts +13 -7
- package/src/config/bundled-skills/schedule/SKILL.md +2 -0
- package/src/config/feature-flag-cache.ts +6 -16
- package/src/config/feature-flag-registry.json +14 -38
- package/src/config/loader.ts +7 -1
- package/src/config/memory-tier.ts +15 -10
- package/src/config/memory-v3-gate.test.ts +278 -1
- package/src/config/memory-v3-gate.ts +88 -26
- package/src/config/raw-config-utils.ts +1 -1
- package/src/config/schemas/__tests__/live-voice.test.ts +27 -16
- package/src/config/schemas/__tests__/memory-v3.test.ts +1 -0
- package/src/config/schemas/live-voice.ts +33 -7
- package/src/config/schemas/llm.ts +1 -0
- package/src/config/schemas/memory-retrospective.ts +13 -0
- package/src/config/schemas/memory-substrate.ts +183 -0
- package/src/config/schemas/memory-v2.ts +3 -2
- package/src/config/schemas/memory-v3.ts +11 -4
- package/src/config/schemas/memory.ts +35 -1
- package/src/config/schemas/services.ts +10 -2
- package/src/config/substrate-twin-shadowing.ts +195 -0
- package/src/config/webhook-routing.ts +67 -0
- package/src/context/compactor.ts +10 -3
- package/src/context/token-estimator.ts +21 -0
- package/src/daemon/__tests__/channel-ui-capability.test.ts +85 -0
- package/src/daemon/__tests__/conversation-surfaces-launch.test.ts +3 -3
- package/src/daemon/__tests__/embedding-reconcile.test.ts +3 -3
- package/src/daemon/__tests__/inference-profile-notification.test.ts +3 -6
- package/src/daemon/__tests__/native-web-search-metadata.test.ts +5 -5
- package/src/daemon/__tests__/web-search-status-text.test.ts +2 -2
- package/src/daemon/channel-ui-capability.ts +70 -1
- package/src/daemon/conversation-agent-loop-handlers.ts +4 -4
- package/src/daemon/conversation-agent-loop.ts +12 -5
- package/src/daemon/conversation-evictor.ts +26 -4
- package/src/daemon/conversation-launch.ts +11 -14
- package/src/daemon/conversation-lineage.ts +76 -0
- package/src/daemon/conversation-messaging.ts +14 -14
- package/src/daemon/conversation-notifiers.ts +2 -2
- package/src/daemon/conversation-process.ts +133 -21
- package/src/daemon/conversation-queue-manager.ts +7 -7
- package/src/daemon/conversation-runtime-assembly.ts +16 -0
- package/src/daemon/conversation-store.ts +11 -3
- package/src/daemon/conversation-surfaces.ts +8 -14
- package/src/daemon/conversation-tool-setup.ts +13 -7
- package/src/daemon/conversation-usage.ts +12 -11
- package/src/daemon/conversation.ts +38 -15
- package/src/daemon/disk-pressure-guard.ts +5 -12
- package/src/daemon/embedding-reconcile.ts +14 -9
- package/src/daemon/handlers/config-channels.ts +40 -134
- package/src/daemon/handlers/config-model.ts +15 -7
- package/src/daemon/handlers/conversations.ts +4 -2
- package/src/daemon/handlers/shared.ts +29 -0
- package/src/daemon/host-proxy-base.ts +4 -4
- package/src/daemon/lifecycle.ts +13 -0
- package/src/daemon/message-protocol.ts +8 -144
- package/src/daemon/message-types/computer-use.ts +0 -12
- package/src/daemon/message-types/conversations.ts +9 -345
- package/src/daemon/message-types/host-app-control.ts +0 -9
- package/src/daemon/message-types/host-browser.ts +0 -7
- package/src/daemon/message-types/host-ui-snapshot.ts +0 -7
- package/src/daemon/message-types/messages.ts +0 -48
- package/src/daemon/message-types/notifications.ts +0 -7
- package/src/daemon/message-types/skills.ts +0 -5
- package/src/daemon/message-types/surfaces.ts +0 -9
- package/src/daemon/message-types/sync.ts +0 -2
- package/src/daemon/process-message.ts +5 -5
- package/src/daemon/skill-memory-refresh.ts +5 -5
- package/src/daemon/tool-side-effects.ts +32 -16
- package/src/daemon/trust-context.ts +1 -1
- package/src/daemon/wake-conversation-ops.ts +4 -4
- package/src/heartbeat/__tests__/heartbeat-run-store.test.ts +100 -7
- package/src/heartbeat/__tests__/heartbeat-service.test.ts +67 -0
- package/src/heartbeat/heartbeat-run-store.ts +81 -4
- package/src/heartbeat/heartbeat-service.ts +32 -1
- package/src/home/__tests__/feed-writer.test.ts +7 -7
- package/src/home/__tests__/home-content-refresh.test.ts +3 -0
- package/src/home/__tests__/suggested-prompts.test.ts +1 -0
- package/src/home/feed-writer.ts +6 -13
- package/src/home/home-content-refresh.ts +6 -9
- package/src/home/relationship-state-writer.ts +5 -12
- package/src/home/suggested-prompts-cache.ts +6 -19
- package/src/ipc/__tests__/events-publish-client.test.ts +67 -0
- package/src/ipc/events-publish-client.ts +44 -0
- package/src/ipc/routes/__tests__/conversation-sync-ipc-routes.test.ts +2 -2
- package/src/ipc/routes/__tests__/events-ipc-routes.test.ts +38 -9
- package/src/ipc/routes/events-ipc-routes.ts +11 -31
- package/src/live-voice/__tests__/front-decision.test.ts +4 -176
- package/src/live-voice/__tests__/live-voice-agent-turn.test.ts +276 -68
- package/src/live-voice/__tests__/live-voice-continuation-fork-hydration.test.ts +286 -0
- package/src/live-voice/__tests__/live-voice-events.test.ts +6 -3
- package/src/live-voice/__tests__/live-voice-progress.test.ts +185 -46
- package/src/live-voice/__tests__/live-voice-triage-escalate.test.ts +24 -59
- package/src/live-voice/__tests__/live-voice-tts-session.test.ts +38 -54
- package/src/live-voice/__tests__/live-voice-vad.test.ts +1154 -636
- package/src/live-voice/front-decision.ts +5 -155
- package/src/live-voice/live-voice-metrics.ts +4 -1
- package/src/live-voice/live-voice-session.ts +1288 -439
- package/src/live-voice/progress-phrases.ts +17 -0
- package/src/live-voice/protocol.ts +15 -0
- package/src/media/app-icon-generator.ts +15 -8
- package/src/media/avatar-router.ts +9 -5
- package/src/media/image-credentials.ts +42 -10
- package/src/media/image-service.ts +1 -13
- package/src/media/types.ts +26 -3
- package/src/notifications/AGENTS.md +2 -0
- package/src/notifications/__tests__/broadcaster.test.ts +95 -0
- package/src/notifications/adapters/macos.ts +3 -3
- package/src/notifications/broadcaster.ts +57 -0
- package/src/notifications/copy-composer.ts +0 -15
- package/src/notifications/decision-engine.ts +21 -22
- package/src/notifications/guardian-question-mode.ts +57 -2
- package/src/notifications/home-feed-side-effect.ts +4 -0
- package/src/notifications/signal.ts +6 -4
- package/src/notifications/trusted-contact-payloads.ts +3 -4
- package/src/notifications/vellum-card-affinity.ts +39 -0
- package/src/oauth/AGENTS.md +1 -1
- package/src/permissions/channel-permission-query.test.ts +69 -0
- package/src/permissions/channel-permission-query.ts +73 -1
- package/src/permissions/checker.test.ts +298 -15
- package/src/permissions/checker.ts +102 -36
- package/src/permissions/confirmation-guardian-request.test.ts +2 -2
- package/src/permissions/confirmation-guardian-request.ts +2 -2
- package/src/permissions/gateway-threshold-reader.test.ts +224 -17
- package/src/permissions/gateway-threshold-reader.ts +82 -10
- package/src/permissions/ipc-risk-types.ts +8 -0
- package/src/permissions/prompter.ts +5 -5
- package/src/permissions/question-guardian-request.test.ts +222 -0
- package/src/permissions/question-guardian-request.ts +179 -0
- package/src/permissions/question-prompter.test.ts +8 -8
- package/src/permissions/question-prompter.ts +18 -0
- package/src/permissions/secret-prompter.ts +2 -4
- package/src/permissions/workspace-policy.ts +269 -34
- package/src/persistence/__tests__/lifecycle-quiesce.test.ts +82 -0
- package/src/persistence/__tests__/message-content-file.test.ts +58 -3
- package/src/persistence/conversation-crud.ts +93 -0
- package/src/persistence/conversation-key-store.ts +63 -0
- package/src/persistence/conversation-parent.ts +27 -0
- package/src/persistence/db-async-query.ts +1 -1
- package/src/persistence/db-singleton.ts +14 -27
- package/src/persistence/delivery-crud.ts +45 -0
- package/src/persistence/embeddings/embedding-backend.ts +2 -1
- package/src/persistence/embeddings/embedding-identity.ts +7 -3
- package/src/persistence/job-utils.ts +32 -27
- package/src/persistence/jobs-store.ts +92 -3
- package/src/persistence/lifecycle-quiesce.ts +80 -0
- package/src/persistence/llm-usage-store.ts +45 -8
- package/src/persistence/message-content-file.ts +4 -4
- package/src/persistence/migrations/336-move-memory-v2-activation-logs-to-memory-db.ts +1 -1
- package/src/persistence/migrations/351-schedule-skill-script-handoff.ts +39 -0
- package/src/persistence/migrations/352-drop-schedule-skill-script-handoff.test.ts +83 -0
- package/src/persistence/migrations/352-drop-schedule-skill-script-handoff.ts +36 -0
- package/src/persistence/migrations/353-add-llm-usage-conversation-type.test.ts +110 -0
- package/src/persistence/migrations/353-add-llm-usage-conversation-type.ts +49 -0
- package/src/persistence/migrations/354-backfill-app-conversation-lineage.ts +41 -0
- package/src/persistence/schema/infrastructure.ts +11 -0
- package/src/persistence/steps.ts +9 -1
- package/src/plugin-api/__tests__/publish-event-forwarding.test.ts +113 -0
- package/src/plugin-api/__tests__/publish-event.test.ts +57 -0
- package/src/plugin-api/conversation-turn.ts +4 -4
- package/src/plugin-api/event-hub-facade.ts +15 -6
- package/src/plugin-api/index.ts +10 -4
- package/src/plugin-api/publish-event.ts +36 -0
- package/src/plugins/AGENTS.md +2 -0
- package/src/plugins/defaults/compaction/window-manager.ts +9 -0
- package/src/plugins/defaults/index.ts +9 -13
- package/src/plugins/defaults/memory/AGENTS.md +775 -0
- package/src/plugins/defaults/memory/__tests__/capability-seed-embed-gate.test.ts +186 -0
- package/src/plugins/defaults/memory/__tests__/fixtures/memory-v2-activation-fixtures.ts +1 -1
- package/src/plugins/defaults/memory/__tests__/indexer-tier-dispatch.test.ts +129 -0
- package/src/plugins/defaults/memory/__tests__/jobs-worker-quiesce-gate.test.ts +91 -0
- package/src/plugins/defaults/memory/__tests__/jobs-worker-v1-graph-substrate-noop.test.ts +342 -0
- package/src/plugins/defaults/memory/__tests__/jobs-worker-v2-graph-trigger-embed.test.ts +9 -1
- package/src/plugins/defaults/memory/__tests__/jobs-worker-v2-schedule.test.ts +36 -1
- package/src/plugins/defaults/memory/__tests__/jobs-worker-v2-sweep-defer.test.ts +9 -1
- package/src/plugins/defaults/memory/__tests__/memory-log-stores-degraded.test.ts +2 -2
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-job.test.ts +44 -10
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-sweep.test.ts +151 -10
- package/src/plugins/defaults/memory/__tests__/memory-tier-boundary-guard.test.ts +1155 -0
- package/src/plugins/defaults/memory/__tests__/memory-v2-activation-log-store.test.ts +1 -1
- package/src/plugins/defaults/memory/__tests__/memory-v2-concept-frequency.test.ts +3 -3
- package/src/plugins/defaults/memory/__tests__/tools-delete-memory-page.test.ts +1 -1
- package/src/plugins/defaults/memory/context-search/sources/memory-v2.ts +71 -35
- package/src/plugins/defaults/memory/context-search/sources/memory.ts +5 -1
- package/src/plugins/defaults/memory/fork-conversation-memory.ts +4 -4
- package/src/plugins/defaults/memory/graph/__tests__/conversation-graph-memory-v2-routing.test.ts +80 -10
- package/src/plugins/defaults/memory/graph/__tests__/handle-remember-v2.test.ts +1 -1
- package/src/plugins/defaults/memory/graph/capability-seed.ts +24 -10
- package/src/plugins/defaults/memory/graph/conversation-graph-memory.ts +123 -49
- package/src/plugins/defaults/memory/graph/graph-search.ts +6 -118
- package/src/plugins/defaults/memory/graph/in-context-tracker.test.ts +173 -0
- package/src/plugins/defaults/memory/graph/in-context-tracker.ts +109 -0
- package/src/plugins/defaults/memory/graph/store.ts +1 -0
- package/src/plugins/defaults/memory/graph/tool-handlers.test.ts +1 -1
- package/src/plugins/defaults/memory/graph/tool-handlers.ts +5 -2
- package/src/plugins/defaults/memory/graph-topology/__tests__/build-memory-graph.test.ts +1 -1
- package/src/plugins/defaults/memory/graph-topology/build-memory-graph.ts +6 -9
- package/src/plugins/defaults/memory/hooks/__tests__/user-prompt-submit.test.ts +30 -16
- package/src/plugins/defaults/memory/hooks/user-prompt-submit.ts +18 -14
- package/src/plugins/defaults/memory/indexer.ts +146 -89
- package/src/plugins/defaults/memory/injectors.ts +86 -41
- package/src/plugins/defaults/memory/job-handlers.ts +130 -39
- package/src/plugins/defaults/memory/jobs/__tests__/embed-concept-page.test.ts +2 -2
- package/src/plugins/defaults/memory/jobs/embed-concept-page.ts +22 -10
- package/src/plugins/defaults/memory/jobs-worker.ts +279 -208
- package/src/plugins/defaults/memory/memory-retrospective-enqueue.ts +4 -2
- package/src/plugins/defaults/memory/memory-retrospective-job.ts +40 -3
- package/src/plugins/defaults/memory/memory-retrospective-sweep.ts +99 -11
- package/src/plugins/defaults/memory/src/__tests__/memory-v2-routes.test.ts +5 -3
- package/src/plugins/defaults/memory/src/__tests__/memory-v2-simulate-route.test.ts +3 -4
- package/src/plugins/defaults/memory/src/memory-graph-routes.ts +2 -1
- package/src/plugins/defaults/memory/src/memory-item-routes.test.ts +7 -5
- package/src/plugins/defaults/memory/src/memory-item-routes.ts +51 -19
- package/src/plugins/defaults/memory/src/memory-v2-routes.ts +31 -20
- package/src/plugins/defaults/memory/startup.ts +58 -21
- package/src/plugins/defaults/memory/substrate/__tests__/boot-maintenance.test.ts +335 -0
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/cli-command-store.test.ts +14 -8
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/consolidation-job.test.ts +51 -21
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/edge-index.test.ts +1 -1
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/frontmatter-sweep.test.ts +3 -3
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/page-index.test.ts +1 -1
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/page-store.test.ts +1 -1
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/prompts-consolidation.test.ts +2 -2
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/qdrant.test.ts +23 -14
- package/src/plugins/defaults/memory/substrate/__tests__/reembed-job.test.ts +262 -0
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/sim.test.ts +9 -12
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/skill-content.test.ts +2 -2
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/skill-store.test.ts +9 -9
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/sparse-bm25.test.ts +12 -5
- package/src/plugins/defaults/memory/substrate/__tests__/spread.test.ts +276 -0
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/static-context.test.ts +17 -8
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/sweep-job.test.ts +9 -9
- package/src/plugins/defaults/memory/{v3/substrate/memory-v2-startup.ts → substrate/boot-maintenance.ts} +43 -24
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/cli-command-store.ts +13 -8
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/consolidation-job.ts +51 -27
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/constants.ts +1 -0
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/edge-index.ts +28 -9
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/frontmatter-sweep.ts +14 -7
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/injected-block-slugs.ts +9 -3
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/page-index.ts +29 -10
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/page-store.ts +35 -12
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/prompts/consolidation.ts +14 -8
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/prompts/sweep.ts +4 -1
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/qdrant.ts +96 -36
- package/src/plugins/defaults/memory/substrate/reembed-job.ts +54 -0
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/sim.ts +26 -15
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/skill-content.ts +1 -1
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/skill-store.ts +13 -8
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/sparse-bm25.ts +27 -10
- package/src/plugins/defaults/memory/substrate/spread.ts +165 -0
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/static-context.ts +12 -7
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/sweep-job.ts +20 -15
- package/src/plugins/defaults/memory/substrate/tuning.ts +93 -0
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/types.ts +7 -4
- package/src/plugins/defaults/memory/tools.test.ts +1 -1
- package/src/plugins/defaults/memory/tools.ts +1 -1
- package/src/plugins/defaults/memory/v1/README.md +5 -0
- package/src/plugins/defaults/memory/{__tests__ → v1/__tests__}/filing-jobs.test.ts +4 -4
- package/src/plugins/defaults/memory/{filing-jobs.ts → v1/filing-jobs.ts} +3 -3
- package/src/plugins/defaults/memory/{graph → v1/graph}/bootstrap.test.ts +68 -6
- package/src/plugins/defaults/memory/{graph → v1/graph}/bootstrap.ts +40 -21
- package/src/plugins/defaults/memory/{graph → v1/graph}/consolidation.ts +14 -14
- package/src/plugins/defaults/memory/{graph → v1/graph}/decay.test.ts +1 -1
- package/src/plugins/defaults/memory/{graph → v1/graph}/decay.ts +3 -3
- package/src/plugins/defaults/memory/{graph → v1/graph}/extraction-job.ts +5 -5
- package/src/plugins/defaults/memory/{graph → v1/graph}/extraction.test.ts +1 -1
- package/src/plugins/defaults/memory/{graph → v1/graph}/extraction.ts +20 -14
- package/src/plugins/defaults/memory/{graph → v1/graph}/graph-search.test.ts +3 -3
- package/src/plugins/defaults/memory/v1/graph/graph-search.ts +123 -0
- package/src/plugins/defaults/memory/{graph → v1/graph}/injection.test.ts +1 -118
- package/src/plugins/defaults/memory/{graph → v1/graph}/injection.ts +9 -111
- package/src/plugins/defaults/memory/{graph → v1/graph}/narrative.ts +6 -6
- package/src/plugins/defaults/memory/{graph → v1/graph}/pattern-scan.ts +5 -5
- package/src/plugins/defaults/memory/{graph → v1/graph}/retriever.test.ts +135 -42
- package/src/plugins/defaults/memory/{graph → v1/graph}/retriever.ts +40 -22
- package/src/plugins/defaults/memory/{graph → v1/graph}/scoring.test.ts +1 -1
- package/src/plugins/defaults/memory/{graph → v1/graph}/scoring.ts +1 -1
- package/src/plugins/defaults/memory/{graph → v1/graph}/serendipity.ts +1 -1
- package/src/plugins/defaults/memory/{graph → v1/graph}/triggers.test.ts +1 -1
- package/src/plugins/defaults/memory/{graph → v1/graph}/triggers.ts +1 -1
- package/src/plugins/defaults/memory/{identity-context.ts → v1/identity-context.ts} +4 -4
- package/src/plugins/defaults/memory/{job-handlers → v1/job-handlers}/backfill.ts +6 -6
- package/src/plugins/defaults/memory/{job-handlers → v1/job-handlers}/embedding.test.ts +6 -6
- package/src/plugins/defaults/memory/{job-handlers → v1/job-handlers}/embedding.ts +6 -6
- package/src/plugins/defaults/memory/{job-handlers → v1/job-handlers}/index-maintenance.ts +8 -8
- package/src/plugins/defaults/memory/{jobs → v1/jobs}/embed-pkb-file.test.ts +7 -7
- package/src/plugins/defaults/memory/{jobs → v1/jobs}/embed-pkb-file.ts +5 -5
- package/src/plugins/defaults/memory/{pkb → v1/pkb}/autoinject.ts +1 -1
- package/src/plugins/defaults/memory/{pkb → v1/pkb}/context.ts +2 -2
- package/src/plugins/defaults/memory/{pkb → v1/pkb}/pkb-index.test.ts +3 -3
- package/src/plugins/defaults/memory/{pkb → v1/pkb}/pkb-index.ts +2 -2
- package/src/plugins/defaults/memory/{pkb → v1/pkb}/pkb-reconcile.test.ts +3 -3
- package/src/plugins/defaults/memory/{pkb → v1/pkb}/pkb-reconcile.ts +2 -2
- package/src/plugins/defaults/memory/{pkb → v1/pkb}/pkb-search.test.ts +3 -3
- package/src/plugins/defaults/memory/{pkb → v1/pkb}/pkb-search.ts +6 -6
- package/src/plugins/defaults/memory/{pkb → v1/pkb}/types.ts +3 -5
- package/src/plugins/defaults/memory/{pkb-schedule.ts → v1/pkb-schedule.ts} +2 -2
- package/src/plugins/defaults/memory/{search/semantic.ts → v1/semantic-search.ts} +1 -1
- package/src/plugins/defaults/memory/v2/__tests__/activation-store.test.ts +1 -1
- package/src/plugins/defaults/memory/v2/__tests__/activation.test.ts +11 -274
- package/src/plugins/defaults/memory/v2/__tests__/backfill-jobs.test.ts +9 -131
- package/src/plugins/defaults/memory/v2/__tests__/harness-compare.test.ts +1 -1
- package/src/plugins/defaults/memory/v2/__tests__/harness-oracle.test.ts +1 -1
- package/src/plugins/defaults/memory/v2/__tests__/harness-replay-input.test.ts +1 -1
- package/src/plugins/defaults/memory/v2/__tests__/injection.test.ts +14 -10
- package/src/plugins/defaults/memory/v2/__tests__/reranker.test.ts +7 -5
- package/src/plugins/defaults/memory/v2/__tests__/router.test.ts +12 -6
- package/src/plugins/defaults/memory/{memory-v2-activation-log-store.ts → v2/activation-log-store.ts} +2 -2
- package/src/plugins/defaults/memory/v2/activation-store.ts +22 -8
- package/src/plugins/defaults/memory/v2/activation.ts +30 -163
- package/src/plugins/defaults/memory/v2/backfill-jobs.ts +7 -54
- package/src/plugins/defaults/memory/{memory-v2-concept-frequency.ts → v2/concept-frequency.ts} +3 -3
- package/src/plugins/defaults/memory/v2/harness/oracle.ts +1 -1
- package/src/plugins/defaults/memory/v2/harness/replay-input.ts +2 -2
- package/src/plugins/defaults/memory/v2/harness/retriever.ts +1 -1
- package/src/plugins/defaults/memory/v2/injection.ts +36 -26
- package/src/plugins/defaults/memory/v2/migration.ts +2 -2
- package/src/plugins/defaults/memory/{rerank-local.ts → v2/rerank-local.ts} +44 -18
- package/src/plugins/defaults/memory/v2/reranker.ts +29 -11
- package/src/plugins/defaults/memory/v2/router.ts +38 -15
- package/src/plugins/defaults/memory/v3/__tests__/candidate-match.test.ts +5 -3
- package/src/plugins/defaults/memory/v3/__tests__/carry-integration.test.ts +21 -8
- package/src/plugins/defaults/memory/v3/__tests__/edge.test.ts +1 -1
- package/src/plugins/defaults/memory/v3/__tests__/live-integration.test.ts +1 -1
- package/src/plugins/defaults/memory/v3/__tests__/maintain-job.test.ts +4 -1
- package/src/plugins/defaults/memory/v3/__tests__/orchestrate.test.ts +181 -3
- package/src/plugins/defaults/memory/v3/__tests__/selection-log-store.test.ts +3 -0
- package/src/plugins/defaults/memory/v3/__tests__/shadow-integration.test.ts +14 -5
- package/src/plugins/defaults/memory/v3/__tests__/shadow-plugin.test.ts +40 -50
- package/src/plugins/defaults/memory/v3/__tests__/tuning-profile.test.ts +1 -0
- package/src/plugins/defaults/memory/v3/candidate-match.ts +2 -2
- package/src/plugins/defaults/memory/v3/capabilities.ts +6 -4
- package/src/plugins/defaults/memory/v3/card.ts +25 -9
- package/src/plugins/defaults/memory/v3/edge.ts +52 -18
- package/src/plugins/defaults/memory/v3/gate.ts +4 -4
- package/src/plugins/defaults/memory/v3/injector.ts +10 -3
- package/src/plugins/defaults/memory/v3/maintain-job.ts +3 -3
- package/src/plugins/defaults/memory/v3/orchestrate.ts +100 -18
- package/src/plugins/defaults/memory/v3/page-content.ts +26 -10
- package/src/plugins/defaults/memory/v3/prune.ts +40 -16
- package/src/plugins/defaults/memory/v3/selection-log-store.ts +34 -12
- package/src/plugins/defaults/memory/v3/shadow-plugin.ts +9 -13
- package/src/plugins/defaults/memory/v3/span-query.test.ts +86 -0
- package/src/plugins/defaults/memory/v3/span-query.ts +71 -0
- package/src/plugins/defaults/memory/v3/tuning-profile.ts +3 -0
- package/src/plugins/defaults/memory/v3/types.ts +4 -1
- package/src/plugins/defaults/memory/v3-eval/eval-packets.ts +2 -2
- package/src/plugins/defaults/memory/validation.ts +1 -1
- package/src/plugins/defaults/memory/worker.ts +2 -1
- package/src/plugins/defaults/platform-hosted/__tests__/reengage.test.ts +104 -12
- package/src/plugins/defaults/platform-hosted/routes/reengage.ts +104 -12
- package/src/plugins/types.ts +2 -2
- package/src/prompts/sections.ts +4 -16
- package/src/providers/__tests__/registry-native-web-search.test.ts +47 -2
- package/src/providers/anthropic/client.ts +15 -6
- package/src/providers/content-block-schema.ts +26 -0
- package/src/providers/inference/__tests__/connections-credential-normalize.test.ts +87 -0
- package/src/providers/inference/adapter-factory.ts +6 -0
- package/src/providers/inference/connections.ts +30 -14
- package/src/providers/model-catalog.ts +77 -0
- package/src/providers/openai/__tests__/chat-completions-input-audio.test.ts +260 -0
- package/src/providers/openai/chat-completions-provider.ts +65 -17
- package/src/providers/openai/input-audio.ts +76 -0
- package/src/providers/openrouter/client.ts +0 -2
- package/src/providers/platform-proxy/constants.ts +4 -0
- package/src/providers/poolside/client.ts +42 -0
- package/src/providers/registry.ts +40 -8
- package/src/providers/retry.ts +2 -0
- package/src/providers/types.ts +39 -1
- package/src/providers/vercel-ai-gateway/client.ts +0 -2
- package/src/runtime/AGENTS.md +7 -5
- package/src/runtime/__tests__/question-request-guardian-bridge.test.ts +173 -0
- package/src/runtime/__tests__/question-resolution.test.ts +111 -0
- package/src/runtime/access-request-helper.ts +7 -15
- package/src/runtime/assistant-event-hub.ts +43 -31
- package/src/runtime/assistant-event-publish-options.ts +31 -0
- package/src/runtime/assistant-event.ts +19 -30
- package/src/runtime/assistant-stream-state.ts +4 -4
- package/src/runtime/channel-approvals.ts +0 -25
- package/src/runtime/channel-readiness-service.ts +99 -40
- package/src/runtime/channel-retry-sweep.ts +38 -15
- package/src/runtime/confirmation-request-guardian-bridge.ts +4 -0
- package/src/runtime/guardian-reply-router.ts +108 -0
- package/src/runtime/http-server.ts +8 -0
- package/src/runtime/http-types.ts +2 -2
- package/src/runtime/process-role.ts +42 -0
- package/src/runtime/question-request-guardian-bridge.ts +184 -0
- package/src/runtime/question-resolution.ts +153 -0
- package/src/runtime/routes/__tests__/bookmark-routes.test.ts +6 -2
- package/src/runtime/routes/__tests__/conversation-query-routes.test.ts +1 -1
- package/src/runtime/routes/__tests__/subagents-routes.test.ts +250 -0
- package/src/runtime/routes/__tests__/webhook-routes.test.ts +201 -0
- package/src/runtime/routes/acp-routes.ts +2 -2
- package/src/runtime/routes/bookmark-routes.ts +3 -15
- package/src/runtime/routes/canned-message-complete.ts +2 -2
- package/src/runtime/routes/consolidation-routes.ts +8 -2
- package/src/runtime/routes/conversation-list-routes.ts +3 -19
- package/src/runtime/routes/conversation-management-routes.ts +1 -5
- package/src/runtime/routes/conversation-query-routes.ts +52 -8
- package/src/runtime/routes/conversation-routes.ts +36 -22
- package/src/runtime/routes/credential-request-routes.ts +0 -1
- package/src/runtime/routes/filing-routes.ts +7 -7
- package/src/runtime/routes/global-search-routes.ts +1 -1
- package/src/runtime/routes/guardian-approval-prompt.ts +2 -2
- package/src/runtime/routes/image-generation-routes.ts +12 -9
- package/src/runtime/routes/inbound-message-handler.ts +17 -0
- package/src/runtime/routes/inbound-stages/acl-enforcement.test.ts +50 -0
- package/src/runtime/routes/inbound-stages/acl-enforcement.ts +19 -0
- package/src/runtime/routes/inbound-stages/background-dispatch.ts +4 -4
- package/src/runtime/routes/inbound-stages/inbound-content-prep.test.ts +129 -0
- package/src/runtime/routes/inbound-stages/inbound-content-prep.ts +141 -9
- package/src/runtime/routes/index.ts +2 -0
- package/src/runtime/routes/lifecycle-routes.ts +199 -0
- package/src/runtime/routes/platform-routes.ts +3 -8
- package/src/runtime/routes/playground/__tests__/inject-failures.test.ts +7 -7
- package/src/runtime/routes/playground/__tests__/reset-circuit.test.ts +5 -5
- package/src/runtime/routes/question-routes.ts +34 -87
- package/src/runtime/routes/secret-routes.ts +11 -1
- package/src/runtime/routes/settings-routes.ts +8 -18
- package/src/runtime/routes/subagents-routes.ts +35 -8
- package/src/runtime/routes/upgrade-broadcast-routes.ts +4 -5
- package/src/runtime/routes/webhook-routes.ts +89 -30
- package/src/runtime/routes/workspace-routes.test.ts +7 -4
- package/src/runtime/slack-reply-session.test.ts +12 -12
- package/src/runtime/slack-reply-session.ts +17 -17
- package/src/runtime/sync/sync-publisher.test.ts +7 -5
- package/src/runtime/tool-grant-request-helper.ts +4 -0
- package/src/schedule/schedule-store.ts +70 -0
- package/src/schedule/scheduler.ts +27 -0
- package/src/security/AGENTS.md +1 -1
- package/src/security/credential-key.ts +17 -0
- package/src/security/secret-ingress.ts +14 -105
- package/src/security/secret-patterns.ts +8 -139
- package/src/security/secret-scanner.ts +8 -3
- package/src/security/store-path-override.ts +6 -16
- package/src/sequence/store.ts +8 -0
- package/src/signals/emit-event.ts +5 -10
- package/src/signals/event-stream.ts +20 -10
- package/src/subagent/manager.ts +76 -25
- package/src/subagent/notify.ts +8 -0
- package/src/subagent/types.ts +13 -3
- package/src/telegram/__tests__/webhook-health.test.ts +442 -0
- package/src/telegram/webhook-health.ts +478 -0
- package/src/telemetry/AGENTS.md +6 -0
- package/src/telemetry/memory-corpus-size.test.ts +129 -0
- package/src/telemetry/memory-corpus-size.ts +184 -0
- package/src/telemetry/memory-tier-reporter.test.ts +67 -2
- package/src/telemetry/memory-tier-reporter.ts +30 -3
- package/src/tools/__tests__/conversation-group-tool-input-schemas.test.ts +61 -0
- package/src/tools/__tests__/document-tool-input-schemas.test.ts +103 -0
- package/src/tools/__tests__/followup-call-acp-tool-input-schemas.test.ts +125 -0
- package/src/tools/__tests__/schedule-tool-input-schemas.test.ts +83 -0
- package/src/tools/__tests__/subagent-workflow-tool-input-schemas.test.ts +116 -0
- package/src/tools/__tests__/tool-input-schemas.test.ts +192 -0
- package/src/tools/__tests__/tool-schema-drift-harness.ts +142 -0
- package/src/tools/acp/abort.ts +21 -1
- package/src/tools/acp/context.ts +5 -5
- package/src/tools/acp/spawn.test.ts +15 -5
- package/src/tools/acp/spawn.ts +26 -3
- package/src/tools/acp/status.ts +20 -1
- package/src/tools/acp/steer.test.ts +20 -0
- package/src/tools/acp/steer.ts +23 -2
- package/src/tools/apps/resolve-app-id.ts +15 -3
- package/src/tools/ask-question/ask-question-tool.test.ts +65 -0
- package/src/tools/ask-question/ask-question-tool.ts +68 -92
- package/src/tools/browser/browser-screencast.ts +6 -6
- package/src/tools/calls/call-end.ts +24 -3
- package/src/tools/calls/call-start.ts +35 -11
- package/src/tools/calls/call-status.ts +20 -1
- package/src/tools/conversation-groups/group_create.ts +63 -0
- package/src/tools/conversation-groups/group_list.ts +20 -0
- package/src/tools/conversation-groups/group_shared.ts +52 -0
- package/src/tools/conversation-groups/move_to_group.ts +106 -0
- package/src/tools/document/document-comment-tool.test.ts +49 -0
- package/src/tools/document/document-comment-tool.ts +45 -6
- package/src/tools/document/document-tool.ts +142 -24
- package/src/tools/executor.ts +27 -13
- package/src/tools/filesystem/edit.ts +62 -60
- package/src/tools/filesystem/list.ts +36 -29
- package/src/tools/filesystem/read.ts +56 -39
- package/src/tools/filesystem/write.ts +44 -38
- package/src/tools/followups/followup_create.ts +41 -12
- package/src/tools/followups/followup_list.ts +34 -6
- package/src/tools/followups/followup_resolve.ts +26 -4
- package/src/tools/host-filesystem/edit.ts +50 -60
- package/src/tools/host-filesystem/read.ts +45 -39
- package/src/tools/host-filesystem/write.ts +31 -36
- package/src/tools/host-terminal/host-shell.ts +76 -62
- package/src/tools/policy-context.ts +27 -10
- package/src/tools/schedule/create.ts +76 -33
- package/src/tools/schedule/delete.ts +21 -2
- package/src/tools/schedule/list.ts +25 -3
- package/src/tools/schedule/update.ts +105 -48
- package/src/tools/shared/filesystem/path-policy.ts +246 -40
- package/src/tools/shared/zod-tool-schema.ts +117 -0
- package/src/tools/subagent/abort.ts +12 -4
- package/src/tools/subagent/consult-deadline.ts +9 -3
- package/src/tools/subagent/message.ts +21 -5
- package/src/tools/subagent/notify-parent.ts +44 -23
- package/src/tools/subagent/read.ts +15 -4
- package/src/tools/subagent/resolve.ts +26 -4
- package/src/tools/subagent/spawn.ts +37 -13
- package/src/tools/subagent/status.ts +12 -4
- package/src/tools/system/request-permission.ts +35 -26
- package/src/tools/terminal/shell.ts +68 -55
- package/src/tools/tool-approval-handler.ts +327 -53
- package/src/tools/tool-input-schemas.ts +85 -0
- package/src/tools/tool-types.ts +7 -0
- package/src/tools/types.ts +8 -0
- package/src/tools/workflows/manage-workflows.ts +30 -2
- package/src/tools/workflows/run-workflow.test.ts +54 -2
- package/src/tools/workflows/run-workflow.ts +32 -4
- package/src/util/fs-symlinks.ts +30 -0
- package/src/util/log-redact.ts +7 -3
- package/src/util/platform.ts +15 -0
- package/src/vellum-assistant-namespace.d.ts +80 -0
- package/src/watcher/watcher-store.ts +8 -0
- package/src/workflows/leaf-runner.test.ts +104 -2
- package/src/workflows/leaf-runner.ts +31 -6
- package/src/workspace/migrations/070-memory-v2-summary-schema-rebuild.ts +2 -2
- package/src/workspace/migrations/134-image-generation-mode-to-provider.ts +128 -0
- package/src/workspace/migrations/135-copy-substrate-tunables.ts +248 -0
- package/src/workspace/migrations/__tests__/135-copy-substrate-tunables.test.ts +446 -0
- package/src/workspace/migrations/__tests__/backfill-v3-leaves.test.ts +2 -2
- package/src/workspace/migrations/registry.ts +4 -0
- package/tsconfig.plugin-api.json +1 -1
- package/src/__tests__/memory-retrieval-budget.test.ts +0 -49
- package/src/__tests__/secret-patterns.test.ts +0 -27
- package/src/config/__tests__/proc-to-skills.test.ts +0 -21
- package/src/config/memory-tier.test.ts +0 -44
- package/src/daemon/message-types/acp.ts +0 -20
- package/src/daemon/message-types/apps.ts +0 -11
- package/src/daemon/message-types/background-tools.ts +0 -14
- package/src/daemon/message-types/bookmarks.ts +0 -13
- package/src/daemon/message-types/contacts.ts +0 -13
- package/src/daemon/message-types/document-comments.ts +0 -14
- package/src/daemon/message-types/documents.ts +0 -13
- package/src/daemon/message-types/home.ts +0 -15
- package/src/daemon/message-types/host-bash.ts +0 -12
- package/src/daemon/message-types/host-cu.ts +0 -10
- package/src/daemon/message-types/host-file.ts +0 -12
- package/src/daemon/message-types/host-transfer.ts +0 -12
- package/src/daemon/message-types/integrations.ts +0 -90
- package/src/daemon/message-types/memory.ts +0 -10
- package/src/daemon/message-types/schedules.ts +0 -14
- package/src/daemon/message-types/settings.ts +0 -17
- package/src/daemon/message-types/subagents.ts +0 -63
- package/src/daemon/message-types/upgrades.ts +0 -14
- package/src/daemon/message-types/workflows.ts +0 -25
- package/src/daemon/message-types/workspace.ts +0 -137
- package/src/live-voice/ack-phrases.ts +0 -58
- package/src/plugins/defaults/memory/retrieval-budget.ts +0 -29
- package/src/plugins/defaults/memory/v3/__tests__/gate-flag.test.ts +0 -69
- package/src/plugins/defaults/memory/v3/gate-flag.ts +0 -20
- package/src/plugins/defaults/memory/v3/substrate/__tests__/memory-v2-startup.test.ts +0 -86
- package/src/plugins/defaults/memory/v3/substrate/memory-v2-startup.test.ts +0 -205
- /package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/concept-page-frontmatter-schema.test.ts +0 -0
- /package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/consolidation-prompt-flag-gating-guard.test.ts +0 -0
- /package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/injected-block-slugs.test.ts +0 -0
- /package/src/plugins/defaults/memory/{v3/substrate → substrate}/cli-command-content.ts +0 -0
|
@@ -7,7 +7,8 @@ import {
|
|
|
7
7
|
} from "../calls/media-turn-detector.js";
|
|
8
8
|
import { sanitizeForTts } from "../calls/tts-text-sanitizer.js";
|
|
9
9
|
import {
|
|
10
|
-
|
|
10
|
+
isIncompleteControlMarkerTail,
|
|
11
|
+
MINIMIZE_ROOM_MARKER,
|
|
11
12
|
stripInternalSpeechMarkers,
|
|
12
13
|
} from "../calls/voice-control-protocol.js";
|
|
13
14
|
import type {
|
|
@@ -30,22 +31,22 @@ import {
|
|
|
30
31
|
MIN_SPOKEN_BRIDGE_CHARS,
|
|
31
32
|
type VoiceRoutingLeg,
|
|
32
33
|
} from "../calls/voice-triage-escalate.js";
|
|
33
|
-
import { isAssistantFeatureFlagEnabled } from "../config/assistant-feature-flags.js";
|
|
34
34
|
import { getConfig } from "../config/loader.js";
|
|
35
|
-
import type { AssistantConfig } from "../config/schema.js";
|
|
36
35
|
import {
|
|
37
36
|
type LiveVoiceFrontModelConfig,
|
|
38
37
|
LiveVoiceFrontModelConfigSchema,
|
|
39
38
|
} from "../config/schemas/live-voice.js";
|
|
40
39
|
import { ABORT_WATCHDOG_MS } from "../daemon/abort-watchdog.js";
|
|
41
|
-
import {
|
|
40
|
+
import { isRefusedInReadOnlyPass } from "../daemon/conversation-tool-setup.js";
|
|
42
41
|
import type { TrustContext } from "../daemon/trust-context-types.js";
|
|
42
|
+
import { isInstalledStaticSkillLoad } from "../permissions/checker.js";
|
|
43
43
|
import { ensureConversationExists } from "../persistence/conversation-crud.js";
|
|
44
44
|
import {
|
|
45
45
|
listProviderIds,
|
|
46
46
|
supportsBoundary,
|
|
47
47
|
} from "../providers/speech-to-text/provider-catalog.js";
|
|
48
48
|
import type { ResolveStreamingTranscriberOptions } from "../providers/speech-to-text/resolve.js";
|
|
49
|
+
import { broadcastMessage } from "../runtime/assistant-event-hub.js";
|
|
49
50
|
import { publishConversationListAndMetadataChanged } from "../runtime/sync/resource-sync-events.js";
|
|
50
51
|
import { detectPcm16SpeechActivity } from "../stt/speech-energy.js";
|
|
51
52
|
import type {
|
|
@@ -54,13 +55,12 @@ import type {
|
|
|
54
55
|
SttStreamServerEvent,
|
|
55
56
|
} from "../stt/types.js";
|
|
56
57
|
import { getSubagentManager } from "../subagent/index.js";
|
|
58
|
+
import { getToolOwner } from "../tools/registry.js";
|
|
57
59
|
import { extractSpeakableSegments } from "../tts/speakable-segments.js";
|
|
58
60
|
import { createAbortReason } from "../util/abort-reasons.js";
|
|
59
61
|
import { getLogger } from "../util/logger.js";
|
|
60
|
-
import { pickAckPhrase, pickProgressPhrase } from "./ack-phrases.js";
|
|
61
62
|
import {
|
|
62
63
|
createVoiceFrontDecider,
|
|
63
|
-
type VoiceEndpointAction,
|
|
64
64
|
type VoiceFrontDecider,
|
|
65
65
|
type VoiceProgressTextInput,
|
|
66
66
|
} from "./front-decision.js";
|
|
@@ -76,6 +76,7 @@ import {
|
|
|
76
76
|
type LiveVoiceMetricsEvent,
|
|
77
77
|
type LiveVoiceSpokenAckKind,
|
|
78
78
|
type LiveVoiceTurnSeedMarks,
|
|
79
|
+
type VoiceEndpointAction,
|
|
79
80
|
} from "./live-voice-metrics.js";
|
|
80
81
|
import {
|
|
81
82
|
type LiveVoiceSession as LiveVoiceSessionContract,
|
|
@@ -88,6 +89,7 @@ import type {
|
|
|
88
89
|
LiveVoiceTtsOptions,
|
|
89
90
|
LiveVoiceTtsResult,
|
|
90
91
|
} from "./live-voice-tts.js";
|
|
92
|
+
import { pickProgressPhrase } from "./progress-phrases.js";
|
|
91
93
|
import {
|
|
92
94
|
type LiveVoiceClientFrame,
|
|
93
95
|
type LiveVoiceClientUpdateConfigFrame,
|
|
@@ -151,6 +153,22 @@ const BARGE_IN_MAX_TOLERATED_SILENCE_RATIO = 4;
|
|
|
151
153
|
// a segment is at most ~180 chars of speech (~10 s of 24 kHz mono PCM
|
|
152
154
|
// ≈ 480 KB), so one buffered segment is an acceptable bound.
|
|
153
155
|
const TTS_MAX_OPEN_SYNTHESIS_JOBS = 2;
|
|
156
|
+
// Audible silence required before a finished background continuation's result
|
|
157
|
+
// is spoken into a live call. Long enough that the announcement lands in a real
|
|
158
|
+
// lull rather than on the heels of the turn that just ended; short enough that
|
|
159
|
+
// the user is not left wondering whether the work survived.
|
|
160
|
+
const CONTINUATION_ANNOUNCE_SILENCE_MS = 1_500;
|
|
161
|
+
// How many times a pending announcement re-arms itself against a client
|
|
162
|
+
// playback tail that outlasted its timer. Each re-arm waits out the whole
|
|
163
|
+
// remaining tail, so one covers a reply of any length; the rest cover a fresh
|
|
164
|
+
// reply's audio landing between two checks. Beyond the cap the announcement
|
|
165
|
+
// falls back to the stash rather than chasing a call that keeps talking.
|
|
166
|
+
const CONTINUATION_ANNOUNCE_MAX_DRAIN_REARMS = 3;
|
|
167
|
+
// `content` of an announcement turn. The answer never rides here — it goes in
|
|
168
|
+
// the model-facing control prompt (buildLiveDeliveryNote), so the only thing
|
|
169
|
+
// persisted on the user side is this marker, and it persists hidden.
|
|
170
|
+
export const CONTINUATION_DELIVERY_CONTENT =
|
|
171
|
+
"(background work finished — deliver it now)";
|
|
154
172
|
|
|
155
173
|
export type LiveVoiceStreamingTranscriberResolver = (
|
|
156
174
|
options: ResolveStreamingTranscriberOptions,
|
|
@@ -167,12 +185,13 @@ export type LiveVoiceTtsStreamer = (
|
|
|
167
185
|
options: LiveVoiceTtsOptions,
|
|
168
186
|
) => Promise<LiveVoiceTtsResult>;
|
|
169
187
|
|
|
170
|
-
// Runs an interrupted live-voice turn to completion on a background subagent
|
|
171
|
-
//
|
|
172
|
-
//
|
|
173
|
-
//
|
|
174
|
-
// the
|
|
175
|
-
// for testability; the factory wires the real SubagentManager-backed
|
|
188
|
+
// Runs an interrupted live-voice turn to completion on a background subagent.
|
|
189
|
+
// The run itself is silent (no parent notification); it RESOLVES with the
|
|
190
|
+
// continuation's final answer text, and the session decides where that goes —
|
|
191
|
+
// the next turn the user starts, an announcement into audible silence, or the
|
|
192
|
+
// conversation once the call has ended. Aborts when `signal` fires (rejecting).
|
|
193
|
+
// Injected for testability; the factory wires the real SubagentManager-backed
|
|
194
|
+
// impl.
|
|
176
195
|
export type LiveVoiceBackgroundContinuationSpawner = (args: {
|
|
177
196
|
parentConversationId: string;
|
|
178
197
|
objective: string;
|
|
@@ -244,17 +263,15 @@ export interface LiveVoiceSessionOptions {
|
|
|
244
263
|
*/
|
|
245
264
|
frontModelConfig?: Partial<LiveVoiceFrontModelConfig>;
|
|
246
265
|
/**
|
|
247
|
-
* Front-model
|
|
248
|
-
*
|
|
249
|
-
*
|
|
250
|
-
*
|
|
251
|
-
* disables front-model LLM calls (energy endpointing + static ack phrasing).
|
|
266
|
+
* Front-model phrasing service: spoken acks and progress narration. The
|
|
267
|
+
* production factory constructs it from `liveVoice.frontModel` config when
|
|
268
|
+
* unset; `null` disables front-model LLM calls, so no ack is ever spoken
|
|
269
|
+
* and narration falls back to its static phrases.
|
|
252
270
|
*/
|
|
253
271
|
frontDecider?: VoiceFrontDecider | null;
|
|
254
272
|
/**
|
|
255
|
-
* Spawns the background continuation for a barged-in turn
|
|
256
|
-
*
|
|
257
|
-
* SubagentManager-backed implementation; tests inject a stub.
|
|
273
|
+
* Spawns the background continuation for a barged-in turn. The factory wires
|
|
274
|
+
* the real SubagentManager-backed implementation; tests inject a stub.
|
|
258
275
|
*/
|
|
259
276
|
spawnBackgroundContinuation?: LiveVoiceBackgroundContinuationSpawner;
|
|
260
277
|
/**
|
|
@@ -272,6 +289,12 @@ export interface LiveVoiceSessionOptions {
|
|
|
272
289
|
* teardown budget (`resolveProcessingWaitMs`).
|
|
273
290
|
*/
|
|
274
291
|
detachTeardownSettleTimeoutMs?: number;
|
|
292
|
+
/**
|
|
293
|
+
* Overrides the audible silence a finished continuation waits out before its
|
|
294
|
+
* result is announced into a live, idle call (test hook). Defaults to
|
|
295
|
+
* `CONTINUATION_ANNOUNCE_SILENCE_MS`.
|
|
296
|
+
*/
|
|
297
|
+
continuationAnnounceSilenceMs?: number;
|
|
275
298
|
}
|
|
276
299
|
|
|
277
300
|
type LiveVoiceUtterancePhase =
|
|
@@ -300,6 +323,13 @@ interface UtteranceCycle {
|
|
|
300
323
|
// this set at a time — see pumpFinalizeQueue.
|
|
301
324
|
finalizeRequested: boolean;
|
|
302
325
|
transcriber: StreamingTranscriber | null;
|
|
326
|
+
// Manual (push-to-talk) capture routed at least one chunk into this cycle.
|
|
327
|
+
// It is the only evidence the user is mid-utterance before STT emits
|
|
328
|
+
// anything: the pending buffers empty out as soon as the transcriber is
|
|
329
|
+
// streaming, and the archival buffer only fills when audio archiving is on.
|
|
330
|
+
// server_vad has the turn detector for the same question, and never sets
|
|
331
|
+
// this — its ingress is handleServerVadAudio.
|
|
332
|
+
manualAudioCaptured: boolean;
|
|
303
333
|
pendingAudioChunks: Buffer[];
|
|
304
334
|
pendingAudioBytes: number;
|
|
305
335
|
finalTranscriptSegments: string[];
|
|
@@ -380,6 +410,13 @@ interface TurnProgressState {
|
|
|
380
410
|
// op, on start (not completion), so a burst of slow tools still trips the
|
|
381
411
|
// threshold while they run.
|
|
382
412
|
opsSinceNarration: number;
|
|
413
|
+
// Bumped by every observable change to the turn's tool activity — an op
|
|
414
|
+
// starting or finishing. The idle trigger compares it against
|
|
415
|
+
// `narratedEpoch` so a tick with nothing new to report stays silent.
|
|
416
|
+
stateEpoch: number;
|
|
417
|
+
// The `stateEpoch` the last spoken narration described: the activity the
|
|
418
|
+
// user has already been told about.
|
|
419
|
+
narratedEpoch: number;
|
|
383
420
|
// Narrations actually spoken this turn — the metrics count and the
|
|
384
421
|
// decider's 1-based updateIndex. Rate, not count, bounds narration:
|
|
385
422
|
// idleIntervalMs/minGapMs cap the cadence and the session duration cap
|
|
@@ -428,6 +465,9 @@ interface ActiveAssistantTurn {
|
|
|
428
465
|
progress: TurnProgressState;
|
|
429
466
|
assistantCompleted: boolean;
|
|
430
467
|
ttsDone: boolean;
|
|
468
|
+
// Latched when the leg's stream contains MINIMIZE_ROOM_MARKER; consumed
|
|
469
|
+
// once at TTS drain, where the minimize_room frame goes out after tts_done.
|
|
470
|
+
minimizeRequested: boolean;
|
|
431
471
|
// A tts_audio frame actually went out to the client — latches on the first
|
|
432
472
|
// forwarded chunk so the firstTtsAudio metric is marked exactly once per turn.
|
|
433
473
|
ttsAudioStarted: boolean;
|
|
@@ -468,11 +508,29 @@ interface ActiveAssistantTurn {
|
|
|
468
508
|
// merges it with this turn's utterance instead of treating that utterance as
|
|
469
509
|
// a fresh follow-up. Null for an ordinary (non-barge-in) turn.
|
|
470
510
|
interruptedRequest: string | null;
|
|
471
|
-
// The completed background continuation's answer
|
|
472
|
-
//
|
|
473
|
-
//
|
|
474
|
-
// Null when no continuation result is pending for this turn.
|
|
511
|
+
// The completed background continuation's answer, folded into this turn's
|
|
512
|
+
// control prompt as context so the model can deliver or reference it in reply
|
|
513
|
+
// to the user's own utterance. Null when no continuation result is pending.
|
|
475
514
|
continuationResult: string | null;
|
|
515
|
+
// Set only on an announcement turn: the interrupted request and the
|
|
516
|
+
// continuation's answer, which the turn exists solely to deliver. The turn has
|
|
517
|
+
// no user utterance behind it — `content` is CONTINUATION_DELIVERY_CONTENT and
|
|
518
|
+
// the answer rides the control prompt (buildLiveDeliveryNote).
|
|
519
|
+
continuationDelivery: ContinuationDelivery | null;
|
|
520
|
+
// Set when a barge-in handed the interrupted work to a background subagent:
|
|
521
|
+
// that request's transcript, so the model can tell the user the work is
|
|
522
|
+
// still running instead of appearing to have dropped it.
|
|
523
|
+
handedOffRequest: string | null;
|
|
524
|
+
// The queued announcement this turn consumed alongside `continuationResult`
|
|
525
|
+
// — the two are routes for one answer, so launching consumes both. Held here
|
|
526
|
+
// so a rolled-back speculative dispatch can re-arm it (see
|
|
527
|
+
// restorePendingTurnContext). Null on the announcement turn itself, which
|
|
528
|
+
// owns its delivery outright.
|
|
529
|
+
consumedAnnouncement: ContinuationDelivery | null;
|
|
530
|
+
// `detachStopGeneration` at the moment this turn consumed the pending
|
|
531
|
+
// context. A bump since then means a stop/interrupt/supersede deliberately
|
|
532
|
+
// dropped the continuation, so a rollback must not resurrect it.
|
|
533
|
+
pendingContextStopGeneration: number;
|
|
476
534
|
// The agent run started a definitive tool use this turn — tool use implies
|
|
477
535
|
// a guaranteed-slow turn, so acknowledgment logic can key off this.
|
|
478
536
|
toolUseStarted: boolean;
|
|
@@ -487,9 +545,9 @@ interface ActiveAssistantTurn {
|
|
|
487
545
|
// A spoken ack was enqueued this turn. Every ack trigger shares this
|
|
488
546
|
// one-per-turn budget so a slow turn never stacks fillers.
|
|
489
547
|
ackSpoken: boolean;
|
|
490
|
-
// An
|
|
491
|
-
//
|
|
492
|
-
//
|
|
548
|
+
// An ack generation is awaiting the decider. While true, the ack has not
|
|
549
|
+
// yet stamped `lastFloorHolderAtMs`, so narration must treat it as a
|
|
550
|
+
// floor-holder-in-waiting and stand down —
|
|
493
551
|
// otherwise a progress phrase could land back-to-back with the ack.
|
|
494
552
|
ackGenerationPending: boolean;
|
|
495
553
|
// Pending slow-first-delta ack timer; null once cleared or fired.
|
|
@@ -516,13 +574,87 @@ interface ActiveAssistantTurn {
|
|
|
516
574
|
assistantAudioSampleRate?: number;
|
|
517
575
|
}
|
|
518
576
|
|
|
577
|
+
/**
|
|
578
|
+
* Control-marker hygiene for one model leg's delta stream, shared by the
|
|
579
|
+
* front-door answer stage and the default/escalated leg. The returned flush
|
|
580
|
+
* forwards the stripped (stripInternalSpeechMarkers) prefix of `raw` that has
|
|
581
|
+
* not been emitted yet and cannot contain a still-streaming control marker:
|
|
582
|
+
* the flush stops at the first "[" whose tail is an incomplete marker
|
|
583
|
+
* (isIncompleteControlMarkerTail) and holds from there until a later delta
|
|
584
|
+
* completes or disproves it; `force` (leg completion) emits the held tail so
|
|
585
|
+
* real text that merely resembles a marker prefix is not dropped. The scan
|
|
586
|
+
* runs forward from the emitted boundary — not from the last "[" — so
|
|
587
|
+
* brackets INSIDE a streaming marker body (a JSON array or "]"-bearing string
|
|
588
|
+
* in ASK_GUARDIAN_APPROVAL) can neither mask the marker's start nor pass as
|
|
589
|
+
* its terminator. As a side effect the force flush latches the turn's
|
|
590
|
+
* `minimizeRequested` when the completed stream ENDS with
|
|
591
|
+
* MINIMIZE_ROOM_MARKER — terminal position only, matching the prompt's
|
|
592
|
+
* "end the reply with it" contract, so a reply whose CONTENT happens to
|
|
593
|
+
* contain "[-1]" (an array literal, a temperature) never minimizes the
|
|
594
|
+
* room. The marker itself is stripped wherever it appears (the shared
|
|
595
|
+
* marker-strip convention), so the latch is the only observable.
|
|
596
|
+
*/
|
|
597
|
+
function createControlMarkerHoldback(
|
|
598
|
+
turn: ActiveAssistantTurn,
|
|
599
|
+
emit: (chunk: string) => void,
|
|
600
|
+
): (raw: string, opts?: { force?: boolean }) => void {
|
|
601
|
+
let emitted = 0;
|
|
602
|
+
return (raw, opts) => {
|
|
603
|
+
if (
|
|
604
|
+
opts?.force === true &&
|
|
605
|
+
!turn.minimizeRequested &&
|
|
606
|
+
raw.trimEnd().endsWith(MINIMIZE_ROOM_MARKER)
|
|
607
|
+
) {
|
|
608
|
+
turn.minimizeRequested = true;
|
|
609
|
+
}
|
|
610
|
+
let safeEnd = raw.length;
|
|
611
|
+
if (opts?.force !== true) {
|
|
612
|
+
for (
|
|
613
|
+
let i = raw.indexOf("[", emitted);
|
|
614
|
+
i !== -1;
|
|
615
|
+
i = raw.indexOf("[", i + 1)
|
|
616
|
+
) {
|
|
617
|
+
if (isIncompleteControlMarkerTail(raw.slice(i))) {
|
|
618
|
+
safeEnd = i;
|
|
619
|
+
break;
|
|
620
|
+
}
|
|
621
|
+
}
|
|
622
|
+
}
|
|
623
|
+
if (safeEnd > emitted) {
|
|
624
|
+
emit(stripInternalSpeechMarkers(raw.slice(emitted, safeEnd)));
|
|
625
|
+
emitted = safeEnd;
|
|
626
|
+
}
|
|
627
|
+
};
|
|
628
|
+
}
|
|
629
|
+
|
|
519
630
|
// Base control prompt for every live-voice turn. When a turn starts from a
|
|
520
631
|
// barge-in, the interruption merge note is appended to it (see
|
|
521
632
|
// buildInterruptionMergeNote) so the model reconciles the interrupted request
|
|
522
633
|
// with the new utterance.
|
|
523
|
-
const
|
|
524
|
-
"You are speaking in a local live voice session. Keep replies brief and conversational. You cannot display cards, forms, or any on-screen UI during the call — convey everything in speech. "
|
|
525
|
-
|
|
634
|
+
const LIVE_VOICE_CONTROL_PROMPT_BASE =
|
|
635
|
+
"You are speaking in a local live voice session. Keep replies brief and conversational. You cannot display cards, forms, or any on-screen UI during the call — convey everything in speech. ";
|
|
636
|
+
|
|
637
|
+
// MINIMIZE_ROOM_MARKER teaching, appended for the legs that can actually put
|
|
638
|
+
// something on screen — the main leg and the escalated leg. The front-door
|
|
639
|
+
// (fast) leg never receives it: that leg is toolless, so it has nothing to
|
|
640
|
+
// show, and its decision rule promises that apart from a leading verdict
|
|
641
|
+
// token every character is spoken verbatim — teaching it the marker would
|
|
642
|
+
// contradict that rule and could only produce spurious minimizes.
|
|
643
|
+
//
|
|
644
|
+
// The teaching deliberately keeps the no-interactive-UI rule intact
|
|
645
|
+
// (live-voice turns are non-interactive, JARVIS-1291): the marker does not
|
|
646
|
+
// render UI — it asks the client to reveal the screen the call overlay
|
|
647
|
+
// covers. The gate in createControlMarkerHoldback strips it from speech, and
|
|
648
|
+
// completeTtsForTurn sends the minimize_room frame only after the reply's
|
|
649
|
+
// TTS drains, so "end your reply with it" is the whole contract the model
|
|
650
|
+
// needs.
|
|
651
|
+
//
|
|
652
|
+
// The narration guidance is load-bearing: without it, the base prompt's
|
|
653
|
+
// convey-everything-in-speech rule reads to the model as "the user can never
|
|
654
|
+
// see the screen", and it minimizes the room while telling the user it has
|
|
655
|
+
// no way to show them anything ("check it from the app later").
|
|
656
|
+
const LIVE_VOICE_MINIMIZE_MARKER_TEACHING =
|
|
657
|
+
"The call renders as a full-screen overlay covering the app. If this reply created or changed something on screen worth looking at (an app, a page, a document), you may end the reply with the marker [-1]: after you finish speaking, the overlay minimizes so the user can see the screen while the call continues. The user is in the app and sees what you made the moment the overlay minimizes, so when you end with the marker, speak as if you are showing it to them right now (for example, close with something like: take a look) — never say you cannot show it, that you cannot display it because this is a voice call, or that they should check it later. Use it at most once per reply, only when there is genuinely something new to show, never speak the marker aloud or mention it, and never emit any other bracketed marker. ";
|
|
526
658
|
|
|
527
659
|
// System-level guidance appended to a barge-in turn's control prompt so the
|
|
528
660
|
// model treats the new utterance as a continuation of the request it was cut
|
|
@@ -533,30 +665,143 @@ function buildInterruptionMergeNote(interruptedRequest: string): string {
|
|
|
533
665
|
}
|
|
534
666
|
|
|
535
667
|
// System-level guidance appended to the NEXT turn's control prompt after a
|
|
536
|
-
// background continuation
|
|
537
|
-
//
|
|
538
|
-
//
|
|
539
|
-
//
|
|
540
|
-
//
|
|
668
|
+
// background continuation finished the reply the user interrupted. Folds the
|
|
669
|
+
// completed answer in as context so the model can deliver or reference it in
|
|
670
|
+
// its reply to whatever the user just said — this turn answers the user, so the
|
|
671
|
+
// result is offered only if it fits. Reaches the model only; it is not a user
|
|
672
|
+
// message and never renders as a transcript bubble.
|
|
541
673
|
function buildResurfaceContextNote(continuationResult: string): string {
|
|
542
674
|
return `Earlier the user interrupted you, and in the background you finished the reply they cut off. What you worked out was: "${continuationResult}". If their current message relates to it, use it to answer; otherwise you may briefly offer it or leave it aside, and do not repeat it verbatim if it no longer fits.`;
|
|
543
675
|
}
|
|
544
676
|
|
|
545
|
-
//
|
|
546
|
-
//
|
|
547
|
-
//
|
|
548
|
-
//
|
|
549
|
-
|
|
550
|
-
|
|
677
|
+
// Appended to the turn that follows a barge-in whose interrupted work was
|
|
678
|
+
// handed to a background subagent. Without this the assistant simply stops
|
|
679
|
+
// talking about the request it was mid-way through, and the user has no way to
|
|
680
|
+
// know the work survived — it reads as dropped. The model decides whether to
|
|
681
|
+
// mention it, because only it can tell "keep working on that" (the foreground
|
|
682
|
+
// turn continues the same work, so announcing a background copy would be
|
|
683
|
+
// confusing) from a genuine topic change (where "I'm still working on that in
|
|
684
|
+
// the background" is exactly what the user needs to hear).
|
|
685
|
+
function buildHandoffAnnouncementNote(handedOffRequest: string): string {
|
|
686
|
+
return `You handed your unfinished work on "${handedOffRequest}" to a background task, which is still running. If the user has moved to a different topic, briefly let them know you are still working on it in the background before answering them. If they are asking you to continue that same work, just continue and do not mention the background task.`;
|
|
687
|
+
}
|
|
688
|
+
|
|
689
|
+
// Names the interrupted request inside a delivery prompt. A barge-in can fire
|
|
690
|
+
// before any transcript landed, so the request may be empty.
|
|
691
|
+
function describeInterruptedRequest(request: string): string {
|
|
692
|
+
return request.length > 0
|
|
693
|
+
? `their earlier request ("${request}")`
|
|
694
|
+
: "their earlier request";
|
|
695
|
+
}
|
|
696
|
+
|
|
697
|
+
// A finished background continuation waiting to be delivered: the request it
|
|
698
|
+
// took over and the answer it produced.
|
|
699
|
+
interface ContinuationDelivery {
|
|
700
|
+
request: string;
|
|
701
|
+
answer: string;
|
|
702
|
+
}
|
|
703
|
+
|
|
704
|
+
// Appended to an announcement turn's control prompt: the turn the session
|
|
705
|
+
// starts on its own when a background continuation finishes while the call is
|
|
706
|
+
// live and nobody is speaking. There is no user utterance behind it, so this
|
|
707
|
+
// note is the whole instruction and the answer rides here rather than in the
|
|
708
|
+
// turn's `content`. Reaches the model only; it is not a user message and never
|
|
709
|
+
// renders as a transcript bubble.
|
|
710
|
+
function buildLiveDeliveryNote(request: string, answer: string): string {
|
|
711
|
+
const what = describeInterruptedRequest(request);
|
|
712
|
+
return `The user interrupted you earlier, and in the background you finished ${what}. The call is still live and nobody is speaking, so tell them briefly that it is done and give them the result. Do not re-run any tool calls; the work is already complete, and do not repeat it verbatim if it no longer fits. What you produced was:\n\n${answer}`;
|
|
713
|
+
}
|
|
714
|
+
|
|
715
|
+
// Assemble a leg's model-facing control prompt: the base live-voice rules,
|
|
716
|
+
// the [-1] minimize teaching (withheld from the front-door leg — see
|
|
717
|
+
// LIVE_VOICE_MINIMIZE_MARKER_TEACHING), the shared no-setup-flows rule, plus
|
|
718
|
+
// any pending barge-in merge context, completed-continuation context, and/or
|
|
719
|
+
// the announcement instruction. A turn can carry several (a barge-in follow-up
|
|
720
|
+
// that also has a continuation result waiting); the notes are model-only and
|
|
721
|
+
// never render as user bubbles.
|
|
722
|
+
function buildVoiceControlPrompt(
|
|
723
|
+
turn: ActiveAssistantTurn,
|
|
724
|
+
leg: { frontDoor?: boolean },
|
|
725
|
+
): string {
|
|
726
|
+
let prompt =
|
|
727
|
+
LIVE_VOICE_CONTROL_PROMPT_BASE +
|
|
728
|
+
(leg.frontDoor === true ? "" : LIVE_VOICE_MINIMIZE_MARKER_TEACHING) +
|
|
729
|
+
VOICE_NO_SETUP_FLOWS_RULE;
|
|
551
730
|
if (turn.interruptedRequest) {
|
|
552
731
|
prompt = `${prompt}\n\n${buildInterruptionMergeNote(turn.interruptedRequest)}`;
|
|
553
732
|
}
|
|
554
733
|
if (turn.continuationResult) {
|
|
555
734
|
prompt = `${prompt}\n\n${buildResurfaceContextNote(turn.continuationResult)}`;
|
|
556
735
|
}
|
|
736
|
+
if (turn.handedOffRequest) {
|
|
737
|
+
prompt = `${prompt}\n\n${buildHandoffAnnouncementNote(turn.handedOffRequest)}`;
|
|
738
|
+
}
|
|
739
|
+
if (turn.continuationDelivery) {
|
|
740
|
+
prompt = `${prompt}\n\n${buildLiveDeliveryNote(
|
|
741
|
+
turn.continuationDelivery.request,
|
|
742
|
+
turn.continuationDelivery.answer,
|
|
743
|
+
)}`;
|
|
744
|
+
}
|
|
557
745
|
return prompt;
|
|
558
746
|
}
|
|
559
747
|
|
|
748
|
+
// Delivered into the conversation when a continuation finishes AFTER the voice
|
|
749
|
+
// session ended. There is no next voice turn to fold into, so this lands as a
|
|
750
|
+
// normal turn in the thread — where the user actually goes looking for the
|
|
751
|
+
// work. Framed as a system-style report rather than a user request so the
|
|
752
|
+
// reply reads as "here is what I finished", not a fresh instruction.
|
|
753
|
+
function buildClosedSessionDeliveryPrompt(
|
|
754
|
+
interruptedRequest: string,
|
|
755
|
+
answer: string,
|
|
756
|
+
): string {
|
|
757
|
+
const what = describeInterruptedRequest(interruptedRequest);
|
|
758
|
+
return `[Background work finished] The voice call ended before the completed result for ${what} could be delivered. Tell the user briefly that it is done and give them the result. Do not re-run any tool calls; the work is already complete. What you produced was:\n\n${answer}`;
|
|
759
|
+
}
|
|
760
|
+
|
|
761
|
+
// A fresh cycle: nothing captured, nothing released, no metrics turn open.
|
|
762
|
+
function createUtteranceCycle(): UtteranceCycle {
|
|
763
|
+
return {
|
|
764
|
+
phase: "pending",
|
|
765
|
+
released: false,
|
|
766
|
+
assistantTurnStarted: false,
|
|
767
|
+
completed: false,
|
|
768
|
+
finalizeRequested: false,
|
|
769
|
+
transcriber: null,
|
|
770
|
+
manualAudioCaptured: false,
|
|
771
|
+
pendingAudioChunks: [],
|
|
772
|
+
pendingAudioBytes: 0,
|
|
773
|
+
finalTranscriptSegments: [],
|
|
774
|
+
latestPartialText: null,
|
|
775
|
+
endpointExtensionCount: 0,
|
|
776
|
+
heldSpeculativeContent: null,
|
|
777
|
+
turnId: null,
|
|
778
|
+
userMessageId: null,
|
|
779
|
+
userAudioChunks: [],
|
|
780
|
+
metricsTurnStarted: false,
|
|
781
|
+
metricsTurnFinished: false,
|
|
782
|
+
stashedMetricsMarks: {
|
|
783
|
+
firstAudioAtMs: null,
|
|
784
|
+
firstPartialAtMs: null,
|
|
785
|
+
speechStartAtMs: null,
|
|
786
|
+
utteranceEndAtMs: null,
|
|
787
|
+
finalTranscriptAtMs: null,
|
|
788
|
+
},
|
|
789
|
+
};
|
|
790
|
+
}
|
|
791
|
+
|
|
792
|
+
// Carrier cycle for an assistant-initiated turn: the turn machinery is
|
|
793
|
+
// utterance-shaped (metrics marks, audio archival, turn ids all hang off a
|
|
794
|
+
// cycle), but an announcement has no capture behind it. The record starts fully
|
|
795
|
+
// released with an empty transcript so nothing routes audio into it, and it is
|
|
796
|
+
// never installed as `currentUtterance` — that field belongs to the VAD cycle.
|
|
797
|
+
function createSyntheticUtterance(): UtteranceCycle {
|
|
798
|
+
return {
|
|
799
|
+
...createUtteranceCycle(),
|
|
800
|
+
phase: "transcriber_closed",
|
|
801
|
+
released: true,
|
|
802
|
+
};
|
|
803
|
+
}
|
|
804
|
+
|
|
560
805
|
// Objective handed to the background subagent that continues a barged-in turn.
|
|
561
806
|
// The subagent forks the live conversation, so it already sees the interrupted
|
|
562
807
|
// turn's completed tool calls in history and resumes from there. The request
|
|
@@ -565,12 +810,63 @@ function buildVoiceControlPrompt(turn: ActiveAssistantTurn): string {
|
|
|
565
810
|
// the forked history.
|
|
566
811
|
function buildDuplexContinuationObjective(interruptedRequest: string): string {
|
|
567
812
|
const base =
|
|
568
|
-
"You were in the middle of responding to the user's most recent request when they interrupted you. Finish that response now. Do not repeat any tool calls whose results are already present in the conversation. You are running unattended in the background
|
|
813
|
+
"You were in the middle of responding to the user's most recent request when they interrupted you. Finish that response now. Do not repeat any tool calls whose results are already present in the conversation. You are running unattended in the background: permission policy may auto-deny higher-risk actions with no one to approve them. If an action you need is denied, do not retry it; finish what you can and say plainly what remains, so the user can trigger it on their next turn.";
|
|
569
814
|
return interruptedRequest.length > 0
|
|
570
815
|
? `${base} Their request was: "${interruptedRequest}".`
|
|
571
816
|
: base;
|
|
572
817
|
}
|
|
573
818
|
|
|
819
|
+
// Built-ins beyond the strict read-only allowlist that cannot contend with a
|
|
820
|
+
// background continuation's writes, whatever their input: they touch no
|
|
821
|
+
// workspace, host, or extension state. `web_fetch` sits on the core
|
|
822
|
+
// SIDE_EFFECT_TOOLS list because an UNATTENDED run firing off external requests
|
|
823
|
+
// is a permission concern — a different question from this gate's, which is
|
|
824
|
+
// only "can these two writers corrupt the same local state?". A network read
|
|
825
|
+
// cannot, so it does not contend.
|
|
826
|
+
//
|
|
827
|
+
// `skill_load` is exempt per-invocation rather than by name. Re-entering an
|
|
828
|
+
// already-installed static skill is the first call of nearly every barge-in
|
|
829
|
+
// follow-up, and killing the continuation there would defeat the feature at the
|
|
830
|
+
// moment it matters most — but a load that auto-installs a missing skill or
|
|
831
|
+
// renders an inline command expansion writes local state and executes shell, so
|
|
832
|
+
// it contends like anything else.
|
|
833
|
+
const FOREGROUND_NON_CONTENDING_TOOLS: ReadonlySet<string> = new Set([
|
|
834
|
+
"web_fetch",
|
|
835
|
+
]);
|
|
836
|
+
|
|
837
|
+
// Foreground-wins classification: does this foreground tool start force the
|
|
838
|
+
// running continuations to be aborted? The question is LOCAL-STATE
|
|
839
|
+
// CONTENTION ("could these two writers corrupt the same workspace, host, or
|
|
840
|
+
// extension state?"), NOT permission — this gate never affects what a tool is
|
|
841
|
+
// allowed to do. Fail closed anyway: anything that is not a provably
|
|
842
|
+
// non-contending BUILT-IN contends, because skill/plugin/MCP/workspace tools
|
|
843
|
+
// carry no "writes local state" metadata and some of them (app_*, document_*)
|
|
844
|
+
// very much do. `skill_execute` always contends: it is a dispatcher whose
|
|
845
|
+
// resolved inner tool can mutate.
|
|
846
|
+
function foregroundToolContendsWithContinuation(
|
|
847
|
+
toolName: string,
|
|
848
|
+
input?: Record<string, unknown>,
|
|
849
|
+
): boolean {
|
|
850
|
+
if (toolName === "skill_execute") {
|
|
851
|
+
return true;
|
|
852
|
+
}
|
|
853
|
+
const ownerKind = getToolOwner(toolName)?.kind;
|
|
854
|
+
if (ownerKind === "default") {
|
|
855
|
+
if (FOREGROUND_NON_CONTENDING_TOOLS.has(toolName)) {
|
|
856
|
+
return false;
|
|
857
|
+
}
|
|
858
|
+
// Last, and behind the name checks above: `isInstalledStaticSkillLoad`
|
|
859
|
+
// re-reads the skill catalog from disk, so only a `skill_load` pays for it.
|
|
860
|
+
// Absent input fails closed — an unknown target could be an auto-install.
|
|
861
|
+
if (toolName === "skill_load") {
|
|
862
|
+
return (
|
|
863
|
+
input === undefined || !isInstalledStaticSkillLoad(toolName, input)
|
|
864
|
+
);
|
|
865
|
+
}
|
|
866
|
+
}
|
|
867
|
+
return isRefusedInReadOnlyPass(toolName, ownerKind);
|
|
868
|
+
}
|
|
869
|
+
|
|
574
870
|
// Upper bound on how long a barge-in waits for the interrupted turn's teardown
|
|
575
871
|
// to settle before giving up on the continuation. The teardown settles once the
|
|
576
872
|
// aborted turn's agent loop reaches its `finally`, which can wait out BOTH the
|
|
@@ -595,19 +891,22 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
595
891
|
private readonly archiveAudio: LiveVoiceSessionAudioArchiver | null;
|
|
596
892
|
private readonly spawnBackgroundContinuation: LiveVoiceBackgroundContinuationSpawner | null;
|
|
597
893
|
// Reads the interrupted turn's teardown promise so the barge-in path can wait
|
|
598
|
-
// for it to settle before forking the continuation
|
|
894
|
+
// for it to settle before forking the continuation.
|
|
599
895
|
private readonly getTurnTeardown:
|
|
600
896
|
| ((conversationId: string) => Promise<void> | undefined)
|
|
601
897
|
| null;
|
|
602
898
|
private readonly detachTeardownSettleTimeoutMs: number;
|
|
899
|
+
private readonly continuationAnnounceSilenceMs: number;
|
|
603
900
|
// Abort handles for background continuations started when a barge-in detached
|
|
604
|
-
// an interrupted turn
|
|
605
|
-
//
|
|
606
|
-
//
|
|
901
|
+
// an interrupted turn. Each controller is registered synchronously before
|
|
902
|
+
// its spawn, so interrupt()/close() abort a continuation even if a stop lands
|
|
903
|
+
// while it is still spawning.
|
|
607
904
|
private readonly detachControllers = new Set<AbortController>();
|
|
608
|
-
// Bumped whenever a stop (interrupt/close)
|
|
609
|
-
//
|
|
610
|
-
//
|
|
905
|
+
// Bumped whenever detached runs are invalidated: a stop (interrupt/close),
|
|
906
|
+
// a newer barge-in superseding them, or a foreground-wins abort. A barge-in
|
|
907
|
+
// captures this (after its own bump) before its async teardown; if it has
|
|
908
|
+
// changed by the time the detach would spawn, an invalidation landed during
|
|
909
|
+
// the gap and the continuation is not started.
|
|
611
910
|
private detachStopGeneration = 0;
|
|
612
911
|
// Bumped SYNCHRONOUSLY at each barge-in (in barge order), before the async
|
|
613
912
|
// detach runs. Only the latest-started detach (detachSeq === detachSequence)
|
|
@@ -665,11 +964,28 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
665
964
|
// Consumed (and cleared) when that turn launches; cleared if the barge-in
|
|
666
965
|
// utterance is discarded, so it can never attach to a later, unrelated turn.
|
|
667
966
|
private pendingInterruptedRequest: string | null = null;
|
|
668
|
-
// Set when a background continuation
|
|
669
|
-
//
|
|
670
|
-
//
|
|
671
|
-
//
|
|
967
|
+
// Set when a background continuation finishes the reply a barge-in cut off:
|
|
968
|
+
// its final answer, folded into the next turn the user starts as context.
|
|
969
|
+
// Consumed (and cleared) when that turn launches; cleared on a hard stop
|
|
970
|
+
// (abortDetachedRuns) so a stale result never surfaces later, and cleared by
|
|
971
|
+
// an announcement that delivers the same answer itself — an announcement the
|
|
972
|
+
// user cuts short hands it straight back.
|
|
672
973
|
private pendingContinuationResult: string | null = null;
|
|
974
|
+
// The same finished continuation, queued for an announcement turn: the
|
|
975
|
+
// session speaks the result on its own once the call has been quiet for
|
|
976
|
+
// `continuationAnnounceSilenceMs`. This and `pendingContinuationResult` are
|
|
977
|
+
// armed together and are two routes for ONE answer — whichever fires first
|
|
978
|
+
// clears the other, so the user hears it exactly once. An announcement whose
|
|
979
|
+
// turn fails to start, or whose turn a barge-in cuts short, hands the answer
|
|
980
|
+
// back to the stash, so a lost delivery costs the announcement, not the
|
|
981
|
+
// answer.
|
|
982
|
+
private pendingAnnouncement: ContinuationDelivery | null = null;
|
|
983
|
+
private announcementTimer: ReturnType<typeof setTimeout> | null = null;
|
|
984
|
+
// Set when a continuation actually spawns: the request it took over, so the
|
|
985
|
+
// NEXT turn can tell the user the work is still running. Consumed by that
|
|
986
|
+
// turn; cleared when the continuation finishes (by then the result note
|
|
987
|
+
// takes over and "still running" would be stale).
|
|
988
|
+
private pendingHandoffRequest: string | null = null;
|
|
673
989
|
private readonly maxPendingAudioBytes: number;
|
|
674
990
|
// Set on VAD speech onset; consumed when the first speech chunk is routed
|
|
675
991
|
// to an utterance so the metric lands on the right turn.
|
|
@@ -706,13 +1022,10 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
706
1022
|
/** The cycle whose grace timer is armed (only the newest release has one). */
|
|
707
1023
|
private finalizeGraceCycle: UtteranceCycle | null = null;
|
|
708
1024
|
private readonly finalizeGraceMs: number;
|
|
709
|
-
// Rotates through the ack phrase list across the session's turns.
|
|
710
|
-
private ackPhraseCounter = 0;
|
|
711
1025
|
// Rotates through the progress fallback phrases across the session's turns.
|
|
712
1026
|
private progressPhraseCounter = 0;
|
|
713
|
-
// Front-model service:
|
|
714
|
-
//
|
|
715
|
-
// capabilities (energy endpointing + static ack phrasing remain).
|
|
1027
|
+
// Front-model service: phrases spoken acks and progress narration; null
|
|
1028
|
+
// disables both (the turn then holds the floor with silence).
|
|
716
1029
|
private readonly frontDecider: VoiceFrontDecider | null;
|
|
717
1030
|
// Complete front-model tunables: the constructor schema-parses the partial
|
|
718
1031
|
// option once, so every field carries its `liveVoice.frontModel` schema
|
|
@@ -757,6 +1070,8 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
757
1070
|
this.detachTeardownSettleTimeoutMs =
|
|
758
1071
|
options.detachTeardownSettleTimeoutMs ??
|
|
759
1072
|
defaultDetachTeardownSettleTimeoutMs();
|
|
1073
|
+
this.continuationAnnounceSilenceMs =
|
|
1074
|
+
options.continuationAnnounceSilenceMs ?? CONTINUATION_ANNOUNCE_SILENCE_MS;
|
|
760
1075
|
this.emitMetrics = options.emitMetrics ?? false;
|
|
761
1076
|
this.createTurnId = options.createTurnId ?? randomUUID;
|
|
762
1077
|
this.conversationId =
|
|
@@ -896,8 +1211,18 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
896
1211
|
this.state = "closed";
|
|
897
1212
|
this.turnDetector?.dispose();
|
|
898
1213
|
this.clearEndpointExtensionTimer();
|
|
1214
|
+
// There is no longer anyone on the call to speak to, so a queued
|
|
1215
|
+
// announcement cannot be spoken — but the answer behind it is finished
|
|
1216
|
+
// work the user asked for, so it takes the same conversation route a
|
|
1217
|
+
// continuation finishing after this point takes. Then the queue is
|
|
1218
|
+
// cleared: the announcement is dead either way.
|
|
1219
|
+
await this.deliverPendingContinuationToConversation();
|
|
1220
|
+
this.clearContinuationAnnouncement();
|
|
899
1221
|
this.stopSessionTranscriber();
|
|
900
|
-
|
|
1222
|
+
// Detached continuations outlive the call. A deliberate `interrupt()`
|
|
1223
|
+
// aborts them; ending the session leaves them running. With no next voice
|
|
1224
|
+
// turn to fold into, a continuation that finishes after this point delivers
|
|
1225
|
+
// into the conversation instead (see detachInterruptedTurn).
|
|
901
1226
|
await this.cancelAssistantTurn("session_closed");
|
|
902
1227
|
if (shouldEmitSessionEndMetrics) {
|
|
903
1228
|
await this.emitSessionEndMetrics();
|
|
@@ -912,32 +1237,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
912
1237
|
// mode, again after every finalized turn (per-utterance phase tracks the
|
|
913
1238
|
// cycle).
|
|
914
1239
|
private async beginUtterance(): Promise<UtteranceStartResult> {
|
|
915
|
-
const utterance
|
|
916
|
-
phase: "pending",
|
|
917
|
-
released: false,
|
|
918
|
-
assistantTurnStarted: false,
|
|
919
|
-
completed: false,
|
|
920
|
-
finalizeRequested: false,
|
|
921
|
-
transcriber: null,
|
|
922
|
-
pendingAudioChunks: [],
|
|
923
|
-
pendingAudioBytes: 0,
|
|
924
|
-
finalTranscriptSegments: [],
|
|
925
|
-
latestPartialText: null,
|
|
926
|
-
endpointExtensionCount: 0,
|
|
927
|
-
heldSpeculativeContent: null,
|
|
928
|
-
turnId: null,
|
|
929
|
-
userMessageId: null,
|
|
930
|
-
userAudioChunks: [],
|
|
931
|
-
metricsTurnStarted: false,
|
|
932
|
-
metricsTurnFinished: false,
|
|
933
|
-
stashedMetricsMarks: {
|
|
934
|
-
firstAudioAtMs: null,
|
|
935
|
-
firstPartialAtMs: null,
|
|
936
|
-
speechStartAtMs: null,
|
|
937
|
-
utteranceEndAtMs: null,
|
|
938
|
-
finalTranscriptAtMs: null,
|
|
939
|
-
},
|
|
940
|
-
};
|
|
1240
|
+
const utterance = createUtteranceCycle();
|
|
941
1241
|
this.currentUtterance = utterance;
|
|
942
1242
|
// Speech parked while the previous cycle wound down belongs to this
|
|
943
1243
|
// cycle: buffer it before the transcriber arms, and capture the detector
|
|
@@ -1118,6 +1418,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1118
1418
|
return;
|
|
1119
1419
|
}
|
|
1120
1420
|
|
|
1421
|
+
// The chunk belongs to an utterance the user is still holding the button
|
|
1422
|
+
// for, whether or not the transcriber has produced any text for it yet.
|
|
1423
|
+
utterance.manualAudioCaptured = true;
|
|
1121
1424
|
this.collectUserAudio(utterance, chunk);
|
|
1122
1425
|
if (utterance.phase === "pending") {
|
|
1123
1426
|
// The transcriber is still arming (session start overlaps the STT
|
|
@@ -1415,11 +1718,32 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1415
1718
|
.trim();
|
|
1416
1719
|
this.pendingInterruptedRequest =
|
|
1417
1720
|
interruptedRequest.length > 0 ? interruptedRequest : null;
|
|
1418
|
-
// A fresh interruption supersedes
|
|
1419
|
-
//
|
|
1420
|
-
//
|
|
1421
|
-
//
|
|
1422
|
-
|
|
1721
|
+
// A fresh interruption supersedes every earlier detached run, not just its
|
|
1722
|
+
// stashed result: abort still-running continuations (and skip pending
|
|
1723
|
+
// detaches) before this barge-in's own continuation can launch, so two
|
|
1724
|
+
// full-ability background writers never share the workspace. The stashed
|
|
1725
|
+
// result is dropped with them so the barge-in follow-up (or any later)
|
|
1726
|
+
// turn can't consume an older answer before this barge-in's own
|
|
1727
|
+
// continuation completes. This barge-in's detach snapshots the stop
|
|
1728
|
+
// generation AFTER this bump (below), so it is unaffected. The abort is
|
|
1729
|
+
// signal-level, with the same accepted residual as the foreground-wins
|
|
1730
|
+
// gate (see the tool_use_start handler): a tool call already executing in
|
|
1731
|
+
// the superseded run is not awaited. The replacement's detach still waits
|
|
1732
|
+
// out the interrupted TURN's teardown before forking, which bounds the
|
|
1733
|
+
// overlap to that one abandoned call.
|
|
1734
|
+
this.abortDetachedRuns({ reason: "superseded_by_new_barge_in" });
|
|
1735
|
+
// An announcement turn is the one interruption with nothing to continue:
|
|
1736
|
+
// the work is already finished, which is why detachInterruptedTurn skips it
|
|
1737
|
+
// as `announcement_turn`. The analog of "the interrupted work survives" is
|
|
1738
|
+
// therefore the stash — the answer goes back into it so the user's next real
|
|
1739
|
+
// turn still carries it, and the announcement (cut short, at best partly
|
|
1740
|
+
// spoken) is not the delivery it was supposed to be. The restore sits after
|
|
1741
|
+
// the abort above, which clears the stash, so nothing in this sequence
|
|
1742
|
+
// re-drops it. Only a barge-in restores: interrupt/close/a superseding new
|
|
1743
|
+
// continuation are resets the user asked for, and their answer stays gone.
|
|
1744
|
+
if (turn.continuationDelivery !== null) {
|
|
1745
|
+
this.pendingContinuationResult = turn.continuationDelivery.answer;
|
|
1746
|
+
}
|
|
1423
1747
|
// Order this barge-in among concurrent detaches SYNCHRONOUSLY, in barge
|
|
1424
1748
|
// order — the actual detach runs after an async teardown chain, and those
|
|
1425
1749
|
// chains can interleave, so bumping there could assign sequences out of
|
|
@@ -1429,8 +1753,8 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1429
1753
|
this.clearFillerTimers(turn);
|
|
1430
1754
|
// Tagged reason: provider catch-sites classify untagged caller aborts as
|
|
1431
1755
|
// retryable transport failures (ERROR log + futile retry against the
|
|
1432
|
-
// aborted signal). This signal reaches the brain leg and
|
|
1433
|
-
//
|
|
1756
|
+
// aborted signal). This signal reaches the brain leg and any in-flight
|
|
1757
|
+
// ack generation.
|
|
1434
1758
|
turn.abortController.abort(
|
|
1435
1759
|
createAbortReason("voice_session_aborted", "live-voice-barge-in"),
|
|
1436
1760
|
);
|
|
@@ -1445,6 +1769,16 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1445
1769
|
// Snapshot the stop generation before the async teardown: a stop that lands
|
|
1446
1770
|
// during it must cancel the pending detach (checked in detachInterruptedTurn).
|
|
1447
1771
|
const stopGeneration = this.detachStopGeneration;
|
|
1772
|
+
log.debug(
|
|
1773
|
+
{
|
|
1774
|
+
turnId: turn.turnId,
|
|
1775
|
+
detachSeq,
|
|
1776
|
+
// The two facts that decide whether a continuation is even eligible.
|
|
1777
|
+
assistantCompleted: turn.assistantCompleted,
|
|
1778
|
+
hasTeardownWait: teardownWait !== undefined,
|
|
1779
|
+
},
|
|
1780
|
+
"Voice barge-in cancelled a turn",
|
|
1781
|
+
);
|
|
1448
1782
|
void (async () => {
|
|
1449
1783
|
await this.finishMetricsTurn(
|
|
1450
1784
|
turn.utterance,
|
|
@@ -1454,21 +1788,26 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1454
1788
|
);
|
|
1455
1789
|
await this.sendFrame({ type: "turn_cancelled", turnId: turn.turnId });
|
|
1456
1790
|
await this.cancelAssistantTurn("barge_in");
|
|
1457
|
-
// Keep the interrupted turn's work alive on a background subagent
|
|
1458
|
-
//
|
|
1459
|
-
//
|
|
1791
|
+
// Keep the interrupted turn's work alive on a background subagent; the
|
|
1792
|
+
// detach waits for its teardown to settle the partial into history before
|
|
1793
|
+
// forking.
|
|
1460
1794
|
this.detachInterruptedTurn(turn, stopGeneration, teardownWait, detachSeq);
|
|
1461
1795
|
})().catch(() => {});
|
|
1462
1796
|
}
|
|
1463
1797
|
|
|
1464
|
-
//
|
|
1465
|
-
//
|
|
1798
|
+
// Keep a barged-in turn's work alive by continuing it on a background
|
|
1799
|
+
// subagent instead of discarding it.
|
|
1466
1800
|
// Waits for the interrupted turn's bridge teardown (captured at barge-in) to
|
|
1467
1801
|
// settle before forking, so its partial — including any completed tool calls —
|
|
1468
1802
|
// is already in the conversation the subagent forks from and a side-effecting
|
|
1469
1803
|
// continuation cannot repeat a call the interrupted turn already ran.
|
|
1470
|
-
//
|
|
1471
|
-
//
|
|
1804
|
+
// The continuation runs with full subagent abilities under the standard
|
|
1805
|
+
// non-interactive permission policy; if a foreground turn starts its own
|
|
1806
|
+
// side-effecting tool, the foreground-wins abort in the tool_use_start
|
|
1807
|
+
// handler kills the continuation before the two can race on the workspace.
|
|
1808
|
+
// The run itself makes no sound; its result reaches the user through the next
|
|
1809
|
+
// turn they start, an announcement into audible silence, or the conversation
|
|
1810
|
+
// once the call ends. A later stop/interrupt aborts it.
|
|
1472
1811
|
private detachInterruptedTurn(
|
|
1473
1812
|
turn: ActiveAssistantTurn,
|
|
1474
1813
|
stopGeneration: number,
|
|
@@ -1476,18 +1815,39 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1476
1815
|
detachSeq: number,
|
|
1477
1816
|
): void {
|
|
1478
1817
|
const spawn = this.spawnBackgroundContinuation;
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1818
|
+
// Every skip is logged with its reason — the handoff is silent by design,
|
|
1819
|
+
// so without this a dropped continuation is indistinguishable from a
|
|
1820
|
+
// never-attempted one (tail with: grep -i "voice duplex").
|
|
1821
|
+
if (!spawn) {
|
|
1822
|
+
log.debug(
|
|
1823
|
+
{ turnId: turn.turnId, skipReason: "no_spawner" },
|
|
1824
|
+
"Voice duplex continuation skipped",
|
|
1825
|
+
);
|
|
1826
|
+
return;
|
|
1827
|
+
}
|
|
1828
|
+
const skipReason = this.isClosed
|
|
1829
|
+
? "session_closed"
|
|
1830
|
+
: // Barging in over an announcement means the user is answering it, not
|
|
1831
|
+
// asking for it to be finished in the background — there is no pending
|
|
1832
|
+
// request behind an announcement turn to continue. Its answer is
|
|
1833
|
+
// already finished, so bargeIn returns it to the stash instead.
|
|
1834
|
+
turn.continuationDelivery !== null
|
|
1835
|
+
? "announcement_turn"
|
|
1836
|
+
: // The model already finished generating (barge-in during TTS playback
|
|
1837
|
+
// of a complete reply): there is nothing to continue, so a
|
|
1838
|
+
// continuation would just re-do a finished answer.
|
|
1839
|
+
turn.assistantCompleted
|
|
1840
|
+
? "assistant_already_completed"
|
|
1841
|
+
: // A stop (interrupt/close) or a superseding invalidation landed
|
|
1842
|
+
// during the barge-in teardown: honor it.
|
|
1843
|
+
this.detachStopGeneration !== stopGeneration
|
|
1844
|
+
? "invalidated_during_barge_teardown"
|
|
1845
|
+
: null;
|
|
1846
|
+
if (skipReason !== null) {
|
|
1847
|
+
log.info(
|
|
1848
|
+
{ turnId: turn.turnId, skipReason },
|
|
1849
|
+
"Voice duplex continuation skipped",
|
|
1850
|
+
);
|
|
1491
1851
|
return;
|
|
1492
1852
|
}
|
|
1493
1853
|
// Embed the interrupted request in the objective so the continuation knows
|
|
@@ -1501,6 +1861,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1501
1861
|
// controller.abort(), which the spawn's signal wiring honors).
|
|
1502
1862
|
const controller = new AbortController();
|
|
1503
1863
|
this.detachControllers.add(controller);
|
|
1864
|
+
const detachStartedAtMs = Date.now();
|
|
1504
1865
|
void (async () => {
|
|
1505
1866
|
try {
|
|
1506
1867
|
// Wait for the interrupted turn's teardown to settle its partial into
|
|
@@ -1528,39 +1889,119 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1528
1889
|
// The continuation is best-effort, so a rare dropped one is the safe
|
|
1529
1890
|
// trade.
|
|
1530
1891
|
if (!settled) {
|
|
1892
|
+
log.info(
|
|
1893
|
+
{
|
|
1894
|
+
turnId: turn.turnId,
|
|
1895
|
+
skipReason: controller.signal.aborted
|
|
1896
|
+
? "invalidated_during_teardown_wait"
|
|
1897
|
+
: "teardown_settle_timeout",
|
|
1898
|
+
waitedMs: Date.now() - detachStartedAtMs,
|
|
1899
|
+
timeoutMs: this.detachTeardownSettleTimeoutMs,
|
|
1900
|
+
},
|
|
1901
|
+
"Voice duplex continuation skipped",
|
|
1902
|
+
);
|
|
1531
1903
|
return;
|
|
1532
1904
|
}
|
|
1533
1905
|
}
|
|
1534
|
-
|
|
1906
|
+
// A closed session is NOT a reason to skip: the work outlives the
|
|
1907
|
+
// call, and its result is delivered into the conversation below.
|
|
1908
|
+
// Only an explicit invalidation (stop/interrupt/supersede) stops it.
|
|
1909
|
+
if (controller.signal.aborted) {
|
|
1910
|
+
log.info(
|
|
1911
|
+
{
|
|
1912
|
+
turnId: turn.turnId,
|
|
1913
|
+
skipReason: "invalidated_before_spawn",
|
|
1914
|
+
waitedMs: Date.now() - detachStartedAtMs,
|
|
1915
|
+
},
|
|
1916
|
+
"Voice duplex continuation skipped",
|
|
1917
|
+
);
|
|
1535
1918
|
return;
|
|
1536
1919
|
}
|
|
1920
|
+
log.debug(
|
|
1921
|
+
{
|
|
1922
|
+
turnId: turn.turnId,
|
|
1923
|
+
teardownWaitMs: Date.now() - detachStartedAtMs,
|
|
1924
|
+
interruptedRequest,
|
|
1925
|
+
},
|
|
1926
|
+
"Voice duplex continuation starting",
|
|
1927
|
+
);
|
|
1928
|
+
// The next turn tells the user this is still running. Set before the
|
|
1929
|
+
// await so a follow-up turn launching during the run picks it up.
|
|
1930
|
+
this.pendingHandoffRequest =
|
|
1931
|
+
interruptedRequest.length > 0 ? interruptedRequest : null;
|
|
1537
1932
|
const resultText = await spawn({
|
|
1538
1933
|
parentConversationId: this.conversationId,
|
|
1539
1934
|
objective: buildDuplexContinuationObjective(interruptedRequest),
|
|
1540
1935
|
label: `voice-continue-${turn.turnId}`,
|
|
1541
1936
|
signal: controller.signal,
|
|
1542
1937
|
});
|
|
1543
|
-
//
|
|
1544
|
-
//
|
|
1545
|
-
//
|
|
1546
|
-
//
|
|
1547
|
-
//
|
|
1548
|
-
//
|
|
1549
|
-
|
|
1550
|
-
|
|
1938
|
+
// Route the completed continuation's answer. Re-check the stop guards
|
|
1939
|
+
// after the await: an interrupt/close during the run must suppress it.
|
|
1940
|
+
// Only the latest-started detach may populate the result, so once a
|
|
1941
|
+
// newer barge-in has started, an older continuation completing (before
|
|
1942
|
+
// or after it, empty or not) can't surface a stale answer. Only
|
|
1943
|
+
// non-empty text is actually surfaced.
|
|
1944
|
+
this.pendingHandoffRequest = null;
|
|
1945
|
+
const answer = resultText.trim();
|
|
1946
|
+
const notInvalidated =
|
|
1551
1947
|
!controller.signal.aborted &&
|
|
1552
|
-
!this.isClosed &&
|
|
1553
1948
|
this.detachStopGeneration === stopGeneration &&
|
|
1554
|
-
detachSeq === this.detachSequence
|
|
1555
|
-
|
|
1556
|
-
|
|
1949
|
+
detachSeq === this.detachSequence;
|
|
1950
|
+
// Two destinations, decided by whether a next voice turn still exists.
|
|
1951
|
+
// Live session: stash for the next turn the user starts, AND queue an
|
|
1952
|
+
// announcement that speaks it into audible silence if the user says
|
|
1953
|
+
// nothing — the two are routes for one answer and cancel each other,
|
|
1954
|
+
// so it is delivered exactly once. Session closed: there is no next
|
|
1955
|
+
// turn and nobody to speak to, so deliver into the conversation — the
|
|
1956
|
+
// thread is exactly where the user goes looking for the work.
|
|
1957
|
+
const deliverToConversation =
|
|
1958
|
+
notInvalidated && this.isClosed && answer.length > 0;
|
|
1959
|
+
if (notInvalidated && !this.isClosed) {
|
|
1557
1960
|
this.pendingContinuationResult = answer.length > 0 ? answer : null;
|
|
1961
|
+
if (answer.length > 0) {
|
|
1962
|
+
this.pendingAnnouncement = { request: interruptedRequest, answer };
|
|
1963
|
+
this.scheduleContinuationAnnouncement();
|
|
1964
|
+
}
|
|
1558
1965
|
}
|
|
1966
|
+
if (deliverToConversation) {
|
|
1967
|
+
const { injectMessageIntoParent } =
|
|
1968
|
+
await import("../subagent/notify.js");
|
|
1969
|
+
injectMessageIntoParent(
|
|
1970
|
+
this.conversationId,
|
|
1971
|
+
buildClosedSessionDeliveryPrompt(interruptedRequest, answer),
|
|
1972
|
+
);
|
|
1973
|
+
}
|
|
1974
|
+
log.info(
|
|
1975
|
+
{
|
|
1976
|
+
turnId: turn.turnId,
|
|
1977
|
+
ranMs: Date.now() - detachStartedAtMs,
|
|
1978
|
+
resultChars: answer.length,
|
|
1979
|
+
// Where the answer went. "stashed" = queued for the next voice
|
|
1980
|
+
// turn and for an announcement into silence, whichever comes first
|
|
1981
|
+
// (the announcement logs "announced" when it wins);
|
|
1982
|
+
// "conversation" = the call had ended, so it was delivered into the
|
|
1983
|
+
// thread; "dropped" = a stop/interrupt or a newer barge-in landed
|
|
1984
|
+
// while it ran.
|
|
1985
|
+
resultDestination: !notInvalidated
|
|
1986
|
+
? "dropped"
|
|
1987
|
+
: answer.length === 0
|
|
1988
|
+
? "empty"
|
|
1989
|
+
: this.isClosed
|
|
1990
|
+
? "conversation"
|
|
1991
|
+
: "stashed",
|
|
1992
|
+
},
|
|
1993
|
+
"Voice duplex continuation finished",
|
|
1994
|
+
);
|
|
1559
1995
|
} catch (err) {
|
|
1560
1996
|
// A stop/interrupt aborts via the signal; that rejection is expected.
|
|
1561
|
-
if (
|
|
1997
|
+
if (controller.signal.aborted) {
|
|
1998
|
+
log.debug(
|
|
1999
|
+
{ turnId: turn.turnId, ranMs: Date.now() - detachStartedAtMs },
|
|
2000
|
+
"Voice duplex continuation aborted mid-run",
|
|
2001
|
+
);
|
|
2002
|
+
} else {
|
|
1562
2003
|
log.warn(
|
|
1563
|
-
{ err, turnId: turn.turnId },
|
|
2004
|
+
{ err, turnId: turn.turnId, ranMs: Date.now() - detachStartedAtMs },
|
|
1564
2005
|
"Voice duplex handoff continuation failed",
|
|
1565
2006
|
);
|
|
1566
2007
|
}
|
|
@@ -1571,9 +2012,37 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1571
2012
|
}
|
|
1572
2013
|
|
|
1573
2014
|
// Abort every background continuation this session started and drop its
|
|
1574
|
-
// handle.
|
|
1575
|
-
//
|
|
1576
|
-
|
|
2015
|
+
// handle. Called on a hard stop (client interrupt / session close), when a
|
|
2016
|
+
// newer barge-in supersedes the detached runs, and on a foreground-wins
|
|
2017
|
+
// abort; the continuation's own `.finally` removes it from the set too.
|
|
2018
|
+
// `keepPendingResult` is the foreground-wins variant: the foreground turn is
|
|
2019
|
+
// claiming the workspace, so running continuations must die (and pending
|
|
2020
|
+
// detaches must be skipped), but an already-completed continuation's stashed
|
|
2021
|
+
// answer stays — it cannot race anything, and the next turn's "use only if
|
|
2022
|
+
// relevant" framing makes a stale one harmless.
|
|
2023
|
+
private abortDetachedRuns(opts?: {
|
|
2024
|
+
keepPendingResult?: boolean;
|
|
2025
|
+
// Why the runs are being invalidated, for the log line below. Every
|
|
2026
|
+
// caller passes one: an unexplained dead continuation is the single
|
|
2027
|
+
// hardest thing to debug about this feature.
|
|
2028
|
+
reason?: string;
|
|
2029
|
+
// The foreground tool whose start tripped the contention gate, if any.
|
|
2030
|
+
toolName?: string;
|
|
2031
|
+
}): void {
|
|
2032
|
+
const aborted = this.detachControllers.size;
|
|
2033
|
+
const hadPendingResult = this.pendingContinuationResult !== null;
|
|
2034
|
+
if (aborted > 0 || hadPendingResult) {
|
|
2035
|
+
log.debug(
|
|
2036
|
+
{
|
|
2037
|
+
conversationId: this.conversationId,
|
|
2038
|
+
reason: opts?.reason ?? "unspecified",
|
|
2039
|
+
...(opts?.toolName ? { toolName: opts.toolName } : {}),
|
|
2040
|
+
abortedRuns: aborted,
|
|
2041
|
+
droppedPendingResult: hadPendingResult && !opts?.keepPendingResult,
|
|
2042
|
+
},
|
|
2043
|
+
"Voice duplex continuations invalidated",
|
|
2044
|
+
);
|
|
2045
|
+
}
|
|
1577
2046
|
// Bump the generation so a barge-in whose async teardown is still in flight
|
|
1578
2047
|
// (its detach not yet spawned) sees the stop and skips the continuation.
|
|
1579
2048
|
this.detachStopGeneration += 1;
|
|
@@ -1583,7 +2052,266 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1583
2052
|
this.detachControllers.clear();
|
|
1584
2053
|
// A hard stop also drops any completed continuation's result still waiting
|
|
1585
2054
|
// to fold into the next turn, so it can't surface after the user reset.
|
|
2055
|
+
if (!opts?.keepPendingResult) {
|
|
2056
|
+
this.pendingContinuationResult = null;
|
|
2057
|
+
}
|
|
2058
|
+
// The announcement dies either way: even the foreground-wins variant (which
|
|
2059
|
+
// keeps the stash) has a foreground turn claiming the floor, so speaking up
|
|
2060
|
+
// on our own is exactly what must not happen.
|
|
2061
|
+
this.clearContinuationAnnouncement();
|
|
2062
|
+
}
|
|
2063
|
+
|
|
2064
|
+
private clearContinuationAnnouncement(): void {
|
|
2065
|
+
this.pendingAnnouncement = null;
|
|
2066
|
+
if (this.announcementTimer) {
|
|
2067
|
+
clearTimeout(this.announcementTimer);
|
|
2068
|
+
this.announcementTimer = null;
|
|
2069
|
+
}
|
|
2070
|
+
}
|
|
2071
|
+
|
|
2072
|
+
/**
|
|
2073
|
+
* Hang-up delivery for a continuation that already finished: the call is
|
|
2074
|
+
* ending, so neither route that was armed for its answer can still run — the
|
|
2075
|
+
* announcement has nobody to speak to and the stash has no next voice turn to
|
|
2076
|
+
* fold into. Deliver it into the conversation instead, exactly as a
|
|
2077
|
+
* continuation finishing after the close does, because that thread is where
|
|
2078
|
+
* the user goes looking for the work ("barge in, hear the answer, close the
|
|
2079
|
+
* room, go look for the result").
|
|
2080
|
+
*
|
|
2081
|
+
* Best-effort and consume-once: the fields are emptied whether or not the
|
|
2082
|
+
* injection lands, so a close cannot deliver twice.
|
|
2083
|
+
*/
|
|
2084
|
+
private async deliverPendingContinuationToConversation(): Promise<void> {
|
|
2085
|
+
const activeTurn = this.activeAssistantTurn;
|
|
2086
|
+
const activeAnnouncement = activeTurn?.continuationDelivery ?? null;
|
|
2087
|
+
const announcement = this.pendingAnnouncement ?? activeAnnouncement;
|
|
2088
|
+
const answer = announcement?.answer ?? this.pendingContinuationResult;
|
|
2089
|
+
const request = announcement?.request ?? "";
|
|
2090
|
+
this.pendingAnnouncement = null;
|
|
2091
|
+
this.pendingContinuationResult = null;
|
|
2092
|
+
if (activeTurn !== null) {
|
|
2093
|
+
activeTurn.continuationDelivery = null;
|
|
2094
|
+
}
|
|
2095
|
+
if (answer === null || answer.length === 0) {
|
|
2096
|
+
return;
|
|
2097
|
+
}
|
|
2098
|
+
try {
|
|
2099
|
+
const { injectMessageIntoParent } = await import("../subagent/notify.js");
|
|
2100
|
+
injectMessageIntoParent(
|
|
2101
|
+
this.conversationId,
|
|
2102
|
+
buildClosedSessionDeliveryPrompt(request, answer),
|
|
2103
|
+
);
|
|
2104
|
+
log.info(
|
|
2105
|
+
{
|
|
2106
|
+
conversationId: this.conversationId,
|
|
2107
|
+
resultChars: answer.length,
|
|
2108
|
+
resultDestination: "conversation",
|
|
2109
|
+
},
|
|
2110
|
+
"Voice duplex continuation delivered on session close",
|
|
2111
|
+
);
|
|
2112
|
+
} catch (err) {
|
|
2113
|
+
log.warn(
|
|
2114
|
+
{ err, conversationId: this.conversationId },
|
|
2115
|
+
"Voice duplex continuation delivery on session close failed",
|
|
2116
|
+
);
|
|
2117
|
+
}
|
|
2118
|
+
}
|
|
2119
|
+
|
|
2120
|
+
/**
|
|
2121
|
+
* Live-voice is non-interactive (JARVIS-1291) and the session never starts a
|
|
2122
|
+
* turn on its own — with one narrow exception, this: a background
|
|
2123
|
+
* continuation the user asked for has finished, the call is still live, and
|
|
2124
|
+
* nobody is speaking. Arms the silence timer that decides whether that
|
|
2125
|
+
* exception applies. Re-arming resets the wait, so a run of arms collapses
|
|
2126
|
+
* into one announcement.
|
|
2127
|
+
*
|
|
2128
|
+
* The silence the timer waits out starts when the audio the client already
|
|
2129
|
+
* has queued runs out, so a continuation finishing at the tail end of a long
|
|
2130
|
+
* spoken reply waits for that reply to actually finish playing.
|
|
2131
|
+
*
|
|
2132
|
+
* `retry` marks the single re-arm a user-speech blocker gets. An active
|
|
2133
|
+
* assistant turn keeps the announcement queued without polling; releasing
|
|
2134
|
+
* the turn re-arms the timer against the actual playback tail.
|
|
2135
|
+
*/
|
|
2136
|
+
private scheduleContinuationAnnouncement(
|
|
2137
|
+
retry = false,
|
|
2138
|
+
drainRearms = 0,
|
|
2139
|
+
): void {
|
|
2140
|
+
if (this.announcementTimer) {
|
|
2141
|
+
clearTimeout(this.announcementTimer);
|
|
2142
|
+
}
|
|
2143
|
+
// Queued client-side audio is the previous reply still being audible, so
|
|
2144
|
+
// the silence window only begins once it has drained.
|
|
2145
|
+
const drainMs = Math.max(0, this.assistantPlaybackTailUntilMs - Date.now());
|
|
2146
|
+
this.announcementTimer = setTimeout(() => {
|
|
2147
|
+
this.announcementTimer = null;
|
|
2148
|
+
const blockedBy = this.continuationAnnouncementBlocker();
|
|
2149
|
+
if (blockedBy === null) {
|
|
2150
|
+
void this.announceContinuation().catch((err: unknown) => {
|
|
2151
|
+
log.warn({ err }, "Voice duplex continuation announcement failed");
|
|
2152
|
+
});
|
|
2153
|
+
return;
|
|
2154
|
+
}
|
|
2155
|
+
log.debug(
|
|
2156
|
+
{ conversationId: this.conversationId, blockedBy, retry, drainRearms },
|
|
2157
|
+
"Voice duplex continuation announcement deferred",
|
|
2158
|
+
);
|
|
2159
|
+
// A tail that grew while the timer ran (the reply was still streaming
|
|
2160
|
+
// TTS when it was armed) is a wait of known length rather than a busy
|
|
2161
|
+
// call, so it re-arms against the new deadline without spending the
|
|
2162
|
+
// single retry the busy-call blockers get — otherwise a long reply's
|
|
2163
|
+
// playback alone outlasts the budget and the announcement is dropped
|
|
2164
|
+
// even though the call goes idle right after. The tail only extends
|
|
2165
|
+
// while a turn is active, which blocks earlier, so the re-arms converge;
|
|
2166
|
+
// the cap bounds the interleaving where a fresh turn's audio lands
|
|
2167
|
+
// between two checks.
|
|
2168
|
+
if (
|
|
2169
|
+
blockedBy === "playback_draining" &&
|
|
2170
|
+
drainRearms < CONTINUATION_ANNOUNCE_MAX_DRAIN_REARMS
|
|
2171
|
+
) {
|
|
2172
|
+
this.scheduleContinuationAnnouncement(retry, drainRearms + 1);
|
|
2173
|
+
return;
|
|
2174
|
+
}
|
|
2175
|
+
if (blockedBy === "turn_active") {
|
|
2176
|
+
return;
|
|
2177
|
+
}
|
|
2178
|
+
if (retry || this.pendingAnnouncement === null) {
|
|
2179
|
+
this.pendingAnnouncement = null;
|
|
2180
|
+
return;
|
|
2181
|
+
}
|
|
2182
|
+
this.scheduleContinuationAnnouncement(true, drainRearms);
|
|
2183
|
+
}, drainMs + this.continuationAnnounceSilenceMs);
|
|
2184
|
+
}
|
|
2185
|
+
|
|
2186
|
+
private clearActiveAssistantTurn(token: symbol): void {
|
|
2187
|
+
if (this.activeAssistantTurn?.token !== token) {
|
|
2188
|
+
return;
|
|
2189
|
+
}
|
|
2190
|
+
this.activeAssistantTurn = null;
|
|
2191
|
+
if (
|
|
2192
|
+
this.pendingAnnouncement !== null &&
|
|
2193
|
+
!this.isClosed &&
|
|
2194
|
+
this.state !== "failed"
|
|
2195
|
+
) {
|
|
2196
|
+
this.scheduleContinuationAnnouncement();
|
|
2197
|
+
}
|
|
2198
|
+
}
|
|
2199
|
+
|
|
2200
|
+
// Why the session must not speak up right now, or null when the call is
|
|
2201
|
+
// genuinely idle: live, no turn running, and no user speech anywhere in
|
|
2202
|
+
// flight (onset latch, barge-in guard, detector, or a cycle that has already
|
|
2203
|
+
// captured something). The server-VAD re-arm keeps an empty cycle armed
|
|
2204
|
+
// between turns, so an untouched `currentUtterance` IS the idle state.
|
|
2205
|
+
private continuationAnnouncementBlocker(): string | null {
|
|
2206
|
+
if (this.pendingAnnouncement === null) {
|
|
2207
|
+
return "nothing_pending";
|
|
2208
|
+
}
|
|
2209
|
+
if (this.isClosed || this.state === "failed" || !this.startVoiceTurn) {
|
|
2210
|
+
return "session_unavailable";
|
|
2211
|
+
}
|
|
2212
|
+
if (this.activeAssistantTurn !== null) {
|
|
2213
|
+
return "turn_active";
|
|
2214
|
+
}
|
|
2215
|
+
// The turn can be cleared server-side (tts_done sent) while the client is
|
|
2216
|
+
// still playing the audio it queued: that tail is the previous reply still
|
|
2217
|
+
// being audible, so announcing over it is announcing over the assistant.
|
|
2218
|
+
if (Date.now() < this.assistantPlaybackTailUntilMs) {
|
|
2219
|
+
return "playback_draining";
|
|
2220
|
+
}
|
|
2221
|
+
if (this.vadSpeechStartPending || this.pendingBargeIn !== null) {
|
|
2222
|
+
return "speech_onset";
|
|
2223
|
+
}
|
|
2224
|
+
if (this.turnDetector?.isActive === true) {
|
|
2225
|
+
return "user_speaking";
|
|
2226
|
+
}
|
|
2227
|
+
const utterance = this.currentUtterance;
|
|
2228
|
+
// Every transcript-derived signal here trails the STT provider, so on its
|
|
2229
|
+
// own it leaves manual mode blind to a user who is talking right now and
|
|
2230
|
+
// has no text yet — hence the captured-audio flag, which manual ingress
|
|
2231
|
+
// sets from the first chunk.
|
|
2232
|
+
if (
|
|
2233
|
+
utterance !== null &&
|
|
2234
|
+
!utterance.completed &&
|
|
2235
|
+
(utterance.released ||
|
|
2236
|
+
utterance.assistantTurnStarted ||
|
|
2237
|
+
utterance.manualAudioCaptured ||
|
|
2238
|
+
utterance.finalTranscriptSegments.length > 0 ||
|
|
2239
|
+
utterance.latestPartialText !== null)
|
|
2240
|
+
) {
|
|
2241
|
+
return "utterance_in_flight";
|
|
2242
|
+
}
|
|
2243
|
+
return null;
|
|
2244
|
+
}
|
|
2245
|
+
|
|
2246
|
+
// Speak the finished continuation's result on a turn the session starts
|
|
2247
|
+
// itself. The turn rides a synthetic cycle rather than the armed VAD one —
|
|
2248
|
+
// `currentUtterance` belongs to the capture loop, and overwriting it would
|
|
2249
|
+
// collide with the user's next real utterance.
|
|
2250
|
+
private async announceContinuation(): Promise<void> {
|
|
2251
|
+
const pending = this.pendingAnnouncement;
|
|
2252
|
+
if (!pending) {
|
|
2253
|
+
return;
|
|
2254
|
+
}
|
|
2255
|
+
this.pendingAnnouncement = null;
|
|
2256
|
+
// The two routes for one answer are mutually exclusive: while this turn is
|
|
2257
|
+
// in flight the stash must not deliver the same answer again. It comes back
|
|
2258
|
+
// only if this turn provably never started (below) or a barge-in cuts it
|
|
2259
|
+
// short before it delivers (see bargeIn).
|
|
1586
2260
|
this.pendingContinuationResult = null;
|
|
2261
|
+
// Pins the invalidation state this hand-off was decided in: a hard stop
|
|
2262
|
+
// landing while the turn starts owns the drop, and must not be undone by
|
|
2263
|
+
// the restore below.
|
|
2264
|
+
const stopGeneration = this.detachStopGeneration;
|
|
2265
|
+
let started: boolean;
|
|
2266
|
+
try {
|
|
2267
|
+
started = await this.launchAssistantTurn(
|
|
2268
|
+
createSyntheticUtterance(),
|
|
2269
|
+
CONTINUATION_DELIVERY_CONTENT,
|
|
2270
|
+
{ continuationDelivery: pending },
|
|
2271
|
+
);
|
|
2272
|
+
} catch (err) {
|
|
2273
|
+
// A throw before the leg's own error handling (an outbound frame write
|
|
2274
|
+
// rejecting, say) leaves the same state a false return does: nothing
|
|
2275
|
+
// persisted, nothing spoken, and the stash already emptied above. Take
|
|
2276
|
+
// the identical fallback rather than letting the answer die with the
|
|
2277
|
+
// rejection.
|
|
2278
|
+
started = false;
|
|
2279
|
+
log.warn(
|
|
2280
|
+
{ err, conversationId: this.conversationId },
|
|
2281
|
+
"Voice duplex continuation announcement threw while starting",
|
|
2282
|
+
);
|
|
2283
|
+
}
|
|
2284
|
+
if (started) {
|
|
2285
|
+
log.info(
|
|
2286
|
+
{
|
|
2287
|
+
conversationId: this.conversationId,
|
|
2288
|
+
resultChars: pending.answer.length,
|
|
2289
|
+
resultDestination: "announced",
|
|
2290
|
+
},
|
|
2291
|
+
"Voice duplex continuation announced",
|
|
2292
|
+
);
|
|
2293
|
+
return;
|
|
2294
|
+
}
|
|
2295
|
+
// The turn never reached the bridge (the conversation is busy from another
|
|
2296
|
+
// surface, or the bridge threw), so nothing was persisted or spoken. Hand
|
|
2297
|
+
// the answer back to the stash — the user's next turn is the fallback route
|
|
2298
|
+
// that exists for exactly this. A newer continuation's answer already in
|
|
2299
|
+
// the stash wins over this older one rather than being overwritten.
|
|
2300
|
+
const restashed =
|
|
2301
|
+
!this.isClosed &&
|
|
2302
|
+
this.detachStopGeneration === stopGeneration &&
|
|
2303
|
+
this.pendingContinuationResult === null;
|
|
2304
|
+
if (restashed) {
|
|
2305
|
+
this.pendingContinuationResult = pending.answer;
|
|
2306
|
+
}
|
|
2307
|
+
log.warn(
|
|
2308
|
+
{
|
|
2309
|
+
conversationId: this.conversationId,
|
|
2310
|
+
resultChars: pending.answer.length,
|
|
2311
|
+
resultDestination: restashed ? "stashed" : "dropped",
|
|
2312
|
+
},
|
|
2313
|
+
"Voice duplex continuation announcement could not start",
|
|
2314
|
+
);
|
|
1587
2315
|
}
|
|
1588
2316
|
|
|
1589
2317
|
// VAD closed the utterance — the analog of ptt_release: emit
|
|
@@ -1618,30 +2346,18 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1618
2346
|
}
|
|
1619
2347
|
return;
|
|
1620
2348
|
}
|
|
1621
|
-
// Semantic endpointing: a silence boundary may be a thinking pause
|
|
1622
|
-
//
|
|
2349
|
+
// Semantic endpointing: a silence boundary may be a thinking pause, so
|
|
2350
|
+
// the unified front-door leg itself is the endpoint decision. When it
|
|
2351
|
+
// launches, the boundary is deferred to the verdict — commit releases
|
|
2352
|
+
// the utterance, hold replays the boundary via the extension timer.
|
|
2353
|
+
// When speculation is inapplicable (no text, cap reached, a turn
|
|
2354
|
+
// already active) fall through and release. Only detector-timer
|
|
1623
2355
|
// silences qualify; max-duration always releases, and a manual client
|
|
1624
2356
|
// release (ptt_release forced the boundary) is the user saying "answer
|
|
1625
|
-
// now" — never second-guess it.
|
|
1626
|
-
// the decider is not consulted, so that release path keeps its exact
|
|
1627
|
-
// event ordering — an extra await here would shift utterance release
|
|
1628
|
-
// across the persistent-transcriber flush attribution.
|
|
2357
|
+
// now" — never second-guess it.
|
|
1629
2358
|
if (reason === "silence" && !manualRelease) {
|
|
1630
|
-
if (this.
|
|
1631
|
-
|
|
1632
|
-
// When it launches, the boundary is deferred to the verdict —
|
|
1633
|
-
// commit releases the utterance, hold replays the boundary via
|
|
1634
|
-
// the extension timer. When speculation is inapplicable (no
|
|
1635
|
-
// text, cap reached, a turn already active) fall through and
|
|
1636
|
-
// release exactly as before.
|
|
1637
|
-
if (await this.launchSpeculativeAssistantTurn(utterance)) {
|
|
1638
|
-
return;
|
|
1639
|
-
}
|
|
1640
|
-
} else {
|
|
1641
|
-
const holdDecision = this.maybeHoldUtteranceOpen(utterance);
|
|
1642
|
-
if (holdDecision && (await holdDecision)) {
|
|
1643
|
-
return;
|
|
1644
|
-
}
|
|
2359
|
+
if (await this.launchSpeculativeAssistantTurn(utterance)) {
|
|
2360
|
+
return;
|
|
1645
2361
|
}
|
|
1646
2362
|
}
|
|
1647
2363
|
await this.sendFrame({ type: "utterance_end", reason });
|
|
@@ -1649,96 +2365,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1649
2365
|
})().catch(() => {});
|
|
1650
2366
|
}
|
|
1651
2367
|
|
|
1652
|
-
/**
|
|
1653
|
-
* Semantic-endpointing gate for a silence-fired turn-end. Returns `null`
|
|
1654
|
-
* synchronously when the decider is not consulted (no decider,
|
|
1655
|
-
* extension cap reached, nothing transcribed) — the boundary then releases
|
|
1656
|
-
* exactly as before. Otherwise resolves `true` when the boundary must NOT
|
|
1657
|
-
* release the utterance now: either the decider held it open (the extension
|
|
1658
|
-
* timer replays the boundary), or the world moved on during the decision
|
|
1659
|
-
* (speech resumed, utterance superseded) and a fresh boundary owns the
|
|
1660
|
-
* release. Every decider failure resolves as "release" within
|
|
1661
|
-
* `endpointDecisionTimeoutMs` (fail-open), so consulting it adds at most
|
|
1662
|
-
* that much end-of-turn latency.
|
|
1663
|
-
*/
|
|
1664
|
-
private maybeHoldUtteranceOpen(
|
|
1665
|
-
utterance: UtteranceCycle,
|
|
1666
|
-
): Promise<boolean> | null {
|
|
1667
|
-
const decider = this.frontDecider;
|
|
1668
|
-
if (!decider) {
|
|
1669
|
-
return null;
|
|
1670
|
-
}
|
|
1671
|
-
if (
|
|
1672
|
-
utterance.endpointExtensionCount >=
|
|
1673
|
-
this.frontModelConfig.endpointMaxExtensions
|
|
1674
|
-
) {
|
|
1675
|
-
return null;
|
|
1676
|
-
}
|
|
1677
|
-
const transcriptSoFar = utterance.finalTranscriptSegments.join(" ").trim();
|
|
1678
|
-
const latestPartial = utterance.latestPartialText;
|
|
1679
|
-
if (transcriptSoFar.length === 0 && !latestPartial) {
|
|
1680
|
-
// Nothing transcribed yet: there is no text to judge.
|
|
1681
|
-
return null;
|
|
1682
|
-
}
|
|
1683
|
-
return this.decideUtteranceHold(
|
|
1684
|
-
decider,
|
|
1685
|
-
utterance,
|
|
1686
|
-
transcriptSoFar,
|
|
1687
|
-
latestPartial,
|
|
1688
|
-
);
|
|
1689
|
-
}
|
|
1690
|
-
|
|
1691
|
-
private async decideUtteranceHold(
|
|
1692
|
-
decider: VoiceFrontDecider,
|
|
1693
|
-
utterance: UtteranceCycle,
|
|
1694
|
-
transcriptSoFar: string,
|
|
1695
|
-
latestPartial: string | null,
|
|
1696
|
-
): Promise<boolean> {
|
|
1697
|
-
const generation = this.vadSpeechGeneration;
|
|
1698
|
-
const decisionStartedAtMs = this.metricsClock();
|
|
1699
|
-
const decision = await decider
|
|
1700
|
-
.decideEndpoint({
|
|
1701
|
-
transcriptSoFar,
|
|
1702
|
-
latestPartial,
|
|
1703
|
-
silenceThresholdMs: this.silenceThresholdMs,
|
|
1704
|
-
extensionCount: utterance.endpointExtensionCount,
|
|
1705
|
-
})
|
|
1706
|
-
// The decider contract never rejects, but the enclosing boundary flow
|
|
1707
|
-
// swallows rejections — a rejecting decider (a buggy stub, a future
|
|
1708
|
-
// regression) would silently drop the silence boundary. Belt and
|
|
1709
|
-
// braces: rejection degrades to release, mirroring speakGeneratedAck.
|
|
1710
|
-
.catch(() => ({ action: "release" as const }));
|
|
1711
|
-
const decisionLatencyMs = Math.max(
|
|
1712
|
-
0,
|
|
1713
|
-
this.metricsClock() - decisionStartedAtMs,
|
|
1714
|
-
);
|
|
1715
|
-
// Re-check the world after the await: the decision window is bounded but
|
|
1716
|
-
// real, and a release based on a stale snapshot could cut off speech.
|
|
1717
|
-
if (
|
|
1718
|
-
this.isUtteranceStale(utterance) ||
|
|
1719
|
-
utterance.released ||
|
|
1720
|
-
utterance.completed
|
|
1721
|
-
) {
|
|
1722
|
-
return true;
|
|
1723
|
-
}
|
|
1724
|
-
if (generation !== this.vadSpeechGeneration) {
|
|
1725
|
-
// Speech resumed during the decision: the utterance keeps accumulating
|
|
1726
|
-
// and the detector's next turn-end re-runs the whole decision. The
|
|
1727
|
-
// discarded decision is not recorded — only acted-on outcomes count.
|
|
1728
|
-
return true;
|
|
1729
|
-
}
|
|
1730
|
-
this.markEndpointDecision(utterance, decision.action, decisionLatencyMs);
|
|
1731
|
-
if (decision.action !== "hold") {
|
|
1732
|
-
return false;
|
|
1733
|
-
}
|
|
1734
|
-
utterance.endpointExtensionCount += 1;
|
|
1735
|
-
this.armEndpointExtensionTimer(utterance);
|
|
1736
|
-
return true;
|
|
1737
|
-
}
|
|
1738
|
-
|
|
1739
2368
|
// Arms the hold-extension replay: after `endpointExtensionMs` of continued
|
|
1740
|
-
// silence the deferred silence boundary re-fires (and
|
|
1741
|
-
//
|
|
2369
|
+
// silence the deferred silence boundary re-fires (and a fresh front-door
|
|
2370
|
+
// leg judges it again, bounded by `endpointMaxExtensions`).
|
|
1742
2371
|
private armEndpointExtensionTimer(utterance: UtteranceCycle): void {
|
|
1743
2372
|
this.clearEndpointExtensionTimer();
|
|
1744
2373
|
this.endpointExtensionTimer = setTimeout(() => {
|
|
@@ -1757,18 +2386,6 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1757
2386
|
}, this.frontModelConfig.endpointExtensionMs);
|
|
1758
2387
|
}
|
|
1759
2388
|
|
|
1760
|
-
/**
|
|
1761
|
-
* Whether front-door routing is active: the single `voice-mode` flag.
|
|
1762
|
-
* Governs both halves of the unified front door — the speculative leg that
|
|
1763
|
-
* decides the endpoint, and the escalation hand-off — so the two can never
|
|
1764
|
-
* disagree. The decider-based hold path (`decideEndpoint`) runs as the
|
|
1765
|
-
* flag-off fallback, so endpointing stays model-decided either way.
|
|
1766
|
-
*/
|
|
1767
|
-
private frontDoorRoutingActive(config?: AssistantConfig): boolean {
|
|
1768
|
-
const cfg = config ?? getConfig();
|
|
1769
|
-
return isAssistantFeatureFlagEnabled("voice-mode", cfg);
|
|
1770
|
-
}
|
|
1771
|
-
|
|
1772
2389
|
/**
|
|
1773
2390
|
* Unified front-door: dispatch the assistant turn speculatively at the
|
|
1774
2391
|
* silence boundary, judging the transcript accumulated so far — the same
|
|
@@ -1903,15 +2520,20 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1903
2520
|
/**
|
|
1904
2521
|
* Abort a speculative leg and roll back everything it touched: the
|
|
1905
2522
|
* persisted user message (via the handle's discard), the active-turn
|
|
1906
|
-
* slot, and the utterance's turn-started
|
|
1907
|
-
* re-dispatched (hold replay) or keep
|
|
1908
|
-
* Nothing was ever user-visible — no frames
|
|
2523
|
+
* slot, the pending context it consumed, and the utterance's turn-started
|
|
2524
|
+
* latch, so the utterance can be re-dispatched (hold replay) or keep
|
|
2525
|
+
* accumulating (speech resumed). Nothing was ever user-visible — no frames
|
|
2526
|
+
* were sent for this turn.
|
|
1909
2527
|
*/
|
|
1910
2528
|
private discardSpeculativeTurn(
|
|
1911
2529
|
turn: ActiveAssistantTurn,
|
|
1912
2530
|
reason: string,
|
|
1913
2531
|
): void {
|
|
1914
2532
|
turn.speculativePending = false;
|
|
2533
|
+
// The dispatch is being unwound, so the barge-in merge note, the finished
|
|
2534
|
+
// continuation's answer, and its queued announcement all go back to the
|
|
2535
|
+
// session — the turn that would have delivered them is gone.
|
|
2536
|
+
this.restorePendingTurnContext(turn);
|
|
1915
2537
|
// Latched before the handle check: when the discard beats the bridge
|
|
1916
2538
|
// handle's resolution (startVoiceTurn still persisting), the handle's
|
|
1917
2539
|
// arrival in startAssistantLeg completes the rollback via discard().
|
|
@@ -2414,7 +3036,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2414
3036
|
// without ever reaching finalizePendingUtterance).
|
|
2415
3037
|
this.pendingInterruptedRequest = null;
|
|
2416
3038
|
// ...and it hard-stops any detached background continuations.
|
|
2417
|
-
this.abortDetachedRuns();
|
|
3039
|
+
this.abortDetachedRuns({ reason: "client_interrupt" });
|
|
2418
3040
|
const utterance = this.currentUtterance;
|
|
2419
3041
|
this.stopSessionTranscriber();
|
|
2420
3042
|
if (utterance) {
|
|
@@ -2514,38 +3136,116 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2514
3136
|
return;
|
|
2515
3137
|
}
|
|
2516
3138
|
|
|
2517
|
-
|
|
2518
|
-
|
|
2519
|
-
|
|
3139
|
+
await this.launchAssistantTurn(utterance, content);
|
|
3140
|
+
}
|
|
3141
|
+
|
|
3142
|
+
/**
|
|
3143
|
+
* Take the barge-in merge context, the completed-continuation context, and
|
|
3144
|
+
* the handoff note for the turn being launched — each feeds exactly the next
|
|
3145
|
+
* launched turn — and cancel the queued announcement, because the turn this
|
|
3146
|
+
* context rides delivers the same answer and would otherwise say it twice.
|
|
3147
|
+
*
|
|
3148
|
+
* Every real user turn consumes here, speculative or not: a speculative
|
|
3149
|
+
* dispatch latches `assistantTurnStarted`, so a turn that skipped this would
|
|
3150
|
+
* never pick the context up on any later pass. A rolled-back dispatch hands
|
|
3151
|
+
* it all back (restorePendingTurnContext).
|
|
3152
|
+
*/
|
|
3153
|
+
private consumePendingTurnContext(): {
|
|
3154
|
+
interruptedRequest: string | null;
|
|
3155
|
+
continuationResult: string | null;
|
|
3156
|
+
handedOffRequest: string | null;
|
|
3157
|
+
announcement: ContinuationDelivery | null;
|
|
3158
|
+
} {
|
|
3159
|
+
const consumed = {
|
|
3160
|
+
interruptedRequest: this.pendingInterruptedRequest,
|
|
3161
|
+
continuationResult: this.pendingContinuationResult,
|
|
3162
|
+
handedOffRequest: this.pendingHandoffRequest,
|
|
3163
|
+
announcement: this.pendingAnnouncement,
|
|
3164
|
+
};
|
|
2520
3165
|
this.pendingInterruptedRequest = null;
|
|
2521
|
-
const continuationResult = this.pendingContinuationResult;
|
|
2522
3166
|
this.pendingContinuationResult = null;
|
|
2523
|
-
|
|
2524
|
-
|
|
2525
|
-
|
|
2526
|
-
|
|
3167
|
+
this.pendingHandoffRequest = null;
|
|
3168
|
+
this.clearContinuationAnnouncement();
|
|
3169
|
+
return consumed;
|
|
3170
|
+
}
|
|
3171
|
+
|
|
3172
|
+
/**
|
|
3173
|
+
* Hand a turn's consumed pending context back to the session because the
|
|
3174
|
+
* turn never happened: a speculative dispatch was discarded or held, or the
|
|
3175
|
+
* launch never reached the bridge. Without this a hold verdict would destroy
|
|
3176
|
+
* a finished continuation's answer outright — the stash was already emptied
|
|
3177
|
+
* and no later turn would ever see it.
|
|
3178
|
+
*
|
|
3179
|
+
* Restores are conservative. A hard stop (interrupt/close/supersede) bumps
|
|
3180
|
+
* `detachStopGeneration`, and that drop is deliberate — never undone here.
|
|
3181
|
+
* Anything newer already sitting in a slot wins over the older value being
|
|
3182
|
+
* returned to it.
|
|
3183
|
+
*/
|
|
3184
|
+
private restorePendingTurnContext(turn: ActiveAssistantTurn): void {
|
|
3185
|
+
const interruptedRequest = turn.interruptedRequest;
|
|
3186
|
+
const continuationResult = turn.continuationResult;
|
|
3187
|
+
const handedOffRequest = turn.handedOffRequest;
|
|
3188
|
+
const announcement = turn.consumedAnnouncement;
|
|
3189
|
+
// One restore per turn: the rollback paths overlap (a discarded turn whose
|
|
3190
|
+
// leg start also fails), and the second pass must be a no-op.
|
|
3191
|
+
turn.interruptedRequest = null;
|
|
3192
|
+
turn.continuationResult = null;
|
|
3193
|
+
turn.handedOffRequest = null;
|
|
3194
|
+
turn.consumedAnnouncement = null;
|
|
3195
|
+
if (
|
|
3196
|
+
this.isClosed ||
|
|
3197
|
+
this.detachStopGeneration !== turn.pendingContextStopGeneration
|
|
3198
|
+
) {
|
|
3199
|
+
return;
|
|
3200
|
+
}
|
|
3201
|
+
if (this.pendingInterruptedRequest === null) {
|
|
3202
|
+
this.pendingInterruptedRequest = interruptedRequest;
|
|
3203
|
+
}
|
|
3204
|
+
if (this.pendingContinuationResult === null) {
|
|
3205
|
+
this.pendingContinuationResult = continuationResult;
|
|
3206
|
+
}
|
|
3207
|
+
// "Still running" is stale the moment a result exists, which is why the
|
|
3208
|
+
// detach clears the handoff note when it finishes — don't reinstate it
|
|
3209
|
+
// against a continuation that has since completed.
|
|
3210
|
+
if (
|
|
3211
|
+
this.pendingHandoffRequest === null &&
|
|
3212
|
+
this.pendingContinuationResult === null
|
|
3213
|
+
) {
|
|
3214
|
+
this.pendingHandoffRequest = handedOffRequest;
|
|
3215
|
+
}
|
|
3216
|
+
if (announcement !== null && this.pendingAnnouncement === null) {
|
|
3217
|
+
this.pendingAnnouncement = announcement;
|
|
3218
|
+
this.scheduleContinuationAnnouncement();
|
|
3219
|
+
}
|
|
2527
3220
|
}
|
|
2528
3221
|
|
|
2529
3222
|
// Build the ActiveAssistantTurn for a released utterance and drive its model
|
|
2530
|
-
// leg.
|
|
3223
|
+
// leg. Resolves true once the leg has a live turn handle, false when the
|
|
3224
|
+
// dispatch never reached the bridge — the signal a caller with a fallback
|
|
3225
|
+
// route (announceContinuation) needs, since a failed start is otherwise
|
|
3226
|
+
// handled entirely inside startAssistantLeg.
|
|
2531
3227
|
private async launchAssistantTurn(
|
|
2532
3228
|
utterance: UtteranceCycle,
|
|
2533
3229
|
content: string,
|
|
2534
3230
|
opts?: {
|
|
2535
|
-
// Set on
|
|
2536
|
-
//
|
|
2537
|
-
|
|
2538
|
-
// Set when a background continuation finished the interrupted reply: its
|
|
2539
|
-
// answer, appended to the turn's control prompt as context.
|
|
2540
|
-
continuationResult?: string | null;
|
|
3231
|
+
// Set on an announcement turn: the finished continuation this turn exists
|
|
3232
|
+
// to deliver. Its answer goes in the control prompt, not in `content`.
|
|
3233
|
+
continuationDelivery?: ContinuationDelivery | null;
|
|
2541
3234
|
// Unified front-door: dispatch without releasing the utterance. The
|
|
2542
3235
|
// thinking frame and floor-holding timers are deferred until the leg's
|
|
2543
3236
|
// leading verdict commits the turn (see commitSpeculativeTurn); a hold
|
|
2544
3237
|
// verdict rolls everything back instead.
|
|
2545
3238
|
speculative?: boolean;
|
|
2546
3239
|
},
|
|
2547
|
-
): Promise<
|
|
3240
|
+
): Promise<boolean> {
|
|
2548
3241
|
utterance.assistantTurnStarted = true;
|
|
3242
|
+
// The announcement turn IS the delivery of the pending continuation, so it
|
|
3243
|
+
// must not also consume the stash — feeding it both would deliver the same
|
|
3244
|
+
// answer twice. Every other turn is a real user turn and takes the context.
|
|
3245
|
+
const pending =
|
|
3246
|
+
opts?.continuationDelivery == null
|
|
3247
|
+
? this.consumePendingTurnContext()
|
|
3248
|
+
: null;
|
|
2549
3249
|
const token = Symbol("live-voice-assistant-turn");
|
|
2550
3250
|
const turnId = this.ensureTurnId(utterance);
|
|
2551
3251
|
this.startMetricsTurnIfNeeded(utterance, turnId);
|
|
@@ -2561,6 +3261,11 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2561
3261
|
progress: {
|
|
2562
3262
|
ops: [],
|
|
2563
3263
|
opsSinceNarration: 0,
|
|
3264
|
+
// Equal epochs at launch: a turn that has done nothing observable yet
|
|
3265
|
+
// has nothing to narrate, so the idle trigger waits for tool activity
|
|
3266
|
+
// or the maxSilenceMs heartbeat.
|
|
3267
|
+
stateEpoch: 0,
|
|
3268
|
+
narratedEpoch: 0,
|
|
2564
3269
|
updatesSpoken: 0,
|
|
2565
3270
|
lastFloorHolderAtMs: null,
|
|
2566
3271
|
lastAudibleAtMs: Date.now(),
|
|
@@ -2569,6 +3274,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2569
3274
|
},
|
|
2570
3275
|
assistantCompleted: false,
|
|
2571
3276
|
ttsDone: false,
|
|
3277
|
+
minimizeRequested: false,
|
|
2572
3278
|
ttsAudioStarted: false,
|
|
2573
3279
|
finalized: false,
|
|
2574
3280
|
speculativePending: opts?.speculative === true,
|
|
@@ -2580,8 +3286,12 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2580
3286
|
verdictDeadlineTimer: null,
|
|
2581
3287
|
discardRequested: false,
|
|
2582
3288
|
speculativeBuffer: "",
|
|
2583
|
-
interruptedRequest:
|
|
2584
|
-
continuationResult:
|
|
3289
|
+
interruptedRequest: pending?.interruptedRequest ?? null,
|
|
3290
|
+
continuationResult: pending?.continuationResult ?? null,
|
|
3291
|
+
handedOffRequest: pending?.handedOffRequest ?? null,
|
|
3292
|
+
consumedAnnouncement: pending?.announcement ?? null,
|
|
3293
|
+
pendingContextStopGeneration: this.detachStopGeneration,
|
|
3294
|
+
continuationDelivery: opts?.continuationDelivery ?? null,
|
|
2585
3295
|
toolUseStarted: false,
|
|
2586
3296
|
firstDeltaSeen: false,
|
|
2587
3297
|
deltaEpoch: 0,
|
|
@@ -2605,7 +3315,8 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2605
3315
|
if (!opts?.speculative) {
|
|
2606
3316
|
await this.sendFrame({ type: "thinking", turnId });
|
|
2607
3317
|
if (!this.isActiveAssistantTurn(token)) {
|
|
2608
|
-
|
|
3318
|
+
this.restorePendingTurnContext(activeTurn);
|
|
3319
|
+
return false;
|
|
2609
3320
|
}
|
|
2610
3321
|
} else {
|
|
2611
3322
|
// Verdict-deadline fail-open: the deferred-everything window is only
|
|
@@ -2635,11 +3346,10 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2635
3346
|
}, this.frontModelConfig.endpointDecisionTimeoutMs);
|
|
2636
3347
|
}
|
|
2637
3348
|
|
|
2638
|
-
const cfg = getConfig();
|
|
2639
|
-
|
|
2640
3349
|
// Slow-first-delta spoken ack: if the model produces no delta within the
|
|
2641
|
-
// budget, a short
|
|
2642
|
-
// sessions — a text-only session has no audio gap to
|
|
3350
|
+
// budget, a short front-model-phrased sentence holds the floor. Only
|
|
3351
|
+
// meaningful on TTS sessions — a text-only session has no audio gap to
|
|
3352
|
+
// bridge.
|
|
2643
3353
|
if (this.streamTtsAudio && !opts?.speculative) {
|
|
2644
3354
|
this.armAckTimer(activeTurn);
|
|
2645
3355
|
}
|
|
@@ -2657,33 +3367,30 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2657
3367
|
this.armProgressIdleTimer(activeTurn);
|
|
2658
3368
|
}
|
|
2659
3369
|
|
|
2660
|
-
//
|
|
2661
|
-
//
|
|
2662
|
-
//
|
|
2663
|
-
|
|
2664
|
-
|
|
2665
|
-
|
|
2666
|
-
|
|
2667
|
-
|
|
2668
|
-
|
|
2669
|
-
|
|
2670
|
-
|
|
2671
|
-
activeTurn
|
|
2672
|
-
|
|
2673
|
-
|
|
2674
|
-
content,
|
|
2675
|
-
routingLeg: "front-door",
|
|
2676
|
-
frontDoor: true,
|
|
2677
|
-
}
|
|
2678
|
-
: { content },
|
|
2679
|
-
);
|
|
3370
|
+
// Front-door leg: a fast model fronts every turn (the `voiceFrontDoor`
|
|
3371
|
+
// call site pins it) and may hand off to a quality leg on the escalate
|
|
3372
|
+
// verdict.
|
|
3373
|
+
const started = await this.startAssistantLeg(activeTurn, {
|
|
3374
|
+
content,
|
|
3375
|
+
routingLeg: "front-door",
|
|
3376
|
+
frontDoor: true,
|
|
3377
|
+
});
|
|
3378
|
+
if (!started) {
|
|
3379
|
+
// Nothing was persisted or spoken, so the context this turn consumed goes
|
|
3380
|
+
// back to the session rather than dying with the failed dispatch.
|
|
3381
|
+
this.restorePendingTurnContext(activeTurn);
|
|
3382
|
+
}
|
|
3383
|
+
return started;
|
|
2680
3384
|
}
|
|
2681
3385
|
|
|
2682
3386
|
/**
|
|
2683
3387
|
* Drive one model leg of an assistant turn through the session bridge,
|
|
2684
3388
|
* streaming its deltas to the live-voice client and TTS. Returns once the
|
|
2685
3389
|
* leg's turn handle is acquired (or the start fails) — turn completion stays
|
|
2686
|
-
* callback-driven via message_complete
|
|
3390
|
+
* callback-driven via message_complete. The resolved boolean is whether the
|
|
3391
|
+
* bridge actually handed back a turn handle: false means the leg never
|
|
3392
|
+
* started (no bridge wired, or the start threw and was reported as an error
|
|
3393
|
+
* frame), so nothing of this turn was persisted or spoken.
|
|
2687
3394
|
*
|
|
2688
3395
|
* A turn runs one leg normally. Under triage-and-escalate the front-door leg
|
|
2689
3396
|
* (`frontDoor: true`) runs the verdict-first protocol: its leading tokens
|
|
@@ -2704,35 +3411,38 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2704
3411
|
frontDoor?: boolean;
|
|
2705
3412
|
spokenEscalationBridge?: string;
|
|
2706
3413
|
},
|
|
2707
|
-
): Promise<
|
|
3414
|
+
): Promise<boolean> {
|
|
2708
3415
|
if (!this.startVoiceTurn) {
|
|
2709
|
-
return;
|
|
3416
|
+
return false;
|
|
2710
3417
|
}
|
|
2711
3418
|
const { token, utterance, turnId } = activeTurn;
|
|
2712
3419
|
|
|
2713
|
-
//
|
|
2714
|
-
//
|
|
2715
|
-
//
|
|
2716
|
-
//
|
|
2717
|
-
//
|
|
2718
|
-
//
|
|
2719
|
-
//
|
|
3420
|
+
// `rawText` accumulates this leg's full stream. A front-door leg starts
|
|
3421
|
+
// in `deciding` until its leading tokens classify as hold / escalate /
|
|
3422
|
+
// answer: an answer flushes through the shared marker holdback, while an
|
|
3423
|
+
// escalation buffers the post-verdict stream into `bridgeRaw` until the
|
|
3424
|
+
// bridge is complete, then hands off. A default/escalated leg flushes
|
|
3425
|
+
// every delta through the same holdback, so a stray control marker from
|
|
3426
|
+
// the main model is stripped instead of spoken.
|
|
2720
3427
|
let rawText = "";
|
|
2721
|
-
let frontDoorEmitted = 0;
|
|
2722
3428
|
let frontDoorStage: "deciding" | "answer" | "bridging" | "handedOff" =
|
|
2723
3429
|
"deciding";
|
|
2724
3430
|
let bridgeRaw = "";
|
|
2725
3431
|
|
|
2726
|
-
const
|
|
3432
|
+
const emitLegText = (chunk: string): void => {
|
|
2727
3433
|
if (chunk.length === 0) {
|
|
2728
3434
|
return;
|
|
2729
3435
|
}
|
|
2730
3436
|
this.markFirstAssistantDelta(utterance, turnId);
|
|
2731
3437
|
this.markFirstDeltaForAck(activeTurn);
|
|
2732
|
-
//
|
|
2733
|
-
//
|
|
2734
|
-
//
|
|
2735
|
-
//
|
|
3438
|
+
// Send-time abort gate: a delta queued behind a backed-up outbound
|
|
3439
|
+
// frame must not be written once barge-in aborts the turn, or the
|
|
3440
|
+
// cancelled reply's text leaks ahead of turn_cancelled. Key off this
|
|
3441
|
+
// turn's own abort signal — a normal message_complete finalizes and
|
|
3442
|
+
// clears activeAssistantTurn while trailing deltas may still be
|
|
3443
|
+
// draining, so an activeAssistantTurn-based guard would drop them.
|
|
3444
|
+
// Escalation aborts the front-door handle, not this turn's controller,
|
|
3445
|
+
// so legitimate front-door text still sends.
|
|
2736
3446
|
void this.sendFrame(
|
|
2737
3447
|
{ type: "assistant_text_delta", text: chunk },
|
|
2738
3448
|
() => !activeTurn.abortController.signal.aborted && !this.isClosed,
|
|
@@ -2740,26 +3450,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2740
3450
|
this.bufferAssistantTextForTts(token, chunk);
|
|
2741
3451
|
};
|
|
2742
3452
|
|
|
2743
|
-
|
|
2744
|
-
// emitted yet (answer stage only). A trailing "[…" that could still
|
|
2745
|
-
// become a control marker is held back until a later delta disproves
|
|
2746
|
-
// it; a real partial that never completes is simply never spoken.
|
|
2747
|
-
const flushFrontDoor = (): void => {
|
|
2748
|
-
let safeEnd = rawText.length;
|
|
2749
|
-
const lastOpen = rawText.lastIndexOf("[");
|
|
2750
|
-
if (lastOpen >= frontDoorEmitted) {
|
|
2751
|
-
const tail = rawText.slice(lastOpen);
|
|
2752
|
-
if (!tail.includes("]") && couldBeControlMarker(tail)) {
|
|
2753
|
-
safeEnd = lastOpen;
|
|
2754
|
-
}
|
|
2755
|
-
}
|
|
2756
|
-
if (safeEnd > frontDoorEmitted) {
|
|
2757
|
-
emitFrontDoor(
|
|
2758
|
-
stripInternalSpeechMarkers(rawText.slice(frontDoorEmitted, safeEnd)),
|
|
2759
|
-
);
|
|
2760
|
-
frontDoorEmitted = safeEnd;
|
|
2761
|
-
}
|
|
2762
|
-
};
|
|
3453
|
+
const flushLegText = createControlMarkerHoldback(activeTurn, emitLegText);
|
|
2763
3454
|
|
|
2764
3455
|
// Hand off once enough of the post-verdict stream has arrived to cap
|
|
2765
3456
|
// the bridge (sentence terminator or hard cap). Until then nothing is
|
|
@@ -2782,11 +3473,19 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2782
3473
|
assistantMessageChannel: "vellum",
|
|
2783
3474
|
userMessageInterface: "macos",
|
|
2784
3475
|
assistantMessageInterface: "macos",
|
|
2785
|
-
voiceControlPrompt: buildVoiceControlPrompt(activeTurn
|
|
3476
|
+
voiceControlPrompt: buildVoiceControlPrompt(activeTurn, {
|
|
3477
|
+
...(leg.frontDoor !== undefined ? { frontDoor: leg.frontDoor } : {}),
|
|
3478
|
+
}),
|
|
2786
3479
|
content: leg.content,
|
|
2787
3480
|
isInbound: true,
|
|
2788
3481
|
launchedAtMs: activeTurn.launchedAtMs,
|
|
2789
3482
|
signal: activeTurn.abortController.signal,
|
|
3483
|
+
// An announcement turn's content is a fixed marker, not user speech:
|
|
3484
|
+
// persist it hidden and suppress its echo so nothing renders as a user
|
|
3485
|
+
// bubble for a turn the user never started.
|
|
3486
|
+
...(activeTurn.continuationDelivery !== null
|
|
3487
|
+
? { hiddenSyntheticPrompt: true }
|
|
3488
|
+
: {}),
|
|
2790
3489
|
...(leg.overrideProfile != null
|
|
2791
3490
|
? { overrideProfile: leg.overrideProfile }
|
|
2792
3491
|
: {}),
|
|
@@ -2856,7 +3555,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2856
3555
|
}
|
|
2857
3556
|
frontDoorStage = "answer";
|
|
2858
3557
|
}
|
|
2859
|
-
|
|
3558
|
+
flushLegText(rawText);
|
|
2860
3559
|
return;
|
|
2861
3560
|
}
|
|
2862
3561
|
// Defensive: speculative legs are always front-door today, but a
|
|
@@ -2871,24 +3570,8 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2871
3570
|
return;
|
|
2872
3571
|
}
|
|
2873
3572
|
}
|
|
2874
|
-
|
|
2875
|
-
|
|
2876
|
-
void this.sendFrame(
|
|
2877
|
-
{
|
|
2878
|
-
type: "assistant_text_delta",
|
|
2879
|
-
text: msg.text,
|
|
2880
|
-
},
|
|
2881
|
-
// Re-check at send time (mirrors the tts_audio path): a delta
|
|
2882
|
-
// already queued behind a backed-up outbound frame must not be
|
|
2883
|
-
// written once barge-in has aborted the turn, or the cancelled
|
|
2884
|
-
// reply's text leaks ahead of turn_cancelled. Key off this turn's
|
|
2885
|
-
// own abort signal — a normal message_complete finalizes and
|
|
2886
|
-
// clears activeAssistantTurn while trailing deltas may still be
|
|
2887
|
-
// draining, so an activeAssistantTurn-based guard would drop them.
|
|
2888
|
-
() =>
|
|
2889
|
-
!activeTurn.abortController.signal.aborted && !this.isClosed,
|
|
2890
|
-
);
|
|
2891
|
-
this.bufferAssistantTextForTts(token, msg.text);
|
|
3573
|
+
rawText += msg.text;
|
|
3574
|
+
flushLegText(rawText);
|
|
2892
3575
|
},
|
|
2893
3576
|
message_complete: (msg) => {
|
|
2894
3577
|
const current = this.activeAssistantTurn;
|
|
@@ -2930,6 +3613,13 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2930
3613
|
if (leg.frontDoor && current.escalationHandedOff) {
|
|
2931
3614
|
return;
|
|
2932
3615
|
}
|
|
3616
|
+
// A held "[…"-tail that never completed a marker is real text —
|
|
3617
|
+
// force-flush it before assistantCompleted closes the TTS buffer
|
|
3618
|
+
// and completeTtsForTurn signals the drain, so it is spoken and
|
|
3619
|
+
// emitted rather than dropped.
|
|
3620
|
+
if (!leg.frontDoor && msg.type === "message_complete") {
|
|
3621
|
+
flushLegText(rawText, { force: true });
|
|
3622
|
+
}
|
|
2933
3623
|
current.assistantCompleted = true;
|
|
2934
3624
|
if (msg.type === "generation_cancelled") {
|
|
2935
3625
|
void this.finalizeAssistantTurn(
|
|
@@ -2940,6 +3630,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2940
3630
|
return;
|
|
2941
3631
|
}
|
|
2942
3632
|
current.assistantMessageId = msg.messageId ?? null;
|
|
3633
|
+
current.continuationDelivery = null;
|
|
2943
3634
|
this.completeTtsForTurn(token);
|
|
2944
3635
|
},
|
|
2945
3636
|
persisted_user_message_id: (messageId) => {
|
|
@@ -2964,6 +3655,40 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2964
3655
|
return;
|
|
2965
3656
|
}
|
|
2966
3657
|
current.toolUseStarted = true;
|
|
3658
|
+
// Foreground wins the workspace: the continuation runs with full
|
|
3659
|
+
// subagent abilities (it can write files, run commands), so the
|
|
3660
|
+
// moment a live turn starts a consequential tool the two could
|
|
3661
|
+
// race on the same workspace, host, or extension state. Kill
|
|
3662
|
+
// running continuations and skip pending detaches; a continuation
|
|
3663
|
+
// only survives while foreground turns stay provably read-only
|
|
3664
|
+
// (the topic-change case it exists for). Fail closed: only
|
|
3665
|
+
// provably non-contending built-ins keep a continuation alive
|
|
3666
|
+
// (see foregroundToolContendsWithContinuation) — a name-based
|
|
3667
|
+
// side-effect denylist misses mutators like plugin/MCP/skill
|
|
3668
|
+
// tools. Over-aborting only drops a best-effort salvage;
|
|
3669
|
+
// under-aborting risks a write race.
|
|
3670
|
+
// An already-completed continuation's stashed answer is kept — it
|
|
3671
|
+
// cannot race anything.
|
|
3672
|
+
//
|
|
3673
|
+
// Accepted residual: the abort is signal-level. A tool call
|
|
3674
|
+
// already executing inside the continuation is not awaited (the
|
|
3675
|
+
// agent loop abandons the in-flight promise on cancellation), so
|
|
3676
|
+
// that one call can briefly overlap the foreground tool. Closing
|
|
3677
|
+
// it would take a cross-conversation execution lock that the
|
|
3678
|
+
// subagent model deliberately does not have — parallel subagents
|
|
3679
|
+
// share the workspace with the parent everywhere — and awaiting
|
|
3680
|
+
// background teardown here would stall the live call's turn.
|
|
3681
|
+
// This gate already makes voice stricter than that baseline; the
|
|
3682
|
+
// residual is bounded to one in-flight call at barge-over time.
|
|
3683
|
+
if (
|
|
3684
|
+
foregroundToolContendsWithContinuation(toolName, detail?.input)
|
|
3685
|
+
) {
|
|
3686
|
+
this.abortDetachedRuns({
|
|
3687
|
+
keepPendingResult: true,
|
|
3688
|
+
reason: "foreground_tool_contends",
|
|
3689
|
+
toolName,
|
|
3690
|
+
});
|
|
3691
|
+
}
|
|
2967
3692
|
// The op counts toward the narration threshold on start (not
|
|
2968
3693
|
// completion) so a burst of slow tools still trips the ops
|
|
2969
3694
|
// trigger while they run.
|
|
@@ -2975,6 +3700,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2975
3700
|
startedAtMs: Date.now(),
|
|
2976
3701
|
});
|
|
2977
3702
|
current.progress.opsSinceNarration += 1;
|
|
3703
|
+
current.progress.stateEpoch += 1;
|
|
2978
3704
|
log.debug({ turnId, toolName }, "Live voice turn started tool use");
|
|
2979
3705
|
// Definitive tool use means the turn is guaranteed slow: speak
|
|
2980
3706
|
// the floor-holding ack now instead of waiting out the
|
|
@@ -3008,14 +3734,27 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3008
3734
|
)
|
|
3009
3735
|
: undefined) ??
|
|
3010
3736
|
findLastIncompleteOp(current.progress.ops, event.toolName);
|
|
3737
|
+
// A long-running op finishing is the beat the user has been
|
|
3738
|
+
// waiting through: it narrates immediately rather than waiting for
|
|
3739
|
+
// `opsThreshold` more ops, which on a one-slow-tool turn never
|
|
3740
|
+
// arrive. Short ops stay on the ops trigger — narrating every
|
|
3741
|
+
// quick lookup is the chatter this cadence exists to avoid.
|
|
3742
|
+
let trigger: "ops" | "op_complete" = "ops";
|
|
3011
3743
|
if (op) {
|
|
3012
3744
|
op.completedAtMs = Date.now();
|
|
3013
3745
|
if (event.isError !== undefined) {
|
|
3014
3746
|
op.isError = event.isError;
|
|
3015
3747
|
}
|
|
3016
3748
|
op.resultPreview = event.resultPreview;
|
|
3749
|
+
if (
|
|
3750
|
+
op.completedAtMs - op.startedAtMs >=
|
|
3751
|
+
this.frontModelConfig.progress.longOpMs
|
|
3752
|
+
) {
|
|
3753
|
+
trigger = "op_complete";
|
|
3754
|
+
}
|
|
3017
3755
|
}
|
|
3018
|
-
|
|
3756
|
+
current.progress.stateEpoch += 1;
|
|
3757
|
+
this.maybeNarrateProgress(current, trigger);
|
|
3019
3758
|
},
|
|
3020
3759
|
},
|
|
3021
3760
|
onError: (message) => {
|
|
@@ -3056,27 +3795,31 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3056
3795
|
} else {
|
|
3057
3796
|
handle.abort();
|
|
3058
3797
|
}
|
|
3059
|
-
|
|
3798
|
+
// The bridge ran: the leg's user row was persisted (or explicitly
|
|
3799
|
+
// rolled back by the discard above), so this counts as started even
|
|
3800
|
+
// though a newer turn owns the session.
|
|
3801
|
+
return true;
|
|
3060
3802
|
}
|
|
3061
3803
|
if (current.finalized) {
|
|
3062
|
-
this.
|
|
3063
|
-
return;
|
|
3804
|
+
this.clearActiveAssistantTurn(token);
|
|
3805
|
+
return true;
|
|
3064
3806
|
}
|
|
3065
3807
|
// The front-door leg may have handed off before its handle resolved;
|
|
3066
3808
|
// abort it rather than exposing it as the turn's live handle.
|
|
3067
3809
|
if (leg.frontDoor && current.escalationHandedOff) {
|
|
3068
3810
|
handle.abort();
|
|
3069
|
-
return;
|
|
3811
|
+
return true;
|
|
3070
3812
|
}
|
|
3071
3813
|
|
|
3072
3814
|
current.handle = handle;
|
|
3815
|
+
return true;
|
|
3073
3816
|
} catch (err) {
|
|
3074
3817
|
if (!this.isActiveAssistantTurn(token)) {
|
|
3075
|
-
return;
|
|
3818
|
+
return false;
|
|
3076
3819
|
}
|
|
3077
3820
|
|
|
3078
3821
|
this.clearFillerTimers(activeTurn);
|
|
3079
|
-
this.
|
|
3822
|
+
this.clearActiveAssistantTurn(token);
|
|
3080
3823
|
await this.sendFrame({
|
|
3081
3824
|
type: "error",
|
|
3082
3825
|
code: LiveVoiceProtocolErrorCode.InvalidField,
|
|
@@ -3086,6 +3829,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3086
3829
|
});
|
|
3087
3830
|
await this.finalizePendingUtterance(utterance, "assistant_start_error");
|
|
3088
3831
|
this.scheduleRearmAfterTurn();
|
|
3832
|
+
return false;
|
|
3089
3833
|
}
|
|
3090
3834
|
}
|
|
3091
3835
|
|
|
@@ -3271,10 +4015,10 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3271
4015
|
// the user's ears — not time since launch, so it covers mid-turn silences
|
|
3272
4016
|
// for the whole turn. On expiry with audio still pending, or with the
|
|
3273
4017
|
// silence not yet a full interval old, it re-arms for the remainder; only a
|
|
3274
|
-
// full interval of audible silence
|
|
3275
|
-
//
|
|
3276
|
-
//
|
|
3277
|
-
// turn
|
|
4018
|
+
// full interval of audible silence reaches the narration gatekeeper. The
|
|
4019
|
+
// interval is a polling cadence, not a speaking cadence: most ticks find
|
|
4020
|
+
// nothing new to report and stay quiet, so what the user hears follows the
|
|
4021
|
+
// turn's tool activity (with `maxSilenceMs` as the heartbeat ceiling).
|
|
3278
4022
|
private armProgressIdleTimer(
|
|
3279
4023
|
turn: ActiveAssistantTurn,
|
|
3280
4024
|
delayMs?: number,
|
|
@@ -3302,16 +4046,22 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3302
4046
|
}, delayMs ?? this.frontModelConfig.progress.idleIntervalMs);
|
|
3303
4047
|
}
|
|
3304
4048
|
|
|
3305
|
-
// Wall-clock instant the current audible silence turns a full interval old
|
|
3306
|
-
// measured from the latest of the last emitted segment, the estimated
|
|
3307
|
-
// client playback end, and the last enqueued filler.
|
|
4049
|
+
// Wall-clock instant the current audible silence turns a full interval old.
|
|
3308
4050
|
private progressIdleDeadlineMs(turn: ActiveAssistantTurn): number {
|
|
3309
4051
|
return (
|
|
3310
|
-
|
|
3311
|
-
|
|
3312
|
-
|
|
3313
|
-
|
|
3314
|
-
|
|
4052
|
+
this.progressSilenceSinceMs(turn) +
|
|
4053
|
+
this.frontModelConfig.progress.idleIntervalMs
|
|
4054
|
+
);
|
|
4055
|
+
}
|
|
4056
|
+
|
|
4057
|
+
// When the turn's current audible silence began: the latest of the last
|
|
4058
|
+
// emitted segment, the estimated client playback end, and the last enqueued
|
|
4059
|
+
// filler.
|
|
4060
|
+
private progressSilenceSinceMs(turn: ActiveAssistantTurn): number {
|
|
4061
|
+
return Math.max(
|
|
4062
|
+
turn.progress.lastAudibleAtMs,
|
|
4063
|
+
this.assistantPlaybackTailUntilMs,
|
|
4064
|
+
turn.progress.lastFloorHolderAtMs ?? 0,
|
|
3315
4065
|
);
|
|
3316
4066
|
}
|
|
3317
4067
|
|
|
@@ -3379,17 +4129,39 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3379
4129
|
);
|
|
3380
4130
|
}
|
|
3381
4131
|
|
|
4132
|
+
// The idle tick has something worth saying when the turn's tool activity has
|
|
4133
|
+
// moved since the last narration described it, or when the silence has run
|
|
4134
|
+
// past `maxSilenceMs` — the heartbeat ceiling that proves the assistant is
|
|
4135
|
+
// still alive on a turn with no observable activity at all. Every other tick
|
|
4136
|
+
// stays quiet, so the cadence follows the work rather than the clock.
|
|
4137
|
+
private progressIdleHasSomethingToSay(turn: ActiveAssistantTurn): boolean {
|
|
4138
|
+
const { progress } = turn;
|
|
4139
|
+
if (progress.stateEpoch !== progress.narratedEpoch) {
|
|
4140
|
+
return true;
|
|
4141
|
+
}
|
|
4142
|
+
const silentForMs = Date.now() - this.progressSilenceSinceMs(turn);
|
|
4143
|
+
if (silentForMs >= this.frontModelConfig.progress.maxSilenceMs) {
|
|
4144
|
+
return true;
|
|
4145
|
+
}
|
|
4146
|
+
log.debug(
|
|
4147
|
+
{ turnId: turn.turnId, silentForMs },
|
|
4148
|
+
"Live voice progress narration held — nothing new since the last update",
|
|
4149
|
+
);
|
|
4150
|
+
return false;
|
|
4151
|
+
}
|
|
4152
|
+
|
|
3382
4153
|
// Gatekeeper for spoken progress narration: it speaks only while the turn
|
|
3383
4154
|
// is audibly silent, spaced `minGapMs` from any spoken floor-holder (ack or
|
|
3384
|
-
// narration), one generation at a time, and —
|
|
3385
|
-
//
|
|
3386
|
-
//
|
|
3387
|
-
//
|
|
3388
|
-
//
|
|
3389
|
-
// count, so the next tool event
|
|
4155
|
+
// narration), one generation at a time, and — per trigger — only once the
|
|
4156
|
+
// ops trigger has `opsThreshold` ops accumulated or the idle trigger has
|
|
4157
|
+
// something new to report. No per-turn count cap: the cadence guards bound
|
|
4158
|
+
// the rate, and going quiet deep into a long turn is the failure mode
|
|
4159
|
+
// narration exists to prevent. Every failing guard short-circuits silently;
|
|
4160
|
+
// a skipped ops trigger keeps its accumulated count, so the next tool event
|
|
4161
|
+
// or idle tick retries.
|
|
3390
4162
|
private maybeNarrateProgress(
|
|
3391
4163
|
turn: ActiveAssistantTurn,
|
|
3392
|
-
trigger: "ops" | "idle",
|
|
4164
|
+
trigger: "ops" | "idle" | "op_complete",
|
|
3393
4165
|
): void {
|
|
3394
4166
|
const cfg = this.frontModelConfig.progress;
|
|
3395
4167
|
const { progress } = turn;
|
|
@@ -3406,7 +4178,8 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3406
4178
|
progress.narrationInFlight ||
|
|
3407
4179
|
(progress.lastFloorHolderAtMs !== null &&
|
|
3408
4180
|
Date.now() - progress.lastFloorHolderAtMs < cfg.minGapMs) ||
|
|
3409
|
-
(trigger === "ops" && progress.opsSinceNarration < cfg.opsThreshold)
|
|
4181
|
+
(trigger === "ops" && progress.opsSinceNarration < cfg.opsThreshold) ||
|
|
4182
|
+
(trigger === "idle" && !this.progressIdleHasSomethingToSay(turn))
|
|
3410
4183
|
) {
|
|
3411
4184
|
return;
|
|
3412
4185
|
}
|
|
@@ -3418,18 +4191,22 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3418
4191
|
// on the turn's ordered TTS queue. The decider internally bounds the call
|
|
3419
4192
|
// by `progress.generationTimeoutMs` and resolves null on every failure
|
|
3420
4193
|
// mode; on null the idle trigger falls back to a static phrase (silence is
|
|
3421
|
-
// actively harmful there) while the
|
|
3422
|
-
// filler is not worth it when narration was merely opportunistic.
|
|
4194
|
+
// actively harmful there) while the tool-activity triggers stay silent — a
|
|
4195
|
+
// generic filler is not worth it when narration was merely opportunistic.
|
|
3423
4196
|
private async speakProgressUpdate(
|
|
3424
4197
|
turn: ActiveAssistantTurn,
|
|
3425
4198
|
frontDecider: VoiceFrontDecider,
|
|
3426
|
-
trigger: "ops" | "idle",
|
|
4199
|
+
trigger: "ops" | "idle" | "op_complete",
|
|
3427
4200
|
): Promise<void> {
|
|
3428
4201
|
const { progress } = turn;
|
|
3429
4202
|
progress.narrationInFlight = true;
|
|
3430
4203
|
// Any delta that lands while the decider call is in flight makes the
|
|
3431
4204
|
// generated text stale — and proves the model is speaking again.
|
|
3432
4205
|
const deltaEpochAtLaunch = turn.deltaEpoch;
|
|
4206
|
+
// The activity this update describes. Tool events that land mid-generation
|
|
4207
|
+
// are news the generated text cannot carry, so they must leave the idle
|
|
4208
|
+
// trigger armed rather than count as already narrated.
|
|
4209
|
+
const stateEpochAtLaunch = progress.stateEpoch;
|
|
3433
4210
|
try {
|
|
3434
4211
|
const now = Date.now();
|
|
3435
4212
|
const currentOp = findLastIncompleteOp(progress.ops);
|
|
@@ -3469,7 +4246,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3469
4246
|
// condition may have flipped while generating, and a delta that landed
|
|
3470
4247
|
// mid-generation (deltaEpoch moved) makes the text stale even when the
|
|
3471
4248
|
// resulting audio has already drained by now. The ack-generation and
|
|
3472
|
-
// floor re-checks close the race with a generated ack
|
|
4249
|
+
// floor re-checks close the race with a generated ack that
|
|
3473
4250
|
// STARTED after this generation did — the entry guard rejects a
|
|
3474
4251
|
// narration while an ack generation is already pending, but cannot see
|
|
3475
4252
|
// one that begins mid-generation: while that ack is pending it has not
|
|
@@ -3489,7 +4266,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3489
4266
|
}
|
|
3490
4267
|
let raw = generated;
|
|
3491
4268
|
if (raw === null) {
|
|
3492
|
-
if (trigger
|
|
4269
|
+
if (trigger !== "idle") {
|
|
3493
4270
|
return;
|
|
3494
4271
|
}
|
|
3495
4272
|
raw = pickProgressPhrase(this.progressPhraseCounter++);
|
|
@@ -3498,6 +4275,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3498
4275
|
return;
|
|
3499
4276
|
}
|
|
3500
4277
|
progress.opsSinceNarration = 0;
|
|
4278
|
+
progress.narratedEpoch = stateEpochAtLaunch;
|
|
3501
4279
|
progress.updatesSpoken += 1;
|
|
3502
4280
|
// Like the ack mark, recorded only when narration audio actually
|
|
3503
4281
|
// enqueued (decider text or static fallback alike).
|
|
@@ -3521,30 +4299,22 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3521
4299
|
kind: LiveVoiceSpokenAckKind,
|
|
3522
4300
|
toolName?: string,
|
|
3523
4301
|
): void {
|
|
3524
|
-
|
|
3525
|
-
|
|
3526
|
-
//
|
|
3527
|
-
// any other ack trigger while the generation is in flight; the async
|
|
3528
|
-
// path undoes it if the ack turns out moot.
|
|
3529
|
-
void this.speakGeneratedAck(
|
|
3530
|
-
activeTurn,
|
|
3531
|
-
this.frontDecider,
|
|
3532
|
-
kind,
|
|
3533
|
-
toolName,
|
|
3534
|
-
);
|
|
4302
|
+
if (!this.frontDecider) {
|
|
4303
|
+
// No decider, no ack: a canned line is worse than the silence, which
|
|
4304
|
+
// progress narration already covers once the turn runs long.
|
|
3535
4305
|
return;
|
|
3536
4306
|
}
|
|
3537
|
-
|
|
3538
|
-
|
|
3539
|
-
|
|
3540
|
-
|
|
3541
|
-
|
|
4307
|
+
activeTurn.ackSpoken = true;
|
|
4308
|
+
// Optimistic ackSpoken above keeps the one-per-turn budget honest for
|
|
4309
|
+
// any other ack trigger while the generation is in flight; the async
|
|
4310
|
+
// path undoes it if the ack turns out moot.
|
|
4311
|
+
void this.speakGeneratedAck(activeTurn, this.frontDecider, kind, toolName);
|
|
3542
4312
|
}
|
|
3543
4313
|
|
|
3544
|
-
//
|
|
3545
|
-
// for one short contextual sentence, static phrase on null. The decider
|
|
4314
|
+
// Ask the front model for one short contextual sentence. The decider
|
|
3546
4315
|
// internally bounds the call by `ackGenerationTimeoutMs` and resolves null
|
|
3547
|
-
// on every failure mode, so the enqueue never waits beyond that budget
|
|
4316
|
+
// on every failure mode, so the enqueue never waits beyond that budget; a
|
|
4317
|
+
// null generation speaks nothing.
|
|
3548
4318
|
private async speakGeneratedAck(
|
|
3549
4319
|
activeTurn: ActiveAssistantTurn,
|
|
3550
4320
|
frontDecider: VoiceFrontDecider,
|
|
@@ -3567,23 +4337,22 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3567
4337
|
// The decider contract never rejects; belt-and-braces for a stub.
|
|
3568
4338
|
.catch(() => null);
|
|
3569
4339
|
// Liveness re-check after the await: any turnCanSpeakFiller condition
|
|
3570
|
-
// may have flipped while generating, mooting the ack.
|
|
3571
|
-
//
|
|
3572
|
-
|
|
4340
|
+
// may have flipped while generating, mooting the ack. A null generation
|
|
4341
|
+
// (no provider, timeout, unusable output) speaks nothing. Either way,
|
|
4342
|
+
// undo the optimistic ackSpoken so the flag reflects reality and a
|
|
4343
|
+
// later trigger this turn can still hold the floor.
|
|
4344
|
+
if (generated === null || !this.turnCanSpeakFiller(activeTurn)) {
|
|
3573
4345
|
activeTurn.ackSpoken = false;
|
|
3574
4346
|
return;
|
|
3575
4347
|
}
|
|
3576
|
-
this.enqueueAckPhrase(
|
|
3577
|
-
activeTurn,
|
|
3578
|
-
generated ?? this.pickStaticAckPhrase(kind),
|
|
3579
|
-
kind,
|
|
3580
|
-
);
|
|
4348
|
+
this.enqueueAckPhrase(activeTurn, generated, kind);
|
|
3581
4349
|
} finally {
|
|
3582
4350
|
activeTurn.ackGenerationPending = false;
|
|
3583
4351
|
}
|
|
3584
4352
|
}
|
|
3585
4353
|
|
|
3586
|
-
// Records the ack-spoken metric only when a phrase actually enqueues
|
|
4354
|
+
// Records the ack-spoken metric only when a phrase actually enqueues; a
|
|
4355
|
+
// phrase that sanitizes away releases the turn's ack budget again.
|
|
3587
4356
|
private enqueueAckPhrase(
|
|
3588
4357
|
activeTurn: ActiveAssistantTurn,
|
|
3589
4358
|
raw: string,
|
|
@@ -3591,7 +4360,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3591
4360
|
): void {
|
|
3592
4361
|
if (this.enqueueFillerPhrase(activeTurn, raw)) {
|
|
3593
4362
|
this.markAckSpoken(activeTurn, kind);
|
|
4363
|
+
return;
|
|
3594
4364
|
}
|
|
4365
|
+
activeTurn.ackSpoken = false;
|
|
3595
4366
|
}
|
|
3596
4367
|
|
|
3597
4368
|
// Sanitize and enqueue one filler sentence (spoken ack or progress
|
|
@@ -3664,10 +4435,22 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3664
4435
|
!this.isClosed,
|
|
3665
4436
|
);
|
|
3666
4437
|
|
|
3667
|
-
|
|
3668
|
-
|
|
3669
|
-
|
|
3670
|
-
|
|
4438
|
+
// Drain-scoped minimize: the latched marker is consumed here, after
|
|
4439
|
+
// the turn's speech has fully drained — never mid-speech, never for
|
|
4440
|
+
// a barged-in turn, at most once per turn.
|
|
4441
|
+
if (
|
|
4442
|
+
currentTurn.minimizeRequested &&
|
|
4443
|
+
!currentTurn.abortController.signal.aborted
|
|
4444
|
+
) {
|
|
4445
|
+
currentTurn.minimizeRequested = false;
|
|
4446
|
+
await this.sendFrame(
|
|
4447
|
+
{ type: "minimize_room", turnId: currentTurn.turnId },
|
|
4448
|
+
() => !this.isClosed,
|
|
4449
|
+
);
|
|
4450
|
+
}
|
|
4451
|
+
|
|
4452
|
+
if (currentTurn.handle && currentTurn.finalized) {
|
|
4453
|
+
this.clearActiveAssistantTurn(token);
|
|
3671
4454
|
}
|
|
3672
4455
|
|
|
3673
4456
|
// Re-arm only after the terminal tts_done frame so a slow or failing
|
|
@@ -4147,7 +4930,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
4147
4930
|
this.activeAssistantTurn?.token === turn.token &&
|
|
4148
4931
|
turn.handle
|
|
4149
4932
|
) {
|
|
4150
|
-
this.
|
|
4933
|
+
this.clearActiveAssistantTurn(turn.token);
|
|
4151
4934
|
}
|
|
4152
4935
|
|
|
4153
4936
|
if (options.rearm ?? true) {
|
|
@@ -4440,21 +5223,70 @@ export function createLiveVoiceSession(
|
|
|
4440
5223
|
// the interrupted turn. The fork inherits the conversation's current messages
|
|
4441
5224
|
// (which include the interrupted turn's completed tool calls after teardown),
|
|
4442
5225
|
// so it resumes without repeating them. Uses spawnAndAwait (synchronous mode)
|
|
4443
|
-
// so the terminal parent-notification is skipped — the
|
|
4444
|
-
//
|
|
4445
|
-
//
|
|
4446
|
-
//
|
|
4447
|
-
|
|
5226
|
+
// so the terminal parent-notification is skipped — the run itself says nothing;
|
|
5227
|
+
// its final answer is RETURNED so the session decides how it reaches the user.
|
|
5228
|
+
// The `signal` aborts the child on a stop/interrupt (spawnAndAwait then
|
|
5229
|
+
// rejects).
|
|
5230
|
+
//
|
|
5231
|
+
// Exported for tests only: every production caller reaches it through the
|
|
5232
|
+
// `spawnBackgroundContinuation` option default in `createLiveVoiceSession`.
|
|
5233
|
+
export async function defaultSpawnBackgroundContinuation(args: {
|
|
4448
5234
|
parentConversationId: string;
|
|
4449
5235
|
objective: string;
|
|
4450
5236
|
label: string;
|
|
4451
5237
|
signal: AbortSignal;
|
|
4452
5238
|
}): Promise<string> {
|
|
4453
|
-
|
|
4454
|
-
|
|
4455
|
-
|
|
4456
|
-
|
|
4457
|
-
|
|
5239
|
+
// Trust first, before anything can hydrate: the bridge stamps trust per-turn
|
|
5240
|
+
// and clears it at teardown, which has settled by now — inheriting from the
|
|
5241
|
+
// parent would read the cleared window and run the continuation fail-closed
|
|
5242
|
+
// as `unknown`, denying every consequential tool. Resolve the same guardian
|
|
5243
|
+
// trust the foreground turn ran under and pass it explicitly (resolution
|
|
5244
|
+
// itself stays fail-closed: on a miss the continuation runs as `unknown`,
|
|
5245
|
+
// exactly as an unstamped turn does).
|
|
5246
|
+
const trustContext = await resolveLocalLiveVoiceTrustContext(
|
|
5247
|
+
args.parentConversationId,
|
|
5248
|
+
);
|
|
5249
|
+
// getOrCreateConversation (not a raw registry read): it rebuilds a stale
|
|
5250
|
+
// instance and awaits loadFromDb, so the snapshot below sees the persisted
|
|
5251
|
+
// history. A raw findConversation can return a cold instance whose in-memory
|
|
5252
|
+
// `messages` is empty, which silently forks a continuation with no context.
|
|
5253
|
+
const { getOrCreateConversation } =
|
|
5254
|
+
await import("../daemon/conversation-store.js");
|
|
5255
|
+
const parentConversation = await getOrCreateConversation(
|
|
5256
|
+
args.parentConversationId,
|
|
5257
|
+
);
|
|
5258
|
+
// Hydration runs under the resolved trust: `loadFromDb` filters history by
|
|
5259
|
+
// the instance's own trust class, so a load with no trust context drops
|
|
5260
|
+
// memory blocks and forces the compaction summary to null — the fork would
|
|
5261
|
+
// inherit a lobotomized snapshot. Stamp for the duration of the load and
|
|
5262
|
+
// restore the prior value, the same set-load-restore idiom
|
|
5263
|
+
// `elevatePointerConversationToGuardian` uses to un-filter a cold pointer
|
|
5264
|
+
// turn. The restore matters: the bridge owns this conversation's trust
|
|
5265
|
+
// per-turn and expects the window between turns to be empty.
|
|
5266
|
+
//
|
|
5267
|
+
// `ensureActorScopedHistory` reloads exactly when the resident history was
|
|
5268
|
+
// loaded under a different trust class — covering both the never-loaded cold
|
|
5269
|
+
// instance and the untrusted load `getOrCreateConversation` may have just
|
|
5270
|
+
// run — and no-ops when the history is already guardian-scoped. Without a
|
|
5271
|
+
// guardian to stamp there is nothing to re-scope, so an unhydrated instance
|
|
5272
|
+
// falls back to a plain load; a genuinely new conversation reads zero rows,
|
|
5273
|
+
// harmless.
|
|
5274
|
+
const priorTrustContext = parentConversation.trustContext;
|
|
5275
|
+
if (trustContext) {
|
|
5276
|
+
const stampedTrustContext = { ...trustContext };
|
|
5277
|
+
parentConversation.setTrustContext(stampedTrustContext);
|
|
5278
|
+
try {
|
|
5279
|
+
await parentConversation.ensureActorScopedHistory();
|
|
5280
|
+
} finally {
|
|
5281
|
+
// Only undo the stamp this block installed: the user's next foreground
|
|
5282
|
+
// turn can start during the load and stamp its own trust, and clearing
|
|
5283
|
+
// that would run the live turn fail-closed.
|
|
5284
|
+
if (parentConversation.trustContext === stampedTrustContext) {
|
|
5285
|
+
parentConversation.setTrustContext(priorTrustContext ?? null);
|
|
5286
|
+
}
|
|
5287
|
+
}
|
|
5288
|
+
} else if (parentConversation.getMessages().length === 0) {
|
|
5289
|
+
await parentConversation.loadFromDb();
|
|
4458
5290
|
}
|
|
4459
5291
|
return await getSubagentManager().spawnAndAwait(
|
|
4460
5292
|
{
|
|
@@ -4463,18 +5295,35 @@ async function defaultSpawnBackgroundContinuation(args: {
|
|
|
4463
5295
|
objective: args.objective,
|
|
4464
5296
|
fork: true,
|
|
4465
5297
|
sendResultToUser: false,
|
|
4466
|
-
//
|
|
4467
|
-
//
|
|
4468
|
-
//
|
|
4469
|
-
//
|
|
4470
|
-
//
|
|
4471
|
-
|
|
4472
|
-
|
|
5298
|
+
// Full subagent abilities: the continuation runs like any other
|
|
5299
|
+
// background subagent, so it can genuinely finish build-shaped work
|
|
5300
|
+
// (JARVIS-1354). Side effects are governed by the standard
|
|
5301
|
+
// non-interactive permission path under the explicit trust context
|
|
5302
|
+
// resolved above — auto-approved up to the background risk threshold,
|
|
5303
|
+
// auto-denied above it — the same policy the foreground voice turn it
|
|
5304
|
+
// continues ran under. Workspace write races with the user's next
|
|
5305
|
+
// foreground turn are prevented by the session's foreground-wins abort
|
|
5306
|
+
// (a side-effecting tool start on a live turn aborts running
|
|
5307
|
+
// continuations; see the tool_use_start handler).
|
|
5308
|
+
...(trustContext ? { trustContext } : {}),
|
|
5309
|
+
parentMessages: [...parentConversation.getMessages()],
|
|
4473
5310
|
parentSystemPrompt: parentConversation.getCurrentSystemPrompt(),
|
|
4474
5311
|
},
|
|
4475
|
-
//
|
|
4476
|
-
//
|
|
4477
|
-
|
|
5312
|
+
// Broadcast subagent events to clients attached to this conversation so
|
|
5313
|
+
// the continuation appears in the UI like any other handoff — a background
|
|
5314
|
+
// run the user cannot see reads as the assistant silently dropping their
|
|
5315
|
+
// work. "Silent" means the RUN makes no sound of its own — its result is
|
|
5316
|
+
// spoken by a session turn, on the session's terms; it was never meant to
|
|
5317
|
+
// mean invisible.
|
|
5318
|
+
//
|
|
5319
|
+
// NOT the conversation's own sender: the voice bridge resets that to a
|
|
5320
|
+
// no-op at turn teardown (see voice-session-bridge's clientCallbackInstalled
|
|
5321
|
+
// reset), and the detach deliberately waits for that teardown before
|
|
5322
|
+
// spawning — so a sender-based route is guaranteed to be dead by the time
|
|
5323
|
+
// these events fire. `broadcastMessage` is the same path the bridge itself
|
|
5324
|
+
// uses to reach an attached web client. The subagent events carry
|
|
5325
|
+
// `parentConversationId`, not `conversationId`, so scope explicitly.
|
|
5326
|
+
(msg) => broadcastMessage(msg, args.parentConversationId),
|
|
4478
5327
|
{ signal: args.signal },
|
|
4479
5328
|
);
|
|
4480
5329
|
}
|