@vellumai/assistant 0.10.12 → 0.11.0-staging.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +1 -1
- package/ARCHITECTURE.md +4 -5
- package/README.md +1 -1
- package/docs/architecture/memory.md +60 -23
- package/docs/error-handling.md +3 -3
- package/docs/guardian-request-flow.md +112 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/package.json +1 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/__tests__/channels.test.ts +9 -1
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +449 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/channels.ts +1 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/secret-detection.ts +468 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@slack/types/dist/events/app.d.ts +28 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@slack/types/dist/events/app.d.ts.map +1 -1
- package/node_modules/@vellumai/gateway-client/node_modules/@slack/types/dist/events/index.d.ts +2 -2
- package/node_modules/@vellumai/gateway-client/node_modules/@slack/types/dist/events/index.d.ts.map +1 -1
- package/node_modules/@vellumai/gateway-client/node_modules/@slack/types/dist/events/index.js.map +1 -1
- package/node_modules/@vellumai/gateway-client/node_modules/@slack/types/dist/events/message.d.ts +2 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@slack/types/dist/events/message.d.ts.map +1 -1
- package/node_modules/@vellumai/gateway-client/node_modules/@slack/types/package.json +1 -1
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/package.json +1 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/__tests__/channels.test.ts +9 -1
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +449 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/channels.ts +1 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/secret-detection.ts +468 -0
- package/node_modules/@vellumai/gateway-client/package.json +1 -1
- package/node_modules/@vellumai/gateway-client/src/admission-policy-contract.ts +6 -0
- package/node_modules/@vellumai/gateway-client/src/inbound-contract.ts +34 -0
- package/node_modules/@vellumai/service-contracts/package.json +1 -0
- package/node_modules/@vellumai/service-contracts/src/__tests__/channels.test.ts +9 -1
- package/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +449 -0
- package/node_modules/@vellumai/service-contracts/src/channels.ts +1 -0
- package/node_modules/@vellumai/service-contracts/src/secret-detection.ts +468 -0
- package/openapi.yaml +246 -10
- package/package.json +2 -2
- package/src/__tests__/acp-session.test.ts +3 -3
- package/src/__tests__/agent-loop-mutable-latest-user-message.test.ts +3 -0
- package/src/__tests__/anthropic-provider.test.ts +40 -0
- package/src/__tests__/approval-cascade.test.ts +5 -5
- package/src/__tests__/approval-routes-http.test.ts +3 -3
- package/src/__tests__/ask-question-resolver.test.ts +207 -0
- package/src/__tests__/assistant-event-hub-self-exclusion.test.ts +16 -16
- package/src/__tests__/assistant-event-hub-targeted.test.ts +13 -11
- package/src/__tests__/assistant-event-hub.test.ts +13 -11
- package/src/__tests__/assistant-event.test.ts +12 -9
- package/src/__tests__/assistant-feature-flag-guard.test.ts +3 -3
- package/src/__tests__/assistant-feature-flag-guardrails.test.ts +1 -1
- package/src/__tests__/assistant-id-boundary-guard.test.ts +0 -58
- package/src/__tests__/assistant-stream-state.test.ts +17 -7
- package/src/__tests__/avatar-identity-sync.test.ts +3 -3
- package/src/__tests__/background-workers-disk-pressure.test.ts +1 -0
- package/src/__tests__/call-conversation-messages.test.ts +134 -18
- package/src/__tests__/call-start-guardian-guard.test.ts +27 -0
- package/src/__tests__/channel-approvals.test.ts +0 -27
- package/src/__tests__/channel-guardian.test.ts +85 -512
- package/src/__tests__/channel-readiness-slack-remote.test.ts +151 -3
- package/src/__tests__/channel-retry-sweep.test.ts +88 -1
- package/src/__tests__/channel-setup-panel-ack.test.ts +12 -9
- package/src/__tests__/chat-reveal-guard-priming.test.ts +31 -34
- package/src/__tests__/cli-memory-v2-reembed-skills.test.ts +2 -2
- package/src/__tests__/compaction-events.test.ts +8 -8
- package/src/__tests__/config-loader-platform-defaults.test.ts +54 -28
- package/src/__tests__/config-model-image-provider.test.ts +20 -0
- package/src/__tests__/config-schema-memory-substrate.test.ts +306 -0
- package/src/__tests__/config-schema.test.ts +14 -2
- package/src/__tests__/config-set-platform-guard.test.ts +11 -12
- package/src/__tests__/config-sounds-sync.test.ts +4 -4
- package/src/__tests__/confirmation-request-guardian-bridge.test.ts +5 -0
- package/src/__tests__/context-search-memory-source.test.ts +1 -1
- package/src/__tests__/context-search-memory-v2-source.test.ts +6 -6
- package/src/__tests__/context-token-estimator.test.ts +54 -1
- package/src/__tests__/conversation-agent-loop-disk-pressure.test.ts +12 -3
- package/src/__tests__/conversation-agent-loop-fatal-cleanup.test.ts +13 -4
- package/src/__tests__/conversation-agent-loop-handlers-max-tokens.test.ts +4 -4
- package/src/__tests__/conversation-agent-loop-inference-profile.test.ts +10 -1
- package/src/__tests__/conversation-agent-loop-overflow.test.ts +22 -13
- package/src/__tests__/conversation-agent-loop.test.ts +39 -30
- package/src/__tests__/conversation-confirmation-signals.test.ts +14 -14
- package/src/__tests__/conversation-evictor.test.ts +49 -0
- package/src/__tests__/conversation-fork-crud.test.ts +1 -1
- package/src/__tests__/conversation-history-web-search.test.ts +1 -1
- package/src/__tests__/conversation-key-store-disk-view.test.ts +5 -5
- package/src/__tests__/conversation-loop-db-lock-resilience.test.ts +5 -5
- package/src/__tests__/conversation-message-sync-tags.test.ts +4 -4
- package/src/__tests__/conversation-queue.test.ts +81 -81
- package/src/__tests__/conversation-routes-enabled-plugins.test.ts +8 -1
- package/src/__tests__/conversation-routes-hidden-queue.test.ts +8 -1
- package/src/__tests__/conversation-routes-slash-commands.test.ts +10 -1
- package/src/__tests__/conversation-runtime-assembly.test.ts +29 -2
- package/src/__tests__/conversation-slash-queue.test.ts +11 -11
- package/src/__tests__/conversation-slash-unknown.test.ts +2 -2
- package/src/__tests__/conversation-speed-override.test.ts +2 -2
- package/src/__tests__/conversation-summarize-route.test.ts +9 -0
- package/src/__tests__/conversation-surfaces-action-delivery.test.ts +18 -16
- package/src/__tests__/conversation-surfaces-activation-emit.test.ts +17 -17
- package/src/__tests__/conversation-surfaces-app-open.test.ts +7 -4
- package/src/__tests__/conversation-surfaces-data-persist.test.ts +7 -7
- package/src/__tests__/conversation-surfaces-standalone-payloads.test.ts +9 -9
- package/src/__tests__/conversation-surfaces-standalone.test.ts +12 -12
- package/src/__tests__/conversation-surfaces-state-update.test.ts +8 -5
- package/src/__tests__/conversation-surfaces-table-action.test.ts +6 -3
- package/src/__tests__/conversation-surfaces-task-progress.test.ts +46 -46
- package/src/__tests__/conversation-sync-tags.test.ts +5 -5
- package/src/__tests__/conversation-wait-for-idle.test.ts +2 -2
- package/src/__tests__/conversation-workspace-cache-state.test.ts +0 -4
- package/src/__tests__/conversation-workspace-injection.test.ts +0 -3
- package/src/__tests__/conversation-workspace-tool-tracking.test.ts +0 -3
- package/src/__tests__/copy-composer-tc-templates.test.ts +3 -131
- package/src/__tests__/credential-security-invariants.test.ts +1 -0
- package/src/__tests__/daemon-assistant-events.test.ts +14 -15
- package/src/__tests__/db-test-helpers.ts +13 -27
- package/src/__tests__/disk-pressure-routes.test.ts +12 -3
- package/src/__tests__/document-tool-security.test.ts +95 -0
- package/src/__tests__/drain-kick-guard.test.ts +232 -0
- package/src/__tests__/dynamic-skill-background-guard.test.ts +19 -6
- package/src/__tests__/emit-event-signal.test.ts +5 -6
- package/src/__tests__/encrypted-store-test-helpers.ts +10 -20
- package/src/__tests__/events-tail-route.test.ts +6 -4
- package/src/__tests__/feature-flag-test-helpers.ts +10 -20
- package/src/__tests__/file-edit-tool.test.ts +24 -4
- package/src/__tests__/file-list-tool.test.ts +2 -1
- package/src/__tests__/file-read-tool.test.ts +62 -10
- package/src/__tests__/file-write-tool.test.ts +24 -5
- package/src/__tests__/followup-tools.test.ts +58 -0
- package/src/__tests__/gateway-threshold-reader-mock.ts +99 -0
- package/src/__tests__/graph-extraction-event-date.test.ts +1 -1
- package/src/__tests__/guardian-question-mode.test.ts +42 -3
- package/src/__tests__/guardian-reply-router-answer-mode.test.ts +283 -0
- package/src/__tests__/guardian-vellum-destination-pinning.test.ts +221 -0
- package/src/__tests__/heartbeat-disk-pressure.test.ts +1 -0
- package/src/__tests__/heartbeat-service.test.ts +1 -0
- package/src/__tests__/helpers/native-web-search-harness.ts +10 -10
- package/src/__tests__/host-browser-proxy.test.ts +1 -1
- package/src/__tests__/host-file-edit-tool.test.ts +3 -3
- package/src/__tests__/host-file-read-tool.test.ts +2 -2
- package/src/__tests__/host-file-write-tool.test.ts +7 -3
- package/src/__tests__/host-shell-tool.test.ts +17 -5
- package/src/__tests__/host-transfer-pending-interactions.test.ts +8 -8
- package/src/__tests__/image-credentials.test.ts +65 -8
- package/src/__tests__/injection-block.test.ts +4 -4
- package/src/__tests__/injector-chain.test.ts +1 -1
- package/src/__tests__/injector-pkb-v2-silenced.test.ts +2 -2
- package/src/__tests__/injector-registry-order-guard.test.ts +4 -9
- package/src/__tests__/injector-v3-suppression.test.ts +6 -4
- package/src/__tests__/lifecycle-memory-v2-seed.test.ts +27 -28
- package/src/__tests__/list-messages-page-latest.test.ts +2 -2
- package/src/__tests__/llm-usage-store.test.ts +67 -4
- package/src/__tests__/log-redact.test.ts +18 -0
- package/src/__tests__/managed-profile-guard.test.ts +27 -0
- package/src/__tests__/managed-skill-lifecycle.test.ts +2 -2
- package/src/__tests__/media-generate-image.test.ts +62 -11
- package/src/__tests__/memory-identity-context-parity.test.ts +1 -1
- package/src/__tests__/memory-jobs-worker-lanes.test.ts +9 -1
- package/src/__tests__/memory-retrieval-hook.test.ts +4 -4
- package/src/__tests__/notification-decision-strategy.test.ts +115 -11
- package/src/__tests__/notification-deep-link.test.ts +12 -12
- package/src/__tests__/notification-vellum-adapter.test.ts +8 -8
- package/src/__tests__/outbound-slack-persistence.test.ts +8 -8
- package/src/__tests__/path-policy.test.ts +315 -0
- package/src/__tests__/pending-interactions-resolved-event.test.ts +6 -6
- package/src/__tests__/persistence-layering-guard.test.ts +2 -2
- package/src/__tests__/pkb-autoinject.test.ts +1 -1
- package/src/__tests__/platform-bash-auto-approve.test.ts +1 -0
- package/src/__tests__/plugin-event-hub-facade.test.ts +22 -15
- package/src/__tests__/plugin-import-boundary-guard.test.ts +15 -15
- package/src/__tests__/processing-flag-persist-failure.test.ts +0 -4
- package/src/__tests__/provider-platform-proxy-integration.test.ts +0 -1
- package/src/__tests__/rebuild-index-graph-nodes.test.ts +1 -1
- package/src/__tests__/request-secret-standalone-channel-gate.test.ts +5 -5
- package/src/__tests__/require-fresh-approval.test.ts +38 -38
- package/src/__tests__/run-due-schedules.test.ts +72 -1
- package/src/__tests__/runtime-events-sse-parity.test.ts +11 -10
- package/src/__tests__/runtime-events-sse.test.ts +1 -1
- package/src/__tests__/schedule-routes.test.ts +44 -5
- package/src/__tests__/schedule-store.test.ts +3 -3
- package/src/__tests__/schedule-tools.test.ts +176 -0
- package/src/__tests__/secret-prompt-log-hygiene.test.ts +3 -3
- package/src/__tests__/secret-prompter-channel-fallback.test.ts +3 -3
- package/src/__tests__/secret-response-routing.test.ts +3 -3
- package/src/__tests__/secret-routes-platform-proxy.test.ts +4 -7
- package/src/__tests__/secret-scanner.test.ts +36 -0
- package/src/__tests__/send-endpoint-busy.test.ts +7 -7
- package/src/__tests__/slack-required-scopes-mirror.test.ts +81 -0
- package/src/__tests__/starter-task-flow.test.ts +6 -3
- package/src/__tests__/subagent-call-site-routing.test.ts +2 -2
- package/src/__tests__/subagent-disposal.test.ts +40 -11
- package/src/__tests__/subagent-fork-notifications.test.ts +4 -4
- package/src/__tests__/subagent-fork-prompt-role.test.ts +3 -3
- package/src/__tests__/subagent-fork-spawn.test.ts +4 -4
- package/src/__tests__/subagent-manager-notify.test.ts +73 -14
- package/src/__tests__/subagent-notify-parent.test.ts +48 -4
- package/src/__tests__/subagent-spawn-and-await.test.ts +62 -14
- package/src/__tests__/subagent-tools.test.ts +64 -0
- package/src/__tests__/sync-message-contract.test.ts +3 -3
- package/src/__tests__/terminal-tools.test.ts +12 -3
- package/src/__tests__/test-preload.ts +9 -0
- package/src/__tests__/tool-approval-handler.test.ts +583 -43
- package/src/__tests__/tool-execution-pipeline.benchmark.test.ts +7 -4
- package/src/__tests__/tool-executor.test.ts +116 -0
- package/src/__tests__/tool-grant-request-escalation.test.ts +5 -0
- package/src/__tests__/tool-preview-lifecycle.test.ts +32 -29
- package/src/__tests__/tool-result-metadata-plumbing.test.ts +5 -5
- package/src/__tests__/tool-start-timestamp.test.ts +4 -4
- package/src/__tests__/trusted-contact-lifecycle-notifications.test.ts +9 -22
- package/src/__tests__/ui-choice-copy-surfaces.test.ts +13 -13
- package/src/__tests__/ui-work-result-surface.test.ts +3 -3
- package/src/__tests__/voice-scoped-grant-consumer.test.ts +6 -6
- package/src/__tests__/voice-session-bridge.test.ts +44 -44
- package/src/__tests__/workspace-migration-134-image-generation-mode-to-provider.test.ts +284 -0
- package/src/__tests__/workspace-policy.test.ts +399 -0
- package/src/acp/__tests__/client-handler.test.ts +3 -3
- package/src/acp/__tests__/session-manager-persistence.test.ts +10 -10
- package/src/acp/__tests__/session-manager-resume.test.ts +7 -7
- package/src/acp/client-handler.ts +3 -3
- package/src/acp/session-manager.ts +8 -8
- package/src/api/events/subagent-event.ts +2 -2
- package/src/api/responses/memory-v2-activation-log.ts +2 -2
- package/src/approvals/AGENTS.md +2 -0
- package/src/approvals/guardian-request-resolvers.ts +118 -24
- package/src/calls/__tests__/voice-control-protocol.test.ts +57 -0
- package/src/calls/__tests__/voice-session-bridge.test.ts +158 -4
- package/src/calls/__tests__/voice-triage-escalate.test.ts +3 -4
- package/src/calls/call-controller.ts +27 -27
- package/src/calls/call-conversation-messages.ts +58 -18
- package/src/calls/voice-control-protocol.ts +56 -0
- package/src/calls/voice-session-bridge.ts +156 -58
- package/src/calls/voice-triage-escalate.ts +2 -3
- package/src/channels/config.ts +8 -0
- package/src/channels/types.ts +1 -0
- package/src/cli/commands/config.ts +49 -13
- package/src/cli/commands/memory/index.help.ts +4 -2
- package/src/cli/commands/memory/memory-retrospective.ts +5 -0
- package/src/cli/lib/bundled-marketplace.json +12 -0
- package/src/cli.ts +2 -109
- package/src/config/__tests__/deployment-context-defaults.test.ts +9 -0
- package/src/config/__tests__/substrate-twin-shadowing.test.ts +191 -0
- package/src/config/assistant-feature-flags.ts +1 -1
- package/src/config/bundled-skills/image-studio/tools/media-generate-image.ts +13 -7
- package/src/config/feature-flag-cache.ts +6 -16
- package/src/config/feature-flag-registry.json +14 -30
- package/src/config/loader.ts +7 -1
- package/src/config/memory-tier.ts +15 -10
- package/src/config/memory-v3-gate.test.ts +278 -1
- package/src/config/memory-v3-gate.ts +88 -26
- package/src/config/raw-config-utils.ts +1 -1
- package/src/config/schemas/__tests__/live-voice.test.ts +27 -16
- package/src/config/schemas/__tests__/memory-v3.test.ts +1 -0
- package/src/config/schemas/live-voice.ts +33 -7
- package/src/config/schemas/llm.ts +1 -0
- package/src/config/schemas/memory-retrospective.ts +13 -0
- package/src/config/schemas/memory-substrate.ts +183 -0
- package/src/config/schemas/memory-v2.ts +3 -2
- package/src/config/schemas/memory-v3.ts +11 -4
- package/src/config/schemas/memory.ts +35 -1
- package/src/config/schemas/services.ts +10 -2
- package/src/config/substrate-twin-shadowing.ts +195 -0
- package/src/config/webhook-routing.ts +67 -0
- package/src/context/compactor.ts +10 -3
- package/src/context/token-estimator.ts +21 -0
- package/src/daemon/__tests__/channel-ui-capability.test.ts +85 -0
- package/src/daemon/__tests__/conversation-surfaces-launch.test.ts +3 -3
- package/src/daemon/__tests__/embedding-reconcile.test.ts +3 -3
- package/src/daemon/__tests__/inference-profile-notification.test.ts +3 -6
- package/src/daemon/__tests__/native-web-search-metadata.test.ts +5 -5
- package/src/daemon/__tests__/web-search-status-text.test.ts +2 -2
- package/src/daemon/channel-ui-capability.ts +70 -1
- package/src/daemon/conversation-agent-loop-handlers.ts +4 -4
- package/src/daemon/conversation-agent-loop.ts +12 -5
- package/src/daemon/conversation-evictor.ts +26 -4
- package/src/daemon/conversation-launch.ts +11 -14
- package/src/daemon/conversation-messaging.ts +14 -14
- package/src/daemon/conversation-notifiers.ts +2 -2
- package/src/daemon/conversation-process.ts +133 -21
- package/src/daemon/conversation-queue-manager.ts +7 -7
- package/src/daemon/conversation-runtime-assembly.ts +16 -0
- package/src/daemon/conversation-store.ts +11 -3
- package/src/daemon/conversation-surfaces.ts +4 -4
- package/src/daemon/conversation-tool-setup.ts +13 -7
- package/src/daemon/conversation-usage.ts +12 -11
- package/src/daemon/conversation.ts +38 -15
- package/src/daemon/disk-pressure-guard.ts +5 -12
- package/src/daemon/embedding-reconcile.ts +14 -9
- package/src/daemon/handlers/config-channels.ts +40 -134
- package/src/daemon/handlers/config-model.ts +15 -7
- package/src/daemon/handlers/conversations.ts +4 -2
- package/src/daemon/handlers/shared.ts +29 -0
- package/src/daemon/host-proxy-base.ts +4 -4
- package/src/daemon/lifecycle.ts +13 -0
- package/src/daemon/message-protocol.ts +8 -144
- package/src/daemon/message-types/computer-use.ts +0 -12
- package/src/daemon/message-types/conversations.ts +9 -345
- package/src/daemon/message-types/host-app-control.ts +0 -9
- package/src/daemon/message-types/host-browser.ts +0 -7
- package/src/daemon/message-types/host-ui-snapshot.ts +0 -7
- package/src/daemon/message-types/messages.ts +0 -48
- package/src/daemon/message-types/notifications.ts +0 -7
- package/src/daemon/message-types/skills.ts +0 -5
- package/src/daemon/message-types/surfaces.ts +0 -9
- package/src/daemon/message-types/sync.ts +0 -2
- package/src/daemon/process-message.ts +5 -5
- package/src/daemon/skill-memory-refresh.ts +5 -5
- package/src/daemon/tool-side-effects.ts +13 -13
- package/src/daemon/trust-context.ts +1 -1
- package/src/daemon/wake-conversation-ops.ts +4 -4
- package/src/heartbeat/__tests__/heartbeat-run-store.test.ts +100 -7
- package/src/heartbeat/__tests__/heartbeat-service.test.ts +67 -0
- package/src/heartbeat/heartbeat-run-store.ts +81 -4
- package/src/heartbeat/heartbeat-service.ts +32 -1
- package/src/home/__tests__/feed-writer.test.ts +7 -7
- package/src/home/__tests__/home-content-refresh.test.ts +3 -0
- package/src/home/__tests__/suggested-prompts.test.ts +1 -0
- package/src/home/feed-writer.ts +6 -13
- package/src/home/home-content-refresh.ts +6 -9
- package/src/home/relationship-state-writer.ts +5 -12
- package/src/home/suggested-prompts-cache.ts +6 -19
- package/src/ipc/__tests__/events-publish-client.test.ts +67 -0
- package/src/ipc/events-publish-client.ts +44 -0
- package/src/ipc/routes/__tests__/conversation-sync-ipc-routes.test.ts +2 -2
- package/src/ipc/routes/__tests__/events-ipc-routes.test.ts +38 -9
- package/src/ipc/routes/events-ipc-routes.ts +11 -31
- package/src/live-voice/__tests__/front-decision.test.ts +4 -176
- package/src/live-voice/__tests__/live-voice-agent-turn.test.ts +276 -68
- package/src/live-voice/__tests__/live-voice-events.test.ts +6 -3
- package/src/live-voice/__tests__/live-voice-progress.test.ts +185 -46
- package/src/live-voice/__tests__/live-voice-triage-escalate.test.ts +24 -59
- package/src/live-voice/__tests__/live-voice-tts-session.test.ts +38 -54
- package/src/live-voice/__tests__/live-voice-vad.test.ts +218 -422
- package/src/live-voice/front-decision.ts +5 -155
- package/src/live-voice/live-voice-metrics.ts +4 -1
- package/src/live-voice/live-voice-session.ts +650 -333
- package/src/live-voice/progress-phrases.ts +17 -0
- package/src/live-voice/protocol.ts +15 -0
- package/src/media/app-icon-generator.ts +15 -8
- package/src/media/avatar-router.ts +9 -5
- package/src/media/image-credentials.ts +42 -10
- package/src/media/image-service.ts +1 -13
- package/src/media/types.ts +26 -3
- package/src/notifications/AGENTS.md +2 -0
- package/src/notifications/__tests__/broadcaster.test.ts +95 -0
- package/src/notifications/adapters/macos.ts +3 -3
- package/src/notifications/broadcaster.ts +57 -0
- package/src/notifications/copy-composer.ts +0 -15
- package/src/notifications/decision-engine.ts +21 -22
- package/src/notifications/guardian-question-mode.ts +57 -2
- package/src/notifications/home-feed-side-effect.ts +4 -0
- package/src/notifications/signal.ts +6 -4
- package/src/notifications/trusted-contact-payloads.ts +3 -4
- package/src/notifications/vellum-card-affinity.ts +39 -0
- package/src/oauth/AGENTS.md +1 -1
- package/src/permissions/channel-permission-query.test.ts +69 -0
- package/src/permissions/channel-permission-query.ts +73 -1
- package/src/permissions/checker.test.ts +85 -9
- package/src/permissions/checker.ts +17 -22
- package/src/permissions/confirmation-guardian-request.test.ts +2 -2
- package/src/permissions/confirmation-guardian-request.ts +2 -2
- package/src/permissions/gateway-threshold-reader.test.ts +224 -17
- package/src/permissions/gateway-threshold-reader.ts +82 -10
- package/src/permissions/ipc-risk-types.ts +8 -0
- package/src/permissions/prompter.ts +5 -5
- package/src/permissions/question-guardian-request.test.ts +222 -0
- package/src/permissions/question-guardian-request.ts +179 -0
- package/src/permissions/question-prompter.test.ts +8 -8
- package/src/permissions/question-prompter.ts +18 -0
- package/src/permissions/secret-prompter.ts +2 -4
- package/src/permissions/workspace-policy.ts +236 -34
- package/src/persistence/__tests__/lifecycle-quiesce.test.ts +82 -0
- package/src/persistence/__tests__/message-content-file.test.ts +58 -3
- package/src/persistence/conversation-crud.ts +93 -0
- package/src/persistence/conversation-key-store.ts +63 -0
- package/src/persistence/db-async-query.ts +1 -1
- package/src/persistence/db-singleton.ts +14 -27
- package/src/persistence/delivery-crud.ts +45 -0
- package/src/persistence/embeddings/embedding-backend.ts +2 -1
- package/src/persistence/embeddings/embedding-identity.ts +7 -3
- package/src/persistence/job-utils.ts +32 -27
- package/src/persistence/jobs-store.ts +92 -3
- package/src/persistence/lifecycle-quiesce.ts +80 -0
- package/src/persistence/llm-usage-store.ts +45 -8
- package/src/persistence/message-content-file.ts +4 -4
- package/src/persistence/migrations/336-move-memory-v2-activation-logs-to-memory-db.ts +1 -1
- package/src/persistence/migrations/351-schedule-skill-script-handoff.ts +39 -0
- package/src/persistence/migrations/352-drop-schedule-skill-script-handoff.test.ts +83 -0
- package/src/persistence/migrations/352-drop-schedule-skill-script-handoff.ts +36 -0
- package/src/persistence/migrations/353-add-llm-usage-conversation-type.test.ts +110 -0
- package/src/persistence/migrations/353-add-llm-usage-conversation-type.ts +49 -0
- package/src/persistence/schema/infrastructure.ts +11 -0
- package/src/persistence/steps.ts +7 -1
- package/src/plugin-api/__tests__/publish-event-forwarding.test.ts +113 -0
- package/src/plugin-api/__tests__/publish-event.test.ts +57 -0
- package/src/plugin-api/conversation-turn.ts +4 -4
- package/src/plugin-api/event-hub-facade.ts +15 -6
- package/src/plugin-api/index.ts +10 -4
- package/src/plugin-api/publish-event.ts +36 -0
- package/src/plugins/AGENTS.md +2 -0
- package/src/plugins/defaults/compaction/window-manager.ts +9 -0
- package/src/plugins/defaults/index.ts +9 -13
- package/src/plugins/defaults/memory/AGENTS.md +775 -0
- package/src/plugins/defaults/memory/__tests__/capability-seed-embed-gate.test.ts +186 -0
- package/src/plugins/defaults/memory/__tests__/fixtures/memory-v2-activation-fixtures.ts +1 -1
- package/src/plugins/defaults/memory/__tests__/indexer-tier-dispatch.test.ts +129 -0
- package/src/plugins/defaults/memory/__tests__/jobs-worker-quiesce-gate.test.ts +91 -0
- package/src/plugins/defaults/memory/__tests__/jobs-worker-v1-graph-substrate-noop.test.ts +342 -0
- package/src/plugins/defaults/memory/__tests__/jobs-worker-v2-graph-trigger-embed.test.ts +9 -1
- package/src/plugins/defaults/memory/__tests__/jobs-worker-v2-schedule.test.ts +36 -1
- package/src/plugins/defaults/memory/__tests__/jobs-worker-v2-sweep-defer.test.ts +9 -1
- package/src/plugins/defaults/memory/__tests__/memory-log-stores-degraded.test.ts +2 -2
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-job.test.ts +44 -10
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-sweep.test.ts +151 -10
- package/src/plugins/defaults/memory/__tests__/memory-tier-boundary-guard.test.ts +1155 -0
- package/src/plugins/defaults/memory/__tests__/memory-v2-activation-log-store.test.ts +1 -1
- package/src/plugins/defaults/memory/__tests__/memory-v2-concept-frequency.test.ts +3 -3
- package/src/plugins/defaults/memory/__tests__/tools-delete-memory-page.test.ts +1 -1
- package/src/plugins/defaults/memory/context-search/sources/memory-v2.ts +71 -35
- package/src/plugins/defaults/memory/context-search/sources/memory.ts +5 -1
- package/src/plugins/defaults/memory/fork-conversation-memory.ts +4 -4
- package/src/plugins/defaults/memory/graph/__tests__/conversation-graph-memory-v2-routing.test.ts +80 -10
- package/src/plugins/defaults/memory/graph/__tests__/handle-remember-v2.test.ts +1 -1
- package/src/plugins/defaults/memory/graph/capability-seed.ts +24 -10
- package/src/plugins/defaults/memory/graph/conversation-graph-memory.ts +123 -49
- package/src/plugins/defaults/memory/graph/graph-search.ts +6 -118
- package/src/plugins/defaults/memory/graph/in-context-tracker.test.ts +173 -0
- package/src/plugins/defaults/memory/graph/in-context-tracker.ts +109 -0
- package/src/plugins/defaults/memory/graph/store.ts +1 -0
- package/src/plugins/defaults/memory/graph/tool-handlers.test.ts +1 -1
- package/src/plugins/defaults/memory/graph/tool-handlers.ts +5 -2
- package/src/plugins/defaults/memory/graph-topology/__tests__/build-memory-graph.test.ts +1 -1
- package/src/plugins/defaults/memory/graph-topology/build-memory-graph.ts +6 -9
- package/src/plugins/defaults/memory/hooks/__tests__/user-prompt-submit.test.ts +30 -16
- package/src/plugins/defaults/memory/hooks/user-prompt-submit.ts +18 -14
- package/src/plugins/defaults/memory/indexer.ts +146 -89
- package/src/plugins/defaults/memory/injectors.ts +86 -41
- package/src/plugins/defaults/memory/job-handlers.ts +130 -39
- package/src/plugins/defaults/memory/jobs/__tests__/embed-concept-page.test.ts +2 -2
- package/src/plugins/defaults/memory/jobs/embed-concept-page.ts +22 -10
- package/src/plugins/defaults/memory/jobs-worker.ts +279 -208
- package/src/plugins/defaults/memory/memory-retrospective-enqueue.ts +4 -2
- package/src/plugins/defaults/memory/memory-retrospective-job.ts +40 -3
- package/src/plugins/defaults/memory/memory-retrospective-sweep.ts +99 -11
- package/src/plugins/defaults/memory/src/__tests__/memory-v2-routes.test.ts +5 -3
- package/src/plugins/defaults/memory/src/__tests__/memory-v2-simulate-route.test.ts +3 -4
- package/src/plugins/defaults/memory/src/memory-graph-routes.ts +2 -1
- package/src/plugins/defaults/memory/src/memory-item-routes.test.ts +7 -5
- package/src/plugins/defaults/memory/src/memory-item-routes.ts +51 -19
- package/src/plugins/defaults/memory/src/memory-v2-routes.ts +31 -20
- package/src/plugins/defaults/memory/startup.ts +58 -21
- package/src/plugins/defaults/memory/substrate/__tests__/boot-maintenance.test.ts +335 -0
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/cli-command-store.test.ts +14 -8
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/consolidation-job.test.ts +51 -21
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/edge-index.test.ts +1 -1
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/frontmatter-sweep.test.ts +3 -3
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/page-index.test.ts +1 -1
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/page-store.test.ts +1 -1
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/prompts-consolidation.test.ts +2 -2
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/qdrant.test.ts +23 -14
- package/src/plugins/defaults/memory/substrate/__tests__/reembed-job.test.ts +262 -0
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/sim.test.ts +9 -12
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/skill-content.test.ts +2 -2
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/skill-store.test.ts +9 -9
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/sparse-bm25.test.ts +12 -5
- package/src/plugins/defaults/memory/substrate/__tests__/spread.test.ts +276 -0
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/static-context.test.ts +17 -8
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/sweep-job.test.ts +9 -9
- package/src/plugins/defaults/memory/{v3/substrate/memory-v2-startup.ts → substrate/boot-maintenance.ts} +43 -24
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/cli-command-store.ts +13 -8
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/consolidation-job.ts +51 -27
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/constants.ts +1 -0
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/edge-index.ts +28 -9
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/frontmatter-sweep.ts +14 -7
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/injected-block-slugs.ts +9 -3
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/page-index.ts +29 -10
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/page-store.ts +35 -12
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/prompts/consolidation.ts +14 -8
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/prompts/sweep.ts +4 -1
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/qdrant.ts +96 -36
- package/src/plugins/defaults/memory/substrate/reembed-job.ts +54 -0
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/sim.ts +26 -15
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/skill-content.ts +1 -1
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/skill-store.ts +13 -8
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/sparse-bm25.ts +27 -10
- package/src/plugins/defaults/memory/substrate/spread.ts +165 -0
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/static-context.ts +12 -7
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/sweep-job.ts +20 -15
- package/src/plugins/defaults/memory/substrate/tuning.ts +93 -0
- package/src/plugins/defaults/memory/{v3/substrate → substrate}/types.ts +7 -4
- package/src/plugins/defaults/memory/tools.test.ts +1 -1
- package/src/plugins/defaults/memory/tools.ts +1 -1
- package/src/plugins/defaults/memory/v1/README.md +5 -0
- package/src/plugins/defaults/memory/{__tests__ → v1/__tests__}/filing-jobs.test.ts +4 -4
- package/src/plugins/defaults/memory/{filing-jobs.ts → v1/filing-jobs.ts} +3 -3
- package/src/plugins/defaults/memory/{graph → v1/graph}/bootstrap.test.ts +68 -6
- package/src/plugins/defaults/memory/{graph → v1/graph}/bootstrap.ts +40 -21
- package/src/plugins/defaults/memory/{graph → v1/graph}/consolidation.ts +14 -14
- package/src/plugins/defaults/memory/{graph → v1/graph}/decay.test.ts +1 -1
- package/src/plugins/defaults/memory/{graph → v1/graph}/decay.ts +3 -3
- package/src/plugins/defaults/memory/{graph → v1/graph}/extraction-job.ts +5 -5
- package/src/plugins/defaults/memory/{graph → v1/graph}/extraction.test.ts +1 -1
- package/src/plugins/defaults/memory/{graph → v1/graph}/extraction.ts +20 -14
- package/src/plugins/defaults/memory/{graph → v1/graph}/graph-search.test.ts +3 -3
- package/src/plugins/defaults/memory/v1/graph/graph-search.ts +123 -0
- package/src/plugins/defaults/memory/{graph → v1/graph}/injection.test.ts +1 -118
- package/src/plugins/defaults/memory/{graph → v1/graph}/injection.ts +9 -111
- package/src/plugins/defaults/memory/{graph → v1/graph}/narrative.ts +6 -6
- package/src/plugins/defaults/memory/{graph → v1/graph}/pattern-scan.ts +5 -5
- package/src/plugins/defaults/memory/{graph → v1/graph}/retriever.test.ts +135 -42
- package/src/plugins/defaults/memory/{graph → v1/graph}/retriever.ts +40 -22
- package/src/plugins/defaults/memory/{graph → v1/graph}/scoring.test.ts +1 -1
- package/src/plugins/defaults/memory/{graph → v1/graph}/scoring.ts +1 -1
- package/src/plugins/defaults/memory/{graph → v1/graph}/serendipity.ts +1 -1
- package/src/plugins/defaults/memory/{graph → v1/graph}/triggers.test.ts +1 -1
- package/src/plugins/defaults/memory/{graph → v1/graph}/triggers.ts +1 -1
- package/src/plugins/defaults/memory/{identity-context.ts → v1/identity-context.ts} +4 -4
- package/src/plugins/defaults/memory/{job-handlers → v1/job-handlers}/backfill.ts +6 -6
- package/src/plugins/defaults/memory/{job-handlers → v1/job-handlers}/embedding.test.ts +6 -6
- package/src/plugins/defaults/memory/{job-handlers → v1/job-handlers}/embedding.ts +6 -6
- package/src/plugins/defaults/memory/{job-handlers → v1/job-handlers}/index-maintenance.ts +8 -8
- package/src/plugins/defaults/memory/{jobs → v1/jobs}/embed-pkb-file.test.ts +7 -7
- package/src/plugins/defaults/memory/{jobs → v1/jobs}/embed-pkb-file.ts +5 -5
- package/src/plugins/defaults/memory/{pkb → v1/pkb}/autoinject.ts +1 -1
- package/src/plugins/defaults/memory/{pkb → v1/pkb}/context.ts +2 -2
- package/src/plugins/defaults/memory/{pkb → v1/pkb}/pkb-index.test.ts +3 -3
- package/src/plugins/defaults/memory/{pkb → v1/pkb}/pkb-index.ts +2 -2
- package/src/plugins/defaults/memory/{pkb → v1/pkb}/pkb-reconcile.test.ts +3 -3
- package/src/plugins/defaults/memory/{pkb → v1/pkb}/pkb-reconcile.ts +2 -2
- package/src/plugins/defaults/memory/{pkb → v1/pkb}/pkb-search.test.ts +3 -3
- package/src/plugins/defaults/memory/{pkb → v1/pkb}/pkb-search.ts +6 -6
- package/src/plugins/defaults/memory/{pkb → v1/pkb}/types.ts +3 -5
- package/src/plugins/defaults/memory/{pkb-schedule.ts → v1/pkb-schedule.ts} +2 -2
- package/src/plugins/defaults/memory/{search/semantic.ts → v1/semantic-search.ts} +1 -1
- package/src/plugins/defaults/memory/v2/__tests__/activation-store.test.ts +1 -1
- package/src/plugins/defaults/memory/v2/__tests__/activation.test.ts +11 -274
- package/src/plugins/defaults/memory/v2/__tests__/backfill-jobs.test.ts +9 -131
- package/src/plugins/defaults/memory/v2/__tests__/harness-compare.test.ts +1 -1
- package/src/plugins/defaults/memory/v2/__tests__/harness-oracle.test.ts +1 -1
- package/src/plugins/defaults/memory/v2/__tests__/harness-replay-input.test.ts +1 -1
- package/src/plugins/defaults/memory/v2/__tests__/injection.test.ts +14 -10
- package/src/plugins/defaults/memory/v2/__tests__/reranker.test.ts +7 -5
- package/src/plugins/defaults/memory/v2/__tests__/router.test.ts +12 -6
- package/src/plugins/defaults/memory/{memory-v2-activation-log-store.ts → v2/activation-log-store.ts} +2 -2
- package/src/plugins/defaults/memory/v2/activation-store.ts +22 -8
- package/src/plugins/defaults/memory/v2/activation.ts +30 -163
- package/src/plugins/defaults/memory/v2/backfill-jobs.ts +7 -54
- package/src/plugins/defaults/memory/{memory-v2-concept-frequency.ts → v2/concept-frequency.ts} +3 -3
- package/src/plugins/defaults/memory/v2/harness/oracle.ts +1 -1
- package/src/plugins/defaults/memory/v2/harness/replay-input.ts +2 -2
- package/src/plugins/defaults/memory/v2/harness/retriever.ts +1 -1
- package/src/plugins/defaults/memory/v2/injection.ts +36 -26
- package/src/plugins/defaults/memory/v2/migration.ts +2 -2
- package/src/plugins/defaults/memory/{rerank-local.ts → v2/rerank-local.ts} +44 -18
- package/src/plugins/defaults/memory/v2/reranker.ts +29 -11
- package/src/plugins/defaults/memory/v2/router.ts +38 -15
- package/src/plugins/defaults/memory/v3/__tests__/candidate-match.test.ts +5 -3
- package/src/plugins/defaults/memory/v3/__tests__/carry-integration.test.ts +21 -8
- package/src/plugins/defaults/memory/v3/__tests__/edge.test.ts +1 -1
- package/src/plugins/defaults/memory/v3/__tests__/live-integration.test.ts +1 -1
- package/src/plugins/defaults/memory/v3/__tests__/maintain-job.test.ts +4 -1
- package/src/plugins/defaults/memory/v3/__tests__/orchestrate.test.ts +181 -3
- package/src/plugins/defaults/memory/v3/__tests__/selection-log-store.test.ts +3 -0
- package/src/plugins/defaults/memory/v3/__tests__/shadow-integration.test.ts +14 -5
- package/src/plugins/defaults/memory/v3/__tests__/shadow-plugin.test.ts +40 -50
- package/src/plugins/defaults/memory/v3/__tests__/tuning-profile.test.ts +1 -0
- package/src/plugins/defaults/memory/v3/candidate-match.ts +2 -2
- package/src/plugins/defaults/memory/v3/capabilities.ts +6 -4
- package/src/plugins/defaults/memory/v3/card.ts +25 -9
- package/src/plugins/defaults/memory/v3/edge.ts +52 -18
- package/src/plugins/defaults/memory/v3/gate.ts +4 -4
- package/src/plugins/defaults/memory/v3/injector.ts +10 -3
- package/src/plugins/defaults/memory/v3/maintain-job.ts +3 -3
- package/src/plugins/defaults/memory/v3/orchestrate.ts +100 -18
- package/src/plugins/defaults/memory/v3/page-content.ts +26 -10
- package/src/plugins/defaults/memory/v3/prune.ts +40 -16
- package/src/plugins/defaults/memory/v3/selection-log-store.ts +34 -12
- package/src/plugins/defaults/memory/v3/shadow-plugin.ts +9 -13
- package/src/plugins/defaults/memory/v3/span-query.test.ts +86 -0
- package/src/plugins/defaults/memory/v3/span-query.ts +71 -0
- package/src/plugins/defaults/memory/v3/tuning-profile.ts +3 -0
- package/src/plugins/defaults/memory/v3/types.ts +4 -1
- package/src/plugins/defaults/memory/v3-eval/eval-packets.ts +2 -2
- package/src/plugins/defaults/memory/validation.ts +1 -1
- package/src/plugins/defaults/memory/worker.ts +2 -1
- package/src/plugins/defaults/platform-hosted/__tests__/reengage.test.ts +104 -12
- package/src/plugins/defaults/platform-hosted/routes/reengage.ts +104 -12
- package/src/plugins/types.ts +2 -2
- package/src/providers/__tests__/registry-native-web-search.test.ts +47 -2
- package/src/providers/anthropic/client.ts +15 -6
- package/src/providers/content-block-schema.ts +26 -0
- package/src/providers/inference/__tests__/connections-credential-normalize.test.ts +87 -0
- package/src/providers/inference/adapter-factory.ts +6 -0
- package/src/providers/inference/connections.ts +30 -14
- package/src/providers/model-catalog.ts +77 -0
- package/src/providers/openai/__tests__/chat-completions-input-audio.test.ts +260 -0
- package/src/providers/openai/chat-completions-provider.ts +65 -17
- package/src/providers/openai/input-audio.ts +76 -0
- package/src/providers/openrouter/client.ts +0 -2
- package/src/providers/platform-proxy/constants.ts +4 -0
- package/src/providers/poolside/client.ts +42 -0
- package/src/providers/registry.ts +40 -8
- package/src/providers/retry.ts +2 -0
- package/src/providers/types.ts +39 -1
- package/src/providers/vercel-ai-gateway/client.ts +0 -2
- package/src/runtime/AGENTS.md +7 -5
- package/src/runtime/__tests__/question-request-guardian-bridge.test.ts +173 -0
- package/src/runtime/__tests__/question-resolution.test.ts +111 -0
- package/src/runtime/access-request-helper.ts +7 -15
- package/src/runtime/assistant-event-hub.ts +43 -31
- package/src/runtime/assistant-event-publish-options.ts +31 -0
- package/src/runtime/assistant-event.ts +19 -30
- package/src/runtime/assistant-stream-state.ts +4 -4
- package/src/runtime/channel-approvals.ts +0 -25
- package/src/runtime/channel-readiness-service.ts +99 -40
- package/src/runtime/channel-retry-sweep.ts +38 -15
- package/src/runtime/confirmation-request-guardian-bridge.ts +4 -0
- package/src/runtime/guardian-reply-router.ts +108 -0
- package/src/runtime/http-server.ts +8 -0
- package/src/runtime/http-types.ts +2 -2
- package/src/runtime/process-role.ts +42 -0
- package/src/runtime/question-request-guardian-bridge.ts +184 -0
- package/src/runtime/question-resolution.ts +153 -0
- package/src/runtime/routes/__tests__/bookmark-routes.test.ts +6 -2
- package/src/runtime/routes/__tests__/conversation-query-routes.test.ts +1 -1
- package/src/runtime/routes/acp-routes.ts +2 -2
- package/src/runtime/routes/bookmark-routes.ts +3 -15
- package/src/runtime/routes/canned-message-complete.ts +2 -2
- package/src/runtime/routes/consolidation-routes.ts +8 -2
- package/src/runtime/routes/conversation-list-routes.ts +3 -19
- package/src/runtime/routes/conversation-management-routes.ts +1 -5
- package/src/runtime/routes/conversation-query-routes.ts +52 -8
- package/src/runtime/routes/conversation-routes.ts +36 -22
- package/src/runtime/routes/credential-request-routes.ts +0 -1
- package/src/runtime/routes/filing-routes.ts +7 -7
- package/src/runtime/routes/global-search-routes.ts +1 -1
- package/src/runtime/routes/guardian-approval-prompt.ts +2 -2
- package/src/runtime/routes/image-generation-routes.ts +12 -9
- package/src/runtime/routes/inbound-message-handler.ts +17 -0
- package/src/runtime/routes/inbound-stages/acl-enforcement.test.ts +50 -0
- package/src/runtime/routes/inbound-stages/acl-enforcement.ts +19 -0
- package/src/runtime/routes/inbound-stages/background-dispatch.ts +4 -4
- package/src/runtime/routes/inbound-stages/inbound-content-prep.test.ts +129 -0
- package/src/runtime/routes/inbound-stages/inbound-content-prep.ts +141 -9
- package/src/runtime/routes/index.ts +2 -0
- package/src/runtime/routes/lifecycle-routes.ts +199 -0
- package/src/runtime/routes/platform-routes.ts +3 -8
- package/src/runtime/routes/playground/__tests__/inject-failures.test.ts +7 -7
- package/src/runtime/routes/playground/__tests__/reset-circuit.test.ts +5 -5
- package/src/runtime/routes/question-routes.ts +34 -87
- package/src/runtime/routes/secret-routes.ts +11 -1
- package/src/runtime/routes/settings-routes.ts +8 -18
- package/src/runtime/routes/upgrade-broadcast-routes.ts +4 -5
- package/src/runtime/routes/workspace-routes.test.ts +7 -4
- package/src/runtime/slack-reply-session.test.ts +12 -12
- package/src/runtime/slack-reply-session.ts +17 -17
- package/src/runtime/sync/sync-publisher.test.ts +7 -5
- package/src/runtime/tool-grant-request-helper.ts +4 -0
- package/src/schedule/schedule-store.ts +70 -0
- package/src/schedule/scheduler.ts +27 -0
- package/src/security/AGENTS.md +1 -1
- package/src/security/credential-key.ts +17 -0
- package/src/security/secret-ingress.ts +14 -105
- package/src/security/secret-patterns.ts +8 -139
- package/src/security/secret-scanner.ts +8 -3
- package/src/security/store-path-override.ts +6 -16
- package/src/sequence/store.ts +8 -0
- package/src/signals/emit-event.ts +5 -10
- package/src/signals/event-stream.ts +20 -10
- package/src/subagent/manager.ts +76 -25
- package/src/subagent/notify.ts +8 -0
- package/src/subagent/types.ts +13 -3
- package/src/telegram/__tests__/webhook-health.test.ts +442 -0
- package/src/telegram/webhook-health.ts +478 -0
- package/src/telemetry/AGENTS.md +6 -0
- package/src/telemetry/memory-corpus-size.test.ts +129 -0
- package/src/telemetry/memory-corpus-size.ts +184 -0
- package/src/telemetry/memory-tier-reporter.test.ts +67 -2
- package/src/telemetry/memory-tier-reporter.ts +30 -3
- package/src/tools/__tests__/document-tool-input-schemas.test.ts +103 -0
- package/src/tools/__tests__/followup-call-acp-tool-input-schemas.test.ts +125 -0
- package/src/tools/__tests__/schedule-tool-input-schemas.test.ts +83 -0
- package/src/tools/__tests__/subagent-workflow-tool-input-schemas.test.ts +116 -0
- package/src/tools/__tests__/tool-input-schemas.test.ts +192 -0
- package/src/tools/__tests__/tool-schema-drift-harness.ts +142 -0
- package/src/tools/acp/abort.ts +21 -1
- package/src/tools/acp/context.ts +5 -5
- package/src/tools/acp/spawn.test.ts +15 -5
- package/src/tools/acp/spawn.ts +26 -3
- package/src/tools/acp/status.ts +20 -1
- package/src/tools/acp/steer.test.ts +20 -0
- package/src/tools/acp/steer.ts +23 -2
- package/src/tools/ask-question/ask-question-tool.test.ts +65 -0
- package/src/tools/ask-question/ask-question-tool.ts +68 -92
- package/src/tools/browser/browser-screencast.ts +6 -6
- package/src/tools/calls/call-end.ts +24 -3
- package/src/tools/calls/call-start.ts +35 -11
- package/src/tools/calls/call-status.ts +20 -1
- package/src/tools/document/document-comment-tool.test.ts +49 -0
- package/src/tools/document/document-comment-tool.ts +45 -6
- package/src/tools/document/document-tool.ts +142 -24
- package/src/tools/executor.ts +27 -13
- package/src/tools/filesystem/edit.ts +62 -60
- package/src/tools/filesystem/list.ts +36 -29
- package/src/tools/filesystem/read.ts +56 -39
- package/src/tools/filesystem/write.ts +44 -38
- package/src/tools/followups/followup_create.ts +41 -12
- package/src/tools/followups/followup_list.ts +34 -6
- package/src/tools/followups/followup_resolve.ts +26 -4
- package/src/tools/host-filesystem/edit.ts +50 -60
- package/src/tools/host-filesystem/read.ts +45 -39
- package/src/tools/host-filesystem/write.ts +31 -36
- package/src/tools/host-terminal/host-shell.ts +76 -62
- package/src/tools/policy-context.ts +27 -10
- package/src/tools/schedule/create.ts +76 -33
- package/src/tools/schedule/delete.ts +21 -2
- package/src/tools/schedule/list.ts +25 -3
- package/src/tools/schedule/update.ts +105 -48
- package/src/tools/shared/filesystem/path-policy.ts +246 -40
- package/src/tools/shared/zod-tool-schema.ts +117 -0
- package/src/tools/subagent/abort.ts +12 -4
- package/src/tools/subagent/consult-deadline.ts +9 -3
- package/src/tools/subagent/message.ts +21 -5
- package/src/tools/subagent/notify-parent.ts +44 -23
- package/src/tools/subagent/read.ts +15 -4
- package/src/tools/subagent/resolve.ts +26 -4
- package/src/tools/subagent/spawn.ts +37 -13
- package/src/tools/subagent/status.ts +12 -4
- package/src/tools/system/request-permission.ts +35 -26
- package/src/tools/terminal/shell.ts +68 -55
- package/src/tools/tool-approval-handler.ts +327 -53
- package/src/tools/tool-input-schemas.ts +85 -0
- package/src/tools/tool-types.ts +7 -0
- package/src/tools/types.ts +8 -0
- package/src/tools/workflows/manage-workflows.ts +30 -2
- package/src/tools/workflows/run-workflow.test.ts +54 -2
- package/src/tools/workflows/run-workflow.ts +32 -4
- package/src/util/fs-symlinks.ts +30 -0
- package/src/util/log-redact.ts +7 -3
- package/src/vellum-assistant-namespace.d.ts +80 -0
- package/src/watcher/watcher-store.ts +8 -0
- package/src/workflows/leaf-runner.test.ts +104 -2
- package/src/workflows/leaf-runner.ts +31 -6
- package/src/workspace/migrations/070-memory-v2-summary-schema-rebuild.ts +2 -2
- package/src/workspace/migrations/134-image-generation-mode-to-provider.ts +128 -0
- package/src/workspace/migrations/135-copy-substrate-tunables.ts +248 -0
- package/src/workspace/migrations/__tests__/135-copy-substrate-tunables.test.ts +446 -0
- package/src/workspace/migrations/__tests__/backfill-v3-leaves.test.ts +2 -2
- package/src/workspace/migrations/registry.ts +4 -0
- package/src/__tests__/memory-retrieval-budget.test.ts +0 -49
- package/src/__tests__/secret-patterns.test.ts +0 -27
- package/src/config/__tests__/proc-to-skills.test.ts +0 -21
- package/src/config/memory-tier.test.ts +0 -44
- package/src/daemon/message-types/acp.ts +0 -20
- package/src/daemon/message-types/apps.ts +0 -11
- package/src/daemon/message-types/background-tools.ts +0 -14
- package/src/daemon/message-types/bookmarks.ts +0 -13
- package/src/daemon/message-types/contacts.ts +0 -13
- package/src/daemon/message-types/document-comments.ts +0 -14
- package/src/daemon/message-types/documents.ts +0 -13
- package/src/daemon/message-types/home.ts +0 -15
- package/src/daemon/message-types/host-bash.ts +0 -12
- package/src/daemon/message-types/host-cu.ts +0 -10
- package/src/daemon/message-types/host-file.ts +0 -12
- package/src/daemon/message-types/host-transfer.ts +0 -12
- package/src/daemon/message-types/integrations.ts +0 -90
- package/src/daemon/message-types/memory.ts +0 -10
- package/src/daemon/message-types/schedules.ts +0 -14
- package/src/daemon/message-types/settings.ts +0 -17
- package/src/daemon/message-types/subagents.ts +0 -63
- package/src/daemon/message-types/upgrades.ts +0 -14
- package/src/daemon/message-types/workflows.ts +0 -25
- package/src/daemon/message-types/workspace.ts +0 -137
- package/src/live-voice/ack-phrases.ts +0 -58
- package/src/plugins/defaults/memory/retrieval-budget.ts +0 -29
- package/src/plugins/defaults/memory/v3/__tests__/gate-flag.test.ts +0 -69
- package/src/plugins/defaults/memory/v3/gate-flag.ts +0 -20
- package/src/plugins/defaults/memory/v3/substrate/__tests__/memory-v2-startup.test.ts +0 -86
- package/src/plugins/defaults/memory/v3/substrate/memory-v2-startup.test.ts +0 -205
- /package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/concept-page-frontmatter-schema.test.ts +0 -0
- /package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/consolidation-prompt-flag-gating-guard.test.ts +0 -0
- /package/src/plugins/defaults/memory/{v3/substrate → substrate}/__tests__/injected-block-slugs.test.ts +0 -0
- /package/src/plugins/defaults/memory/{v3/substrate → substrate}/cli-command-content.ts +0 -0
|
@@ -7,7 +7,8 @@ import {
|
|
|
7
7
|
} from "../calls/media-turn-detector.js";
|
|
8
8
|
import { sanitizeForTts } from "../calls/tts-text-sanitizer.js";
|
|
9
9
|
import {
|
|
10
|
-
|
|
10
|
+
isIncompleteControlMarkerTail,
|
|
11
|
+
MINIMIZE_ROOM_MARKER,
|
|
11
12
|
stripInternalSpeechMarkers,
|
|
12
13
|
} from "../calls/voice-control-protocol.js";
|
|
13
14
|
import type {
|
|
@@ -32,13 +33,12 @@ import {
|
|
|
32
33
|
} from "../calls/voice-triage-escalate.js";
|
|
33
34
|
import { isAssistantFeatureFlagEnabled } from "../config/assistant-feature-flags.js";
|
|
34
35
|
import { getConfig } from "../config/loader.js";
|
|
35
|
-
import type { AssistantConfig } from "../config/schema.js";
|
|
36
36
|
import {
|
|
37
37
|
type LiveVoiceFrontModelConfig,
|
|
38
38
|
LiveVoiceFrontModelConfigSchema,
|
|
39
39
|
} from "../config/schemas/live-voice.js";
|
|
40
40
|
import { ABORT_WATCHDOG_MS } from "../daemon/abort-watchdog.js";
|
|
41
|
-
import {
|
|
41
|
+
import { isRefusedInReadOnlyPass } from "../daemon/conversation-tool-setup.js";
|
|
42
42
|
import type { TrustContext } from "../daemon/trust-context-types.js";
|
|
43
43
|
import { ensureConversationExists } from "../persistence/conversation-crud.js";
|
|
44
44
|
import {
|
|
@@ -46,6 +46,7 @@ import {
|
|
|
46
46
|
supportsBoundary,
|
|
47
47
|
} from "../providers/speech-to-text/provider-catalog.js";
|
|
48
48
|
import type { ResolveStreamingTranscriberOptions } from "../providers/speech-to-text/resolve.js";
|
|
49
|
+
import { broadcastMessage } from "../runtime/assistant-event-hub.js";
|
|
49
50
|
import { publishConversationListAndMetadataChanged } from "../runtime/sync/resource-sync-events.js";
|
|
50
51
|
import { detectPcm16SpeechActivity } from "../stt/speech-energy.js";
|
|
51
52
|
import type {
|
|
@@ -54,13 +55,12 @@ import type {
|
|
|
54
55
|
SttStreamServerEvent,
|
|
55
56
|
} from "../stt/types.js";
|
|
56
57
|
import { getSubagentManager } from "../subagent/index.js";
|
|
58
|
+
import { getToolOwner } from "../tools/registry.js";
|
|
57
59
|
import { extractSpeakableSegments } from "../tts/speakable-segments.js";
|
|
58
60
|
import { createAbortReason } from "../util/abort-reasons.js";
|
|
59
61
|
import { getLogger } from "../util/logger.js";
|
|
60
|
-
import { pickAckPhrase, pickProgressPhrase } from "./ack-phrases.js";
|
|
61
62
|
import {
|
|
62
63
|
createVoiceFrontDecider,
|
|
63
|
-
type VoiceEndpointAction,
|
|
64
64
|
type VoiceFrontDecider,
|
|
65
65
|
type VoiceProgressTextInput,
|
|
66
66
|
} from "./front-decision.js";
|
|
@@ -76,6 +76,7 @@ import {
|
|
|
76
76
|
type LiveVoiceMetricsEvent,
|
|
77
77
|
type LiveVoiceSpokenAckKind,
|
|
78
78
|
type LiveVoiceTurnSeedMarks,
|
|
79
|
+
type VoiceEndpointAction,
|
|
79
80
|
} from "./live-voice-metrics.js";
|
|
80
81
|
import {
|
|
81
82
|
type LiveVoiceSession as LiveVoiceSessionContract,
|
|
@@ -88,6 +89,7 @@ import type {
|
|
|
88
89
|
LiveVoiceTtsOptions,
|
|
89
90
|
LiveVoiceTtsResult,
|
|
90
91
|
} from "./live-voice-tts.js";
|
|
92
|
+
import { pickProgressPhrase } from "./progress-phrases.js";
|
|
91
93
|
import {
|
|
92
94
|
type LiveVoiceClientFrame,
|
|
93
95
|
type LiveVoiceClientUpdateConfigFrame,
|
|
@@ -244,11 +246,10 @@ export interface LiveVoiceSessionOptions {
|
|
|
244
246
|
*/
|
|
245
247
|
frontModelConfig?: Partial<LiveVoiceFrontModelConfig>;
|
|
246
248
|
/**
|
|
247
|
-
* Front-model
|
|
248
|
-
*
|
|
249
|
-
*
|
|
250
|
-
*
|
|
251
|
-
* disables front-model LLM calls (energy endpointing + static ack phrasing).
|
|
249
|
+
* Front-model phrasing service: spoken acks and progress narration. The
|
|
250
|
+
* production factory constructs it from `liveVoice.frontModel` config when
|
|
251
|
+
* unset; `null` disables front-model LLM calls, so no ack is ever spoken
|
|
252
|
+
* and narration falls back to its static phrases.
|
|
252
253
|
*/
|
|
253
254
|
frontDecider?: VoiceFrontDecider | null;
|
|
254
255
|
/**
|
|
@@ -380,6 +381,13 @@ interface TurnProgressState {
|
|
|
380
381
|
// op, on start (not completion), so a burst of slow tools still trips the
|
|
381
382
|
// threshold while they run.
|
|
382
383
|
opsSinceNarration: number;
|
|
384
|
+
// Bumped by every observable change to the turn's tool activity — an op
|
|
385
|
+
// starting or finishing. The idle trigger compares it against
|
|
386
|
+
// `narratedEpoch` so a tick with nothing new to report stays silent.
|
|
387
|
+
stateEpoch: number;
|
|
388
|
+
// The `stateEpoch` the last spoken narration described: the activity the
|
|
389
|
+
// user has already been told about.
|
|
390
|
+
narratedEpoch: number;
|
|
383
391
|
// Narrations actually spoken this turn — the metrics count and the
|
|
384
392
|
// decider's 1-based updateIndex. Rate, not count, bounds narration:
|
|
385
393
|
// idleIntervalMs/minGapMs cap the cadence and the session duration cap
|
|
@@ -428,6 +436,9 @@ interface ActiveAssistantTurn {
|
|
|
428
436
|
progress: TurnProgressState;
|
|
429
437
|
assistantCompleted: boolean;
|
|
430
438
|
ttsDone: boolean;
|
|
439
|
+
// Latched when the leg's stream contains MINIMIZE_ROOM_MARKER; consumed
|
|
440
|
+
// once at TTS drain, where the minimize_room frame goes out after tts_done.
|
|
441
|
+
minimizeRequested: boolean;
|
|
431
442
|
// A tts_audio frame actually went out to the client — latches on the first
|
|
432
443
|
// forwarded chunk so the firstTtsAudio metric is marked exactly once per turn.
|
|
433
444
|
ttsAudioStarted: boolean;
|
|
@@ -473,6 +484,10 @@ interface ActiveAssistantTurn {
|
|
|
473
484
|
// or reference it in reply to the user, without ever speaking it unprompted.
|
|
474
485
|
// Null when no continuation result is pending for this turn.
|
|
475
486
|
continuationResult: string | null;
|
|
487
|
+
// Set when a barge-in handed the interrupted work to a background subagent:
|
|
488
|
+
// that request's transcript, so the model can tell the user the work is
|
|
489
|
+
// still running instead of appearing to have dropped it.
|
|
490
|
+
handedOffRequest: string | null;
|
|
476
491
|
// The agent run started a definitive tool use this turn — tool use implies
|
|
477
492
|
// a guaranteed-slow turn, so acknowledgment logic can key off this.
|
|
478
493
|
toolUseStarted: boolean;
|
|
@@ -487,9 +502,9 @@ interface ActiveAssistantTurn {
|
|
|
487
502
|
// A spoken ack was enqueued this turn. Every ack trigger shares this
|
|
488
503
|
// one-per-turn budget so a slow turn never stacks fillers.
|
|
489
504
|
ackSpoken: boolean;
|
|
490
|
-
// An
|
|
491
|
-
//
|
|
492
|
-
//
|
|
505
|
+
// An ack generation is awaiting the decider. While true, the ack has not
|
|
506
|
+
// yet stamped `lastFloorHolderAtMs`, so narration must treat it as a
|
|
507
|
+
// floor-holder-in-waiting and stand down —
|
|
493
508
|
// otherwise a progress phrase could land back-to-back with the ack.
|
|
494
509
|
ackGenerationPending: boolean;
|
|
495
510
|
// Pending slow-first-delta ack timer; null once cleared or fired.
|
|
@@ -516,13 +531,87 @@ interface ActiveAssistantTurn {
|
|
|
516
531
|
assistantAudioSampleRate?: number;
|
|
517
532
|
}
|
|
518
533
|
|
|
534
|
+
/**
|
|
535
|
+
* Control-marker hygiene for one model leg's delta stream, shared by the
|
|
536
|
+
* front-door answer stage and the default/escalated leg. The returned flush
|
|
537
|
+
* forwards the stripped (stripInternalSpeechMarkers) prefix of `raw` that has
|
|
538
|
+
* not been emitted yet and cannot contain a still-streaming control marker:
|
|
539
|
+
* the flush stops at the first "[" whose tail is an incomplete marker
|
|
540
|
+
* (isIncompleteControlMarkerTail) and holds from there until a later delta
|
|
541
|
+
* completes or disproves it; `force` (leg completion) emits the held tail so
|
|
542
|
+
* real text that merely resembles a marker prefix is not dropped. The scan
|
|
543
|
+
* runs forward from the emitted boundary — not from the last "[" — so
|
|
544
|
+
* brackets INSIDE a streaming marker body (a JSON array or "]"-bearing string
|
|
545
|
+
* in ASK_GUARDIAN_APPROVAL) can neither mask the marker's start nor pass as
|
|
546
|
+
* its terminator. As a side effect the force flush latches the turn's
|
|
547
|
+
* `minimizeRequested` when the completed stream ENDS with
|
|
548
|
+
* MINIMIZE_ROOM_MARKER — terminal position only, matching the prompt's
|
|
549
|
+
* "end the reply with it" contract, so a reply whose CONTENT happens to
|
|
550
|
+
* contain "[-1]" (an array literal, a temperature) never minimizes the
|
|
551
|
+
* room. The marker itself is stripped wherever it appears (the shared
|
|
552
|
+
* marker-strip convention), so the latch is the only observable.
|
|
553
|
+
*/
|
|
554
|
+
function createControlMarkerHoldback(
|
|
555
|
+
turn: ActiveAssistantTurn,
|
|
556
|
+
emit: (chunk: string) => void,
|
|
557
|
+
): (raw: string, opts?: { force?: boolean }) => void {
|
|
558
|
+
let emitted = 0;
|
|
559
|
+
return (raw, opts) => {
|
|
560
|
+
if (
|
|
561
|
+
opts?.force === true &&
|
|
562
|
+
!turn.minimizeRequested &&
|
|
563
|
+
raw.trimEnd().endsWith(MINIMIZE_ROOM_MARKER)
|
|
564
|
+
) {
|
|
565
|
+
turn.minimizeRequested = true;
|
|
566
|
+
}
|
|
567
|
+
let safeEnd = raw.length;
|
|
568
|
+
if (opts?.force !== true) {
|
|
569
|
+
for (
|
|
570
|
+
let i = raw.indexOf("[", emitted);
|
|
571
|
+
i !== -1;
|
|
572
|
+
i = raw.indexOf("[", i + 1)
|
|
573
|
+
) {
|
|
574
|
+
if (isIncompleteControlMarkerTail(raw.slice(i))) {
|
|
575
|
+
safeEnd = i;
|
|
576
|
+
break;
|
|
577
|
+
}
|
|
578
|
+
}
|
|
579
|
+
}
|
|
580
|
+
if (safeEnd > emitted) {
|
|
581
|
+
emit(stripInternalSpeechMarkers(raw.slice(emitted, safeEnd)));
|
|
582
|
+
emitted = safeEnd;
|
|
583
|
+
}
|
|
584
|
+
};
|
|
585
|
+
}
|
|
586
|
+
|
|
519
587
|
// Base control prompt for every live-voice turn. When a turn starts from a
|
|
520
588
|
// barge-in, the interruption merge note is appended to it (see
|
|
521
589
|
// buildInterruptionMergeNote) so the model reconciles the interrupted request
|
|
522
590
|
// with the new utterance.
|
|
523
|
-
const
|
|
524
|
-
"You are speaking in a local live voice session. Keep replies brief and conversational. You cannot display cards, forms, or any on-screen UI during the call — convey everything in speech. "
|
|
525
|
-
|
|
591
|
+
const LIVE_VOICE_CONTROL_PROMPT_BASE =
|
|
592
|
+
"You are speaking in a local live voice session. Keep replies brief and conversational. You cannot display cards, forms, or any on-screen UI during the call — convey everything in speech. ";
|
|
593
|
+
|
|
594
|
+
// MINIMIZE_ROOM_MARKER teaching, appended for the legs that can actually put
|
|
595
|
+
// something on screen — the main leg and the escalated leg. The front-door
|
|
596
|
+
// (fast) leg never receives it: that leg is toolless, so it has nothing to
|
|
597
|
+
// show, and its decision rule promises that apart from a leading verdict
|
|
598
|
+
// token every character is spoken verbatim — teaching it the marker would
|
|
599
|
+
// contradict that rule and could only produce spurious minimizes.
|
|
600
|
+
//
|
|
601
|
+
// The teaching deliberately keeps the no-interactive-UI rule intact
|
|
602
|
+
// (live-voice turns are non-interactive, JARVIS-1291): the marker does not
|
|
603
|
+
// render UI — it asks the client to reveal the screen the call overlay
|
|
604
|
+
// covers. The gate in createControlMarkerHoldback strips it from speech, and
|
|
605
|
+
// completeTtsForTurn sends the minimize_room frame only after the reply's
|
|
606
|
+
// TTS drains, so "end your reply with it" is the whole contract the model
|
|
607
|
+
// needs.
|
|
608
|
+
//
|
|
609
|
+
// The narration guidance is load-bearing: without it, the base prompt's
|
|
610
|
+
// convey-everything-in-speech rule reads to the model as "the user can never
|
|
611
|
+
// see the screen", and it minimizes the room while telling the user it has
|
|
612
|
+
// no way to show them anything ("check it from the app later").
|
|
613
|
+
const LIVE_VOICE_MINIMIZE_MARKER_TEACHING =
|
|
614
|
+
"The call renders as a full-screen overlay covering the app. If this reply created or changed something on screen worth looking at (an app, a page, a document), you may end the reply with the marker [-1]: after you finish speaking, the overlay minimizes so the user can see the screen while the call continues. The user is in the app and sees what you made the moment the overlay minimizes, so when you end with the marker, speak as if you are showing it to them right now (for example, close with something like: take a look) — never say you cannot show it, that you cannot display it because this is a voice call, or that they should check it later. Use it at most once per reply, only when there is genuinely something new to show, never speak the marker aloud or mention it, and never emit any other bracketed marker. ";
|
|
526
615
|
|
|
527
616
|
// System-level guidance appended to a barge-in turn's control prompt so the
|
|
528
617
|
// model treats the new utterance as a continuation of the request it was cut
|
|
@@ -542,21 +631,60 @@ function buildResurfaceContextNote(continuationResult: string): string {
|
|
|
542
631
|
return `Earlier the user interrupted you, and in the background you finished the reply they cut off. What you worked out was: "${continuationResult}". If their current message relates to it, use it to answer; otherwise you may briefly offer it or leave it aside, and do not repeat it verbatim if it no longer fits.`;
|
|
543
632
|
}
|
|
544
633
|
|
|
545
|
-
//
|
|
634
|
+
// Appended to the turn that follows a barge-in whose interrupted work was
|
|
635
|
+
// handed to a background subagent. Without this the assistant simply stops
|
|
636
|
+
// talking about the request it was mid-way through, and the user has no way to
|
|
637
|
+
// know the work survived — it reads as dropped. The model decides whether to
|
|
638
|
+
// mention it, because only it can tell "keep working on that" (the foreground
|
|
639
|
+
// turn continues the same work, so announcing a background copy would be
|
|
640
|
+
// confusing) from a genuine topic change (where "I'm still working on that in
|
|
641
|
+
// the background" is exactly what the user needs to hear).
|
|
642
|
+
function buildHandoffAnnouncementNote(handedOffRequest: string): string {
|
|
643
|
+
return `You handed your unfinished work on "${handedOffRequest}" to a background task, which is still running. If the user has moved to a different topic, briefly let them know you are still working on it in the background before answering them. If they are asking you to continue that same work, just continue and do not mention the background task.`;
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
// Assemble a leg's model-facing control prompt: the base live-voice rules,
|
|
647
|
+
// the [-1] minimize teaching (withheld from the front-door leg — see
|
|
648
|
+
// LIVE_VOICE_MINIMIZE_MARKER_TEACHING), the shared no-setup-flows rule, plus
|
|
546
649
|
// any pending barge-in merge context and/or completed-continuation context. A
|
|
547
|
-
// turn can carry both (a barge-in follow-up that also has a continuation
|
|
548
|
-
// waiting); the notes are model-only and never render as user bubbles.
|
|
549
|
-
function buildVoiceControlPrompt(
|
|
550
|
-
|
|
650
|
+
// turn can carry both (a barge-in follow-up that also has a continuation
|
|
651
|
+
// result waiting); the notes are model-only and never render as user bubbles.
|
|
652
|
+
function buildVoiceControlPrompt(
|
|
653
|
+
turn: ActiveAssistantTurn,
|
|
654
|
+
leg: { frontDoor?: boolean },
|
|
655
|
+
): string {
|
|
656
|
+
let prompt =
|
|
657
|
+
LIVE_VOICE_CONTROL_PROMPT_BASE +
|
|
658
|
+
(leg.frontDoor === true ? "" : LIVE_VOICE_MINIMIZE_MARKER_TEACHING) +
|
|
659
|
+
VOICE_NO_SETUP_FLOWS_RULE;
|
|
551
660
|
if (turn.interruptedRequest) {
|
|
552
661
|
prompt = `${prompt}\n\n${buildInterruptionMergeNote(turn.interruptedRequest)}`;
|
|
553
662
|
}
|
|
554
663
|
if (turn.continuationResult) {
|
|
555
664
|
prompt = `${prompt}\n\n${buildResurfaceContextNote(turn.continuationResult)}`;
|
|
556
665
|
}
|
|
666
|
+
if (turn.handedOffRequest) {
|
|
667
|
+
prompt = `${prompt}\n\n${buildHandoffAnnouncementNote(turn.handedOffRequest)}`;
|
|
668
|
+
}
|
|
557
669
|
return prompt;
|
|
558
670
|
}
|
|
559
671
|
|
|
672
|
+
// Delivered into the conversation when a continuation finishes AFTER the voice
|
|
673
|
+
// session ended. There is no next voice turn to fold into, so this lands as a
|
|
674
|
+
// normal turn in the thread — where the user actually goes looking for the
|
|
675
|
+
// work. Framed as a system-style report rather than a user request so the
|
|
676
|
+
// reply reads as "here is what I finished", not a fresh instruction.
|
|
677
|
+
function buildClosedSessionDeliveryPrompt(
|
|
678
|
+
interruptedRequest: string,
|
|
679
|
+
answer: string,
|
|
680
|
+
): string {
|
|
681
|
+
const what =
|
|
682
|
+
interruptedRequest.length > 0
|
|
683
|
+
? `their earlier request ("${interruptedRequest}")`
|
|
684
|
+
: "their earlier request";
|
|
685
|
+
return `[Background work finished] The voice call ended while you were still finishing ${what} in the background. You have now finished it. Tell the user briefly that it is done and give them the result. Do not re-run any tool calls; the work is already complete. What you produced was:\n\n${answer}`;
|
|
686
|
+
}
|
|
687
|
+
|
|
560
688
|
// Objective handed to the background subagent that continues a barged-in turn.
|
|
561
689
|
// The subagent forks the live conversation, so it already sees the interrupted
|
|
562
690
|
// turn's completed tool calls in history and resumes from there. The request
|
|
@@ -565,12 +693,51 @@ function buildVoiceControlPrompt(turn: ActiveAssistantTurn): string {
|
|
|
565
693
|
// the forked history.
|
|
566
694
|
function buildDuplexContinuationObjective(interruptedRequest: string): string {
|
|
567
695
|
const base =
|
|
568
|
-
"You were in the middle of responding to the user's most recent request when they interrupted you. Finish that response now. Do not repeat any tool calls whose results are already present in the conversation. You are running unattended in the background
|
|
696
|
+
"You were in the middle of responding to the user's most recent request when they interrupted you. Finish that response now. Do not repeat any tool calls whose results are already present in the conversation. You are running unattended in the background: permission policy may auto-deny higher-risk actions with no one to approve them. If an action you need is denied, do not retry it; finish what you can and say plainly what remains, so the user can trigger it on their next turn.";
|
|
569
697
|
return interruptedRequest.length > 0
|
|
570
698
|
? `${base} Their request was: "${interruptedRequest}".`
|
|
571
699
|
: base;
|
|
572
700
|
}
|
|
573
701
|
|
|
702
|
+
// Built-ins beyond the strict read-only allowlist that cannot contend with a
|
|
703
|
+
// background continuation's writes: they touch no workspace, host, or
|
|
704
|
+
// extension state. `skill_load` reads skill files and registers tool
|
|
705
|
+
// definitions — and it is the FIRST call of a barge-in follow-up that
|
|
706
|
+
// re-enters the skill the interrupted turn was using, so counting it as
|
|
707
|
+
// consequential would kill nearly every continuation doing skill-based work
|
|
708
|
+
// at the moment it matters most.
|
|
709
|
+
// `web_fetch` sits on the core SIDE_EFFECT_TOOLS list because an UNATTENDED
|
|
710
|
+
// run firing off external requests is a permission concern — a different
|
|
711
|
+
// question from this gate's, which is only "can these two writers corrupt the
|
|
712
|
+
// same local state?". A network read cannot, so it does not contend.
|
|
713
|
+
const FOREGROUND_NON_CONTENDING_TOOLS: ReadonlySet<string> = new Set([
|
|
714
|
+
"skill_load",
|
|
715
|
+
"web_fetch",
|
|
716
|
+
]);
|
|
717
|
+
|
|
718
|
+
// Foreground-wins classification: does this foreground tool start force the
|
|
719
|
+
// running continuations to be aborted? The question is LOCAL-STATE
|
|
720
|
+
// CONTENTION ("could these two writers corrupt the same workspace, host, or
|
|
721
|
+
// extension state?"), NOT permission — this gate never affects what a tool is
|
|
722
|
+
// allowed to do. Fail closed anyway: anything that is not a provably
|
|
723
|
+
// non-contending BUILT-IN contends, because skill/plugin/MCP/workspace tools
|
|
724
|
+
// carry no "writes local state" metadata and some of them (app_*, document_*)
|
|
725
|
+
// very much do. `skill_execute` always contends: it is a dispatcher whose
|
|
726
|
+
// resolved inner tool can mutate.
|
|
727
|
+
function foregroundToolContendsWithContinuation(toolName: string): boolean {
|
|
728
|
+
if (toolName === "skill_execute") {
|
|
729
|
+
return true;
|
|
730
|
+
}
|
|
731
|
+
const ownerKind = getToolOwner(toolName)?.kind;
|
|
732
|
+
if (
|
|
733
|
+
FOREGROUND_NON_CONTENDING_TOOLS.has(toolName) &&
|
|
734
|
+
ownerKind === "default"
|
|
735
|
+
) {
|
|
736
|
+
return false;
|
|
737
|
+
}
|
|
738
|
+
return isRefusedInReadOnlyPass(toolName, ownerKind);
|
|
739
|
+
}
|
|
740
|
+
|
|
574
741
|
// Upper bound on how long a barge-in waits for the interrupted turn's teardown
|
|
575
742
|
// to settle before giving up on the continuation. The teardown settles once the
|
|
576
743
|
// aborted turn's agent loop reaches its `finally`, which can wait out BOTH the
|
|
@@ -605,9 +772,11 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
605
772
|
// synchronously before its spawn, so interrupt()/close() abort a continuation
|
|
606
773
|
// even if a stop lands while it is still spawning.
|
|
607
774
|
private readonly detachControllers = new Set<AbortController>();
|
|
608
|
-
// Bumped whenever a stop (interrupt/close)
|
|
609
|
-
//
|
|
610
|
-
//
|
|
775
|
+
// Bumped whenever detached runs are invalidated: a stop (interrupt/close),
|
|
776
|
+
// a newer barge-in superseding them, or a foreground-wins abort. A barge-in
|
|
777
|
+
// captures this (after its own bump) before its async teardown; if it has
|
|
778
|
+
// changed by the time the detach would spawn, an invalidation landed during
|
|
779
|
+
// the gap and the continuation is not started.
|
|
611
780
|
private detachStopGeneration = 0;
|
|
612
781
|
// Bumped SYNCHRONOUSLY at each barge-in (in barge order), before the async
|
|
613
782
|
// detach runs. Only the latest-started detach (detachSeq === detachSequence)
|
|
@@ -670,6 +839,11 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
670
839
|
// starts as context. Consumed (and cleared) when that turn launches; cleared
|
|
671
840
|
// on a hard stop (abortDetachedRuns) so a stale result never surfaces later.
|
|
672
841
|
private pendingContinuationResult: string | null = null;
|
|
842
|
+
// Set when a continuation actually spawns: the request it took over, so the
|
|
843
|
+
// NEXT turn can tell the user the work is still running. Consumed by that
|
|
844
|
+
// turn; cleared when the continuation finishes (by then the result note
|
|
845
|
+
// takes over and "still running" would be stale).
|
|
846
|
+
private pendingHandoffRequest: string | null = null;
|
|
673
847
|
private readonly maxPendingAudioBytes: number;
|
|
674
848
|
// Set on VAD speech onset; consumed when the first speech chunk is routed
|
|
675
849
|
// to an utterance so the metric lands on the right turn.
|
|
@@ -706,13 +880,10 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
706
880
|
/** The cycle whose grace timer is armed (only the newest release has one). */
|
|
707
881
|
private finalizeGraceCycle: UtteranceCycle | null = null;
|
|
708
882
|
private readonly finalizeGraceMs: number;
|
|
709
|
-
// Rotates through the ack phrase list across the session's turns.
|
|
710
|
-
private ackPhraseCounter = 0;
|
|
711
883
|
// Rotates through the progress fallback phrases across the session's turns.
|
|
712
884
|
private progressPhraseCounter = 0;
|
|
713
|
-
// Front-model service:
|
|
714
|
-
//
|
|
715
|
-
// capabilities (energy endpointing + static ack phrasing remain).
|
|
885
|
+
// Front-model service: phrases spoken acks and progress narration; null
|
|
886
|
+
// disables both (the turn then holds the floor with silence).
|
|
716
887
|
private readonly frontDecider: VoiceFrontDecider | null;
|
|
717
888
|
// Complete front-model tunables: the constructor schema-parses the partial
|
|
718
889
|
// option once, so every field carries its `liveVoice.frontModel` schema
|
|
@@ -897,7 +1068,14 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
897
1068
|
this.turnDetector?.dispose();
|
|
898
1069
|
this.clearEndpointExtensionTimer();
|
|
899
1070
|
this.stopSessionTranscriber();
|
|
900
|
-
|
|
1071
|
+
// Deliberately NOT aborting detached continuations: outliving the call is
|
|
1072
|
+
// the entire premise of the handoff. Hanging up used to destroy the work
|
|
1073
|
+
// the user had just asked to keep — and the natural test sequence (barge
|
|
1074
|
+
// in, hear the answer, close the room, go look for the result) hit that
|
|
1075
|
+
// every time. A deliberate `interrupt()` still aborts; ending the session
|
|
1076
|
+
// does not. With no next voice turn to fold into, a continuation that
|
|
1077
|
+
// finishes after this point delivers into the conversation instead (see
|
|
1078
|
+
// the completion handler in detachInterruptedTurn).
|
|
901
1079
|
await this.cancelAssistantTurn("session_closed");
|
|
902
1080
|
if (shouldEmitSessionEndMetrics) {
|
|
903
1081
|
await this.emitSessionEndMetrics();
|
|
@@ -1415,11 +1593,20 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1415
1593
|
.trim();
|
|
1416
1594
|
this.pendingInterruptedRequest =
|
|
1417
1595
|
interruptedRequest.length > 0 ? interruptedRequest : null;
|
|
1418
|
-
// A fresh interruption supersedes
|
|
1419
|
-
//
|
|
1420
|
-
//
|
|
1421
|
-
//
|
|
1422
|
-
|
|
1596
|
+
// A fresh interruption supersedes every earlier detached run, not just its
|
|
1597
|
+
// stashed result: abort still-running continuations (and skip pending
|
|
1598
|
+
// detaches) before this barge-in's own continuation can launch, so two
|
|
1599
|
+
// full-ability background writers never share the workspace. The stashed
|
|
1600
|
+
// result is dropped with them so the barge-in follow-up (or any later)
|
|
1601
|
+
// turn can't consume an older answer before this barge-in's own
|
|
1602
|
+
// continuation completes. This barge-in's detach snapshots the stop
|
|
1603
|
+
// generation AFTER this bump (below), so it is unaffected. The abort is
|
|
1604
|
+
// signal-level, with the same accepted residual as the foreground-wins
|
|
1605
|
+
// gate (see the tool_use_start handler): a tool call already executing in
|
|
1606
|
+
// the superseded run is not awaited. The replacement's detach still waits
|
|
1607
|
+
// out the interrupted TURN's teardown before forking, which bounds the
|
|
1608
|
+
// overlap to that one abandoned call.
|
|
1609
|
+
this.abortDetachedRuns({ reason: "superseded_by_new_barge_in" });
|
|
1423
1610
|
// Order this barge-in among concurrent detaches SYNCHRONOUSLY, in barge
|
|
1424
1611
|
// order — the actual detach runs after an async teardown chain, and those
|
|
1425
1612
|
// chains can interleave, so bumping there could assign sequences out of
|
|
@@ -1429,8 +1616,8 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1429
1616
|
this.clearFillerTimers(turn);
|
|
1430
1617
|
// Tagged reason: provider catch-sites classify untagged caller aborts as
|
|
1431
1618
|
// retryable transport failures (ERROR log + futile retry against the
|
|
1432
|
-
// aborted signal). This signal reaches the brain leg and
|
|
1433
|
-
//
|
|
1619
|
+
// aborted signal). This signal reaches the brain leg and any in-flight
|
|
1620
|
+
// ack generation.
|
|
1434
1621
|
turn.abortController.abort(
|
|
1435
1622
|
createAbortReason("voice_session_aborted", "live-voice-barge-in"),
|
|
1436
1623
|
);
|
|
@@ -1445,6 +1632,16 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1445
1632
|
// Snapshot the stop generation before the async teardown: a stop that lands
|
|
1446
1633
|
// during it must cancel the pending detach (checked in detachInterruptedTurn).
|
|
1447
1634
|
const stopGeneration = this.detachStopGeneration;
|
|
1635
|
+
log.info(
|
|
1636
|
+
{
|
|
1637
|
+
turnId: turn.turnId,
|
|
1638
|
+
detachSeq,
|
|
1639
|
+
// The two facts that decide whether a continuation is even eligible.
|
|
1640
|
+
assistantCompleted: turn.assistantCompleted,
|
|
1641
|
+
hasTeardownWait: teardownWait !== undefined,
|
|
1642
|
+
},
|
|
1643
|
+
"Voice barge-in cancelled a turn",
|
|
1644
|
+
);
|
|
1448
1645
|
void (async () => {
|
|
1449
1646
|
await this.finishMetricsTurn(
|
|
1450
1647
|
turn.utterance,
|
|
@@ -1467,8 +1664,12 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1467
1664
|
// settle before forking, so its partial — including any completed tool calls —
|
|
1468
1665
|
// is already in the conversation the subagent forks from and a side-effecting
|
|
1469
1666
|
// continuation cannot repeat a call the interrupted turn already ran.
|
|
1470
|
-
//
|
|
1471
|
-
//
|
|
1667
|
+
// The continuation runs with full subagent abilities under the standard
|
|
1668
|
+
// non-interactive permission policy; if a foreground turn starts its own
|
|
1669
|
+
// side-effecting tool, the foreground-wins abort in the tool_use_start
|
|
1670
|
+
// handler kills the continuation before the two can race on the workspace.
|
|
1671
|
+
// The run stays silent (never spoken unprompted) and a later stop/interrupt
|
|
1672
|
+
// aborts it.
|
|
1472
1673
|
private detachInterruptedTurn(
|
|
1473
1674
|
turn: ActiveAssistantTurn,
|
|
1474
1675
|
stopGeneration: number,
|
|
@@ -1476,18 +1677,41 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1476
1677
|
detachSeq: number,
|
|
1477
1678
|
): void {
|
|
1478
1679
|
const spawn = this.spawnBackgroundContinuation;
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1680
|
+
// Every skip is logged with its reason — the handoff is silent by design,
|
|
1681
|
+
// so without this a dropped continuation is indistinguishable from a
|
|
1682
|
+
// never-attempted one (tail with: grep -i "voice duplex").
|
|
1683
|
+
const skipReason = !spawn
|
|
1684
|
+
? "no_spawner"
|
|
1685
|
+
: this.isClosed
|
|
1686
|
+
? "session_closed"
|
|
1687
|
+
: // The model already finished generating (barge-in during TTS playback
|
|
1688
|
+
// of a complete reply): there is nothing to continue, so a
|
|
1689
|
+
// continuation would just re-do a finished answer.
|
|
1690
|
+
turn.assistantCompleted
|
|
1691
|
+
? "assistant_already_completed"
|
|
1692
|
+
: // A stop (interrupt/close) or a superseding invalidation landed
|
|
1693
|
+
// during the barge-in teardown: honor it.
|
|
1694
|
+
this.detachStopGeneration !== stopGeneration
|
|
1695
|
+
? "invalidated_during_barge_teardown"
|
|
1696
|
+
: !isAssistantFeatureFlagEnabled(
|
|
1697
|
+
"voice-duplex-handoff",
|
|
1698
|
+
getConfig(),
|
|
1699
|
+
)
|
|
1700
|
+
? "flag_disabled"
|
|
1701
|
+
: null;
|
|
1702
|
+
if (skipReason !== null || !spawn) {
|
|
1703
|
+
// debug for the always-off configurations, info for the dynamic skips.
|
|
1704
|
+
if (skipReason === "flag_disabled" || skipReason === "no_spawner") {
|
|
1705
|
+
log.debug(
|
|
1706
|
+
{ turnId: turn.turnId, skipReason },
|
|
1707
|
+
"Voice duplex continuation skipped",
|
|
1708
|
+
);
|
|
1709
|
+
} else {
|
|
1710
|
+
log.info(
|
|
1711
|
+
{ turnId: turn.turnId, skipReason },
|
|
1712
|
+
"Voice duplex continuation skipped",
|
|
1713
|
+
);
|
|
1714
|
+
}
|
|
1491
1715
|
return;
|
|
1492
1716
|
}
|
|
1493
1717
|
// Embed the interrupted request in the objective so the continuation knows
|
|
@@ -1501,6 +1725,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1501
1725
|
// controller.abort(), which the spawn's signal wiring honors).
|
|
1502
1726
|
const controller = new AbortController();
|
|
1503
1727
|
this.detachControllers.add(controller);
|
|
1728
|
+
const detachStartedAtMs = Date.now();
|
|
1504
1729
|
void (async () => {
|
|
1505
1730
|
try {
|
|
1506
1731
|
// Wait for the interrupted turn's teardown to settle its partial into
|
|
@@ -1528,12 +1753,46 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1528
1753
|
// The continuation is best-effort, so a rare dropped one is the safe
|
|
1529
1754
|
// trade.
|
|
1530
1755
|
if (!settled) {
|
|
1756
|
+
log.info(
|
|
1757
|
+
{
|
|
1758
|
+
turnId: turn.turnId,
|
|
1759
|
+
skipReason: controller.signal.aborted
|
|
1760
|
+
? "invalidated_during_teardown_wait"
|
|
1761
|
+
: "teardown_settle_timeout",
|
|
1762
|
+
waitedMs: Date.now() - detachStartedAtMs,
|
|
1763
|
+
timeoutMs: this.detachTeardownSettleTimeoutMs,
|
|
1764
|
+
},
|
|
1765
|
+
"Voice duplex continuation skipped",
|
|
1766
|
+
);
|
|
1531
1767
|
return;
|
|
1532
1768
|
}
|
|
1533
1769
|
}
|
|
1534
|
-
|
|
1770
|
+
// A closed session is NOT a reason to skip: the work outlives the
|
|
1771
|
+
// call, and its result is delivered into the conversation below.
|
|
1772
|
+
// Only an explicit invalidation (stop/interrupt/supersede) stops it.
|
|
1773
|
+
if (controller.signal.aborted) {
|
|
1774
|
+
log.info(
|
|
1775
|
+
{
|
|
1776
|
+
turnId: turn.turnId,
|
|
1777
|
+
skipReason: "invalidated_before_spawn",
|
|
1778
|
+
waitedMs: Date.now() - detachStartedAtMs,
|
|
1779
|
+
},
|
|
1780
|
+
"Voice duplex continuation skipped",
|
|
1781
|
+
);
|
|
1535
1782
|
return;
|
|
1536
1783
|
}
|
|
1784
|
+
log.info(
|
|
1785
|
+
{
|
|
1786
|
+
turnId: turn.turnId,
|
|
1787
|
+
teardownWaitMs: Date.now() - detachStartedAtMs,
|
|
1788
|
+
interruptedRequest,
|
|
1789
|
+
},
|
|
1790
|
+
"Voice duplex continuation starting",
|
|
1791
|
+
);
|
|
1792
|
+
// The next turn tells the user this is still running. Set before the
|
|
1793
|
+
// await so a follow-up turn launching during the run picks it up.
|
|
1794
|
+
this.pendingHandoffRequest =
|
|
1795
|
+
interruptedRequest.length > 0 ? interruptedRequest : null;
|
|
1537
1796
|
const resultText = await spawn({
|
|
1538
1797
|
parentConversationId: this.conversationId,
|
|
1539
1798
|
objective: buildDuplexContinuationObjective(interruptedRequest),
|
|
@@ -1547,20 +1806,59 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1547
1806
|
// barge-in has started, an older continuation completing (before or
|
|
1548
1807
|
// after it, empty or not) can't surface a stale answer. Only non-empty
|
|
1549
1808
|
// text is actually surfaced.
|
|
1550
|
-
|
|
1809
|
+
this.pendingHandoffRequest = null;
|
|
1810
|
+
const answer = resultText.trim();
|
|
1811
|
+
const notInvalidated =
|
|
1551
1812
|
!controller.signal.aborted &&
|
|
1552
|
-
!this.isClosed &&
|
|
1553
1813
|
this.detachStopGeneration === stopGeneration &&
|
|
1554
|
-
detachSeq === this.detachSequence
|
|
1555
|
-
|
|
1556
|
-
|
|
1814
|
+
detachSeq === this.detachSequence;
|
|
1815
|
+
// Two destinations, decided by whether a next voice turn still exists.
|
|
1816
|
+
// Live session: stash for that turn's control prompt (never spoken
|
|
1817
|
+
// unprompted). Session closed: there is no next turn, so deliver into
|
|
1818
|
+
// the conversation — the call is over, nobody is being talked over,
|
|
1819
|
+
// and the thread is exactly where the user goes looking for the work.
|
|
1820
|
+
const deliverToConversation =
|
|
1821
|
+
notInvalidated && this.isClosed && answer.length > 0;
|
|
1822
|
+
if (notInvalidated && !this.isClosed) {
|
|
1557
1823
|
this.pendingContinuationResult = answer.length > 0 ? answer : null;
|
|
1558
1824
|
}
|
|
1825
|
+
if (deliverToConversation) {
|
|
1826
|
+
const { injectMessageIntoParent } =
|
|
1827
|
+
await import("../subagent/notify.js");
|
|
1828
|
+
injectMessageIntoParent(
|
|
1829
|
+
this.conversationId,
|
|
1830
|
+
buildClosedSessionDeliveryPrompt(interruptedRequest, answer),
|
|
1831
|
+
);
|
|
1832
|
+
}
|
|
1833
|
+
log.info(
|
|
1834
|
+
{
|
|
1835
|
+
turnId: turn.turnId,
|
|
1836
|
+
ranMs: Date.now() - detachStartedAtMs,
|
|
1837
|
+
resultChars: answer.length,
|
|
1838
|
+
// Where the answer went. "stashed" = folded into the next voice
|
|
1839
|
+
// turn; "conversation" = the call had ended, so it was delivered
|
|
1840
|
+
// into the thread; "dropped" = a stop/interrupt or a newer
|
|
1841
|
+
// barge-in landed while it ran.
|
|
1842
|
+
resultDestination: !notInvalidated
|
|
1843
|
+
? "dropped"
|
|
1844
|
+
: answer.length === 0
|
|
1845
|
+
? "empty"
|
|
1846
|
+
: this.isClosed
|
|
1847
|
+
? "conversation"
|
|
1848
|
+
: "stashed",
|
|
1849
|
+
},
|
|
1850
|
+
"Voice duplex continuation finished",
|
|
1851
|
+
);
|
|
1559
1852
|
} catch (err) {
|
|
1560
1853
|
// A stop/interrupt aborts via the signal; that rejection is expected.
|
|
1561
|
-
if (
|
|
1854
|
+
if (controller.signal.aborted) {
|
|
1855
|
+
log.info(
|
|
1856
|
+
{ turnId: turn.turnId, ranMs: Date.now() - detachStartedAtMs },
|
|
1857
|
+
"Voice duplex continuation aborted mid-run",
|
|
1858
|
+
);
|
|
1859
|
+
} else {
|
|
1562
1860
|
log.warn(
|
|
1563
|
-
{ err, turnId: turn.turnId },
|
|
1861
|
+
{ err, turnId: turn.turnId, ranMs: Date.now() - detachStartedAtMs },
|
|
1564
1862
|
"Voice duplex handoff continuation failed",
|
|
1565
1863
|
);
|
|
1566
1864
|
}
|
|
@@ -1571,9 +1869,37 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1571
1869
|
}
|
|
1572
1870
|
|
|
1573
1871
|
// Abort every background continuation this session started and drop its
|
|
1574
|
-
// handle.
|
|
1575
|
-
//
|
|
1576
|
-
|
|
1872
|
+
// handle. Called on a hard stop (client interrupt / session close), when a
|
|
1873
|
+
// newer barge-in supersedes the detached runs, and on a foreground-wins
|
|
1874
|
+
// abort; the continuation's own `.finally` removes it from the set too.
|
|
1875
|
+
// `keepPendingResult` is the foreground-wins variant: the foreground turn is
|
|
1876
|
+
// claiming the workspace, so running continuations must die (and pending
|
|
1877
|
+
// detaches must be skipped), but an already-completed continuation's stashed
|
|
1878
|
+
// answer stays — it cannot race anything, and the next turn's "use only if
|
|
1879
|
+
// relevant" framing makes a stale one harmless.
|
|
1880
|
+
private abortDetachedRuns(opts?: {
|
|
1881
|
+
keepPendingResult?: boolean;
|
|
1882
|
+
// Why the runs are being invalidated, for the log line below. Every
|
|
1883
|
+
// caller passes one: an unexplained dead continuation is the single
|
|
1884
|
+
// hardest thing to debug about this feature.
|
|
1885
|
+
reason?: string;
|
|
1886
|
+
// The foreground tool whose start tripped the contention gate, if any.
|
|
1887
|
+
toolName?: string;
|
|
1888
|
+
}): void {
|
|
1889
|
+
const aborted = this.detachControllers.size;
|
|
1890
|
+
const hadPendingResult = this.pendingContinuationResult !== null;
|
|
1891
|
+
if (aborted > 0 || hadPendingResult) {
|
|
1892
|
+
log.info(
|
|
1893
|
+
{
|
|
1894
|
+
conversationId: this.conversationId,
|
|
1895
|
+
reason: opts?.reason ?? "unspecified",
|
|
1896
|
+
...(opts?.toolName ? { toolName: opts.toolName } : {}),
|
|
1897
|
+
abortedRuns: aborted,
|
|
1898
|
+
droppedPendingResult: hadPendingResult && !opts?.keepPendingResult,
|
|
1899
|
+
},
|
|
1900
|
+
"Voice duplex continuations invalidated",
|
|
1901
|
+
);
|
|
1902
|
+
}
|
|
1577
1903
|
// Bump the generation so a barge-in whose async teardown is still in flight
|
|
1578
1904
|
// (its detach not yet spawned) sees the stop and skips the continuation.
|
|
1579
1905
|
this.detachStopGeneration += 1;
|
|
@@ -1583,7 +1909,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1583
1909
|
this.detachControllers.clear();
|
|
1584
1910
|
// A hard stop also drops any completed continuation's result still waiting
|
|
1585
1911
|
// to fold into the next turn, so it can't surface after the user reset.
|
|
1586
|
-
|
|
1912
|
+
if (!opts?.keepPendingResult) {
|
|
1913
|
+
this.pendingContinuationResult = null;
|
|
1914
|
+
}
|
|
1587
1915
|
}
|
|
1588
1916
|
|
|
1589
1917
|
// VAD closed the utterance — the analog of ptt_release: emit
|
|
@@ -1618,30 +1946,18 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1618
1946
|
}
|
|
1619
1947
|
return;
|
|
1620
1948
|
}
|
|
1621
|
-
// Semantic endpointing: a silence boundary may be a thinking pause
|
|
1622
|
-
//
|
|
1949
|
+
// Semantic endpointing: a silence boundary may be a thinking pause, so
|
|
1950
|
+
// the unified front-door leg itself is the endpoint decision. When it
|
|
1951
|
+
// launches, the boundary is deferred to the verdict — commit releases
|
|
1952
|
+
// the utterance, hold replays the boundary via the extension timer.
|
|
1953
|
+
// When speculation is inapplicable (no text, cap reached, a turn
|
|
1954
|
+
// already active) fall through and release. Only detector-timer
|
|
1623
1955
|
// silences qualify; max-duration always releases, and a manual client
|
|
1624
1956
|
// release (ptt_release forced the boundary) is the user saying "answer
|
|
1625
|
-
// now" — never second-guess it.
|
|
1626
|
-
// the decider is not consulted, so that release path keeps its exact
|
|
1627
|
-
// event ordering — an extra await here would shift utterance release
|
|
1628
|
-
// across the persistent-transcriber flush attribution.
|
|
1957
|
+
// now" — never second-guess it.
|
|
1629
1958
|
if (reason === "silence" && !manualRelease) {
|
|
1630
|
-
if (this.
|
|
1631
|
-
|
|
1632
|
-
// When it launches, the boundary is deferred to the verdict —
|
|
1633
|
-
// commit releases the utterance, hold replays the boundary via
|
|
1634
|
-
// the extension timer. When speculation is inapplicable (no
|
|
1635
|
-
// text, cap reached, a turn already active) fall through and
|
|
1636
|
-
// release exactly as before.
|
|
1637
|
-
if (await this.launchSpeculativeAssistantTurn(utterance)) {
|
|
1638
|
-
return;
|
|
1639
|
-
}
|
|
1640
|
-
} else {
|
|
1641
|
-
const holdDecision = this.maybeHoldUtteranceOpen(utterance);
|
|
1642
|
-
if (holdDecision && (await holdDecision)) {
|
|
1643
|
-
return;
|
|
1644
|
-
}
|
|
1959
|
+
if (await this.launchSpeculativeAssistantTurn(utterance)) {
|
|
1960
|
+
return;
|
|
1645
1961
|
}
|
|
1646
1962
|
}
|
|
1647
1963
|
await this.sendFrame({ type: "utterance_end", reason });
|
|
@@ -1649,96 +1965,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1649
1965
|
})().catch(() => {});
|
|
1650
1966
|
}
|
|
1651
1967
|
|
|
1652
|
-
/**
|
|
1653
|
-
* Semantic-endpointing gate for a silence-fired turn-end. Returns `null`
|
|
1654
|
-
* synchronously when the decider is not consulted (no decider,
|
|
1655
|
-
* extension cap reached, nothing transcribed) — the boundary then releases
|
|
1656
|
-
* exactly as before. Otherwise resolves `true` when the boundary must NOT
|
|
1657
|
-
* release the utterance now: either the decider held it open (the extension
|
|
1658
|
-
* timer replays the boundary), or the world moved on during the decision
|
|
1659
|
-
* (speech resumed, utterance superseded) and a fresh boundary owns the
|
|
1660
|
-
* release. Every decider failure resolves as "release" within
|
|
1661
|
-
* `endpointDecisionTimeoutMs` (fail-open), so consulting it adds at most
|
|
1662
|
-
* that much end-of-turn latency.
|
|
1663
|
-
*/
|
|
1664
|
-
private maybeHoldUtteranceOpen(
|
|
1665
|
-
utterance: UtteranceCycle,
|
|
1666
|
-
): Promise<boolean> | null {
|
|
1667
|
-
const decider = this.frontDecider;
|
|
1668
|
-
if (!decider) {
|
|
1669
|
-
return null;
|
|
1670
|
-
}
|
|
1671
|
-
if (
|
|
1672
|
-
utterance.endpointExtensionCount >=
|
|
1673
|
-
this.frontModelConfig.endpointMaxExtensions
|
|
1674
|
-
) {
|
|
1675
|
-
return null;
|
|
1676
|
-
}
|
|
1677
|
-
const transcriptSoFar = utterance.finalTranscriptSegments.join(" ").trim();
|
|
1678
|
-
const latestPartial = utterance.latestPartialText;
|
|
1679
|
-
if (transcriptSoFar.length === 0 && !latestPartial) {
|
|
1680
|
-
// Nothing transcribed yet: there is no text to judge.
|
|
1681
|
-
return null;
|
|
1682
|
-
}
|
|
1683
|
-
return this.decideUtteranceHold(
|
|
1684
|
-
decider,
|
|
1685
|
-
utterance,
|
|
1686
|
-
transcriptSoFar,
|
|
1687
|
-
latestPartial,
|
|
1688
|
-
);
|
|
1689
|
-
}
|
|
1690
|
-
|
|
1691
|
-
private async decideUtteranceHold(
|
|
1692
|
-
decider: VoiceFrontDecider,
|
|
1693
|
-
utterance: UtteranceCycle,
|
|
1694
|
-
transcriptSoFar: string,
|
|
1695
|
-
latestPartial: string | null,
|
|
1696
|
-
): Promise<boolean> {
|
|
1697
|
-
const generation = this.vadSpeechGeneration;
|
|
1698
|
-
const decisionStartedAtMs = this.metricsClock();
|
|
1699
|
-
const decision = await decider
|
|
1700
|
-
.decideEndpoint({
|
|
1701
|
-
transcriptSoFar,
|
|
1702
|
-
latestPartial,
|
|
1703
|
-
silenceThresholdMs: this.silenceThresholdMs,
|
|
1704
|
-
extensionCount: utterance.endpointExtensionCount,
|
|
1705
|
-
})
|
|
1706
|
-
// The decider contract never rejects, but the enclosing boundary flow
|
|
1707
|
-
// swallows rejections — a rejecting decider (a buggy stub, a future
|
|
1708
|
-
// regression) would silently drop the silence boundary. Belt and
|
|
1709
|
-
// braces: rejection degrades to release, mirroring speakGeneratedAck.
|
|
1710
|
-
.catch(() => ({ action: "release" as const }));
|
|
1711
|
-
const decisionLatencyMs = Math.max(
|
|
1712
|
-
0,
|
|
1713
|
-
this.metricsClock() - decisionStartedAtMs,
|
|
1714
|
-
);
|
|
1715
|
-
// Re-check the world after the await: the decision window is bounded but
|
|
1716
|
-
// real, and a release based on a stale snapshot could cut off speech.
|
|
1717
|
-
if (
|
|
1718
|
-
this.isUtteranceStale(utterance) ||
|
|
1719
|
-
utterance.released ||
|
|
1720
|
-
utterance.completed
|
|
1721
|
-
) {
|
|
1722
|
-
return true;
|
|
1723
|
-
}
|
|
1724
|
-
if (generation !== this.vadSpeechGeneration) {
|
|
1725
|
-
// Speech resumed during the decision: the utterance keeps accumulating
|
|
1726
|
-
// and the detector's next turn-end re-runs the whole decision. The
|
|
1727
|
-
// discarded decision is not recorded — only acted-on outcomes count.
|
|
1728
|
-
return true;
|
|
1729
|
-
}
|
|
1730
|
-
this.markEndpointDecision(utterance, decision.action, decisionLatencyMs);
|
|
1731
|
-
if (decision.action !== "hold") {
|
|
1732
|
-
return false;
|
|
1733
|
-
}
|
|
1734
|
-
utterance.endpointExtensionCount += 1;
|
|
1735
|
-
this.armEndpointExtensionTimer(utterance);
|
|
1736
|
-
return true;
|
|
1737
|
-
}
|
|
1738
|
-
|
|
1739
1968
|
// Arms the hold-extension replay: after `endpointExtensionMs` of continued
|
|
1740
|
-
// silence the deferred silence boundary re-fires (and
|
|
1741
|
-
//
|
|
1969
|
+
// silence the deferred silence boundary re-fires (and a fresh front-door
|
|
1970
|
+
// leg judges it again, bounded by `endpointMaxExtensions`).
|
|
1742
1971
|
private armEndpointExtensionTimer(utterance: UtteranceCycle): void {
|
|
1743
1972
|
this.clearEndpointExtensionTimer();
|
|
1744
1973
|
this.endpointExtensionTimer = setTimeout(() => {
|
|
@@ -1757,18 +1986,6 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1757
1986
|
}, this.frontModelConfig.endpointExtensionMs);
|
|
1758
1987
|
}
|
|
1759
1988
|
|
|
1760
|
-
/**
|
|
1761
|
-
* Whether front-door routing is active: the single `voice-mode` flag.
|
|
1762
|
-
* Governs both halves of the unified front door — the speculative leg that
|
|
1763
|
-
* decides the endpoint, and the escalation hand-off — so the two can never
|
|
1764
|
-
* disagree. The decider-based hold path (`decideEndpoint`) runs as the
|
|
1765
|
-
* flag-off fallback, so endpointing stays model-decided either way.
|
|
1766
|
-
*/
|
|
1767
|
-
private frontDoorRoutingActive(config?: AssistantConfig): boolean {
|
|
1768
|
-
const cfg = config ?? getConfig();
|
|
1769
|
-
return isAssistantFeatureFlagEnabled("voice-mode", cfg);
|
|
1770
|
-
}
|
|
1771
|
-
|
|
1772
1989
|
/**
|
|
1773
1990
|
* Unified front-door: dispatch the assistant turn speculatively at the
|
|
1774
1991
|
* silence boundary, judging the transcript accumulated so far — the same
|
|
@@ -2414,7 +2631,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2414
2631
|
// without ever reaching finalizePendingUtterance).
|
|
2415
2632
|
this.pendingInterruptedRequest = null;
|
|
2416
2633
|
// ...and it hard-stops any detached background continuations.
|
|
2417
|
-
this.abortDetachedRuns();
|
|
2634
|
+
this.abortDetachedRuns({ reason: "client_interrupt" });
|
|
2418
2635
|
const utterance = this.currentUtterance;
|
|
2419
2636
|
this.stopSessionTranscriber();
|
|
2420
2637
|
if (utterance) {
|
|
@@ -2520,9 +2737,12 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2520
2737
|
this.pendingInterruptedRequest = null;
|
|
2521
2738
|
const continuationResult = this.pendingContinuationResult;
|
|
2522
2739
|
this.pendingContinuationResult = null;
|
|
2740
|
+
const handedOffRequest = this.pendingHandoffRequest;
|
|
2741
|
+
this.pendingHandoffRequest = null;
|
|
2523
2742
|
await this.launchAssistantTurn(utterance, content, {
|
|
2524
2743
|
interruptedRequest,
|
|
2525
2744
|
continuationResult,
|
|
2745
|
+
handedOffRequest,
|
|
2526
2746
|
});
|
|
2527
2747
|
}
|
|
2528
2748
|
|
|
@@ -2538,6 +2758,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2538
2758
|
// Set when a background continuation finished the interrupted reply: its
|
|
2539
2759
|
// answer, appended to the turn's control prompt as context.
|
|
2540
2760
|
continuationResult?: string | null;
|
|
2761
|
+
// Set when a barge-in handed this request's work to a background
|
|
2762
|
+
// subagent, so the model can say it is still running.
|
|
2763
|
+
handedOffRequest?: string | null;
|
|
2541
2764
|
// Unified front-door: dispatch without releasing the utterance. The
|
|
2542
2765
|
// thinking frame and floor-holding timers are deferred until the leg's
|
|
2543
2766
|
// leading verdict commits the turn (see commitSpeculativeTurn); a hold
|
|
@@ -2561,6 +2784,11 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2561
2784
|
progress: {
|
|
2562
2785
|
ops: [],
|
|
2563
2786
|
opsSinceNarration: 0,
|
|
2787
|
+
// Equal epochs at launch: a turn that has done nothing observable yet
|
|
2788
|
+
// has nothing to narrate, so the idle trigger waits for tool activity
|
|
2789
|
+
// or the maxSilenceMs heartbeat.
|
|
2790
|
+
stateEpoch: 0,
|
|
2791
|
+
narratedEpoch: 0,
|
|
2564
2792
|
updatesSpoken: 0,
|
|
2565
2793
|
lastFloorHolderAtMs: null,
|
|
2566
2794
|
lastAudibleAtMs: Date.now(),
|
|
@@ -2569,6 +2797,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2569
2797
|
},
|
|
2570
2798
|
assistantCompleted: false,
|
|
2571
2799
|
ttsDone: false,
|
|
2800
|
+
minimizeRequested: false,
|
|
2572
2801
|
ttsAudioStarted: false,
|
|
2573
2802
|
finalized: false,
|
|
2574
2803
|
speculativePending: opts?.speculative === true,
|
|
@@ -2582,6 +2811,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2582
2811
|
speculativeBuffer: "",
|
|
2583
2812
|
interruptedRequest: opts?.interruptedRequest ?? null,
|
|
2584
2813
|
continuationResult: opts?.continuationResult ?? null,
|
|
2814
|
+
handedOffRequest: opts?.handedOffRequest ?? null,
|
|
2585
2815
|
toolUseStarted: false,
|
|
2586
2816
|
firstDeltaSeen: false,
|
|
2587
2817
|
deltaEpoch: 0,
|
|
@@ -2635,11 +2865,10 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2635
2865
|
}, this.frontModelConfig.endpointDecisionTimeoutMs);
|
|
2636
2866
|
}
|
|
2637
2867
|
|
|
2638
|
-
const cfg = getConfig();
|
|
2639
|
-
|
|
2640
2868
|
// Slow-first-delta spoken ack: if the model produces no delta within the
|
|
2641
|
-
// budget, a short
|
|
2642
|
-
// sessions — a text-only session has no audio gap to
|
|
2869
|
+
// budget, a short front-model-phrased sentence holds the floor. Only
|
|
2870
|
+
// meaningful on TTS sessions — a text-only session has no audio gap to
|
|
2871
|
+
// bridge.
|
|
2643
2872
|
if (this.streamTtsAudio && !opts?.speculative) {
|
|
2644
2873
|
this.armAckTimer(activeTurn);
|
|
2645
2874
|
}
|
|
@@ -2657,26 +2886,14 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2657
2886
|
this.armProgressIdleTimer(activeTurn);
|
|
2658
2887
|
}
|
|
2659
2888
|
|
|
2660
|
-
//
|
|
2661
|
-
//
|
|
2662
|
-
//
|
|
2663
|
-
|
|
2664
|
-
|
|
2665
|
-
|
|
2666
|
-
|
|
2667
|
-
|
|
2668
|
-
// verdict; with it off, a single leg runs on the call-site default —
|
|
2669
|
-
// byte-for-byte the prior behavior.
|
|
2670
|
-
await this.startAssistantLeg(
|
|
2671
|
-
activeTurn,
|
|
2672
|
-
escalateEnabled
|
|
2673
|
-
? {
|
|
2674
|
-
content,
|
|
2675
|
-
routingLeg: "front-door",
|
|
2676
|
-
frontDoor: true,
|
|
2677
|
-
}
|
|
2678
|
-
: { content },
|
|
2679
|
-
);
|
|
2889
|
+
// Front-door leg: a fast model fronts every turn (the `voiceFrontDoor`
|
|
2890
|
+
// call site pins it) and may hand off to a quality leg on the escalate
|
|
2891
|
+
// verdict.
|
|
2892
|
+
await this.startAssistantLeg(activeTurn, {
|
|
2893
|
+
content,
|
|
2894
|
+
routingLeg: "front-door",
|
|
2895
|
+
frontDoor: true,
|
|
2896
|
+
});
|
|
2680
2897
|
}
|
|
2681
2898
|
|
|
2682
2899
|
/**
|
|
@@ -2710,29 +2927,32 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2710
2927
|
}
|
|
2711
2928
|
const { token, utterance, turnId } = activeTurn;
|
|
2712
2929
|
|
|
2713
|
-
//
|
|
2714
|
-
//
|
|
2715
|
-
//
|
|
2716
|
-
//
|
|
2717
|
-
//
|
|
2718
|
-
//
|
|
2719
|
-
//
|
|
2930
|
+
// `rawText` accumulates this leg's full stream. A front-door leg starts
|
|
2931
|
+
// in `deciding` until its leading tokens classify as hold / escalate /
|
|
2932
|
+
// answer: an answer flushes through the shared marker holdback, while an
|
|
2933
|
+
// escalation buffers the post-verdict stream into `bridgeRaw` until the
|
|
2934
|
+
// bridge is complete, then hands off. A default/escalated leg flushes
|
|
2935
|
+
// every delta through the same holdback, so a stray control marker from
|
|
2936
|
+
// the main model is stripped instead of spoken.
|
|
2720
2937
|
let rawText = "";
|
|
2721
|
-
let frontDoorEmitted = 0;
|
|
2722
2938
|
let frontDoorStage: "deciding" | "answer" | "bridging" | "handedOff" =
|
|
2723
2939
|
"deciding";
|
|
2724
2940
|
let bridgeRaw = "";
|
|
2725
2941
|
|
|
2726
|
-
const
|
|
2942
|
+
const emitLegText = (chunk: string): void => {
|
|
2727
2943
|
if (chunk.length === 0) {
|
|
2728
2944
|
return;
|
|
2729
2945
|
}
|
|
2730
2946
|
this.markFirstAssistantDelta(utterance, turnId);
|
|
2731
2947
|
this.markFirstDeltaForAck(activeTurn);
|
|
2732
|
-
//
|
|
2733
|
-
//
|
|
2734
|
-
//
|
|
2735
|
-
//
|
|
2948
|
+
// Send-time abort gate: a delta queued behind a backed-up outbound
|
|
2949
|
+
// frame must not be written once barge-in aborts the turn, or the
|
|
2950
|
+
// cancelled reply's text leaks ahead of turn_cancelled. Key off this
|
|
2951
|
+
// turn's own abort signal — a normal message_complete finalizes and
|
|
2952
|
+
// clears activeAssistantTurn while trailing deltas may still be
|
|
2953
|
+
// draining, so an activeAssistantTurn-based guard would drop them.
|
|
2954
|
+
// Escalation aborts the front-door handle, not this turn's controller,
|
|
2955
|
+
// so legitimate front-door text still sends.
|
|
2736
2956
|
void this.sendFrame(
|
|
2737
2957
|
{ type: "assistant_text_delta", text: chunk },
|
|
2738
2958
|
() => !activeTurn.abortController.signal.aborted && !this.isClosed,
|
|
@@ -2740,26 +2960,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2740
2960
|
this.bufferAssistantTextForTts(token, chunk);
|
|
2741
2961
|
};
|
|
2742
2962
|
|
|
2743
|
-
|
|
2744
|
-
// emitted yet (answer stage only). A trailing "[…" that could still
|
|
2745
|
-
// become a control marker is held back until a later delta disproves
|
|
2746
|
-
// it; a real partial that never completes is simply never spoken.
|
|
2747
|
-
const flushFrontDoor = (): void => {
|
|
2748
|
-
let safeEnd = rawText.length;
|
|
2749
|
-
const lastOpen = rawText.lastIndexOf("[");
|
|
2750
|
-
if (lastOpen >= frontDoorEmitted) {
|
|
2751
|
-
const tail = rawText.slice(lastOpen);
|
|
2752
|
-
if (!tail.includes("]") && couldBeControlMarker(tail)) {
|
|
2753
|
-
safeEnd = lastOpen;
|
|
2754
|
-
}
|
|
2755
|
-
}
|
|
2756
|
-
if (safeEnd > frontDoorEmitted) {
|
|
2757
|
-
emitFrontDoor(
|
|
2758
|
-
stripInternalSpeechMarkers(rawText.slice(frontDoorEmitted, safeEnd)),
|
|
2759
|
-
);
|
|
2760
|
-
frontDoorEmitted = safeEnd;
|
|
2761
|
-
}
|
|
2762
|
-
};
|
|
2963
|
+
const flushLegText = createControlMarkerHoldback(activeTurn, emitLegText);
|
|
2763
2964
|
|
|
2764
2965
|
// Hand off once enough of the post-verdict stream has arrived to cap
|
|
2765
2966
|
// the bridge (sentence terminator or hard cap). Until then nothing is
|
|
@@ -2782,7 +2983,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2782
2983
|
assistantMessageChannel: "vellum",
|
|
2783
2984
|
userMessageInterface: "macos",
|
|
2784
2985
|
assistantMessageInterface: "macos",
|
|
2785
|
-
voiceControlPrompt: buildVoiceControlPrompt(activeTurn
|
|
2986
|
+
voiceControlPrompt: buildVoiceControlPrompt(activeTurn, {
|
|
2987
|
+
...(leg.frontDoor !== undefined ? { frontDoor: leg.frontDoor } : {}),
|
|
2988
|
+
}),
|
|
2786
2989
|
content: leg.content,
|
|
2787
2990
|
isInbound: true,
|
|
2788
2991
|
launchedAtMs: activeTurn.launchedAtMs,
|
|
@@ -2856,7 +3059,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2856
3059
|
}
|
|
2857
3060
|
frontDoorStage = "answer";
|
|
2858
3061
|
}
|
|
2859
|
-
|
|
3062
|
+
flushLegText(rawText);
|
|
2860
3063
|
return;
|
|
2861
3064
|
}
|
|
2862
3065
|
// Defensive: speculative legs are always front-door today, but a
|
|
@@ -2871,24 +3074,8 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2871
3074
|
return;
|
|
2872
3075
|
}
|
|
2873
3076
|
}
|
|
2874
|
-
|
|
2875
|
-
|
|
2876
|
-
void this.sendFrame(
|
|
2877
|
-
{
|
|
2878
|
-
type: "assistant_text_delta",
|
|
2879
|
-
text: msg.text,
|
|
2880
|
-
},
|
|
2881
|
-
// Re-check at send time (mirrors the tts_audio path): a delta
|
|
2882
|
-
// already queued behind a backed-up outbound frame must not be
|
|
2883
|
-
// written once barge-in has aborted the turn, or the cancelled
|
|
2884
|
-
// reply's text leaks ahead of turn_cancelled. Key off this turn's
|
|
2885
|
-
// own abort signal — a normal message_complete finalizes and
|
|
2886
|
-
// clears activeAssistantTurn while trailing deltas may still be
|
|
2887
|
-
// draining, so an activeAssistantTurn-based guard would drop them.
|
|
2888
|
-
() =>
|
|
2889
|
-
!activeTurn.abortController.signal.aborted && !this.isClosed,
|
|
2890
|
-
);
|
|
2891
|
-
this.bufferAssistantTextForTts(token, msg.text);
|
|
3077
|
+
rawText += msg.text;
|
|
3078
|
+
flushLegText(rawText);
|
|
2892
3079
|
},
|
|
2893
3080
|
message_complete: (msg) => {
|
|
2894
3081
|
const current = this.activeAssistantTurn;
|
|
@@ -2930,6 +3117,13 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2930
3117
|
if (leg.frontDoor && current.escalationHandedOff) {
|
|
2931
3118
|
return;
|
|
2932
3119
|
}
|
|
3120
|
+
// A held "[…"-tail that never completed a marker is real text —
|
|
3121
|
+
// force-flush it before assistantCompleted closes the TTS buffer
|
|
3122
|
+
// and completeTtsForTurn signals the drain, so it is spoken and
|
|
3123
|
+
// emitted rather than dropped.
|
|
3124
|
+
if (!leg.frontDoor && msg.type === "message_complete") {
|
|
3125
|
+
flushLegText(rawText, { force: true });
|
|
3126
|
+
}
|
|
2933
3127
|
current.assistantCompleted = true;
|
|
2934
3128
|
if (msg.type === "generation_cancelled") {
|
|
2935
3129
|
void this.finalizeAssistantTurn(
|
|
@@ -2964,6 +3158,38 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2964
3158
|
return;
|
|
2965
3159
|
}
|
|
2966
3160
|
current.toolUseStarted = true;
|
|
3161
|
+
// Foreground wins the workspace: the continuation runs with full
|
|
3162
|
+
// subagent abilities (it can write files, run commands), so the
|
|
3163
|
+
// moment a live turn starts a consequential tool the two could
|
|
3164
|
+
// race on the same workspace, host, or extension state. Kill
|
|
3165
|
+
// running continuations and skip pending detaches; a continuation
|
|
3166
|
+
// only survives while foreground turns stay provably read-only
|
|
3167
|
+
// (the topic-change case it exists for). Fail closed: only
|
|
3168
|
+
// provably non-contending built-ins keep a continuation alive
|
|
3169
|
+
// (see foregroundToolContendsWithContinuation) — a name-based
|
|
3170
|
+
// side-effect denylist misses mutators like plugin/MCP/skill
|
|
3171
|
+
// tools. Over-aborting only drops a best-effort salvage;
|
|
3172
|
+
// under-aborting risks a write race.
|
|
3173
|
+
// An already-completed continuation's stashed answer is kept — it
|
|
3174
|
+
// cannot race anything.
|
|
3175
|
+
//
|
|
3176
|
+
// Accepted residual: the abort is signal-level. A tool call
|
|
3177
|
+
// already executing inside the continuation is not awaited (the
|
|
3178
|
+
// agent loop abandons the in-flight promise on cancellation), so
|
|
3179
|
+
// that one call can briefly overlap the foreground tool. Closing
|
|
3180
|
+
// it would take a cross-conversation execution lock that the
|
|
3181
|
+
// subagent model deliberately does not have — parallel subagents
|
|
3182
|
+
// share the workspace with the parent everywhere — and awaiting
|
|
3183
|
+
// background teardown here would stall the live call's turn.
|
|
3184
|
+
// This gate already makes voice stricter than that baseline; the
|
|
3185
|
+
// residual is bounded to one in-flight call at barge-over time.
|
|
3186
|
+
if (foregroundToolContendsWithContinuation(toolName)) {
|
|
3187
|
+
this.abortDetachedRuns({
|
|
3188
|
+
keepPendingResult: true,
|
|
3189
|
+
reason: "foreground_tool_contends",
|
|
3190
|
+
toolName,
|
|
3191
|
+
});
|
|
3192
|
+
}
|
|
2967
3193
|
// The op counts toward the narration threshold on start (not
|
|
2968
3194
|
// completion) so a burst of slow tools still trips the ops
|
|
2969
3195
|
// trigger while they run.
|
|
@@ -2975,6 +3201,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2975
3201
|
startedAtMs: Date.now(),
|
|
2976
3202
|
});
|
|
2977
3203
|
current.progress.opsSinceNarration += 1;
|
|
3204
|
+
current.progress.stateEpoch += 1;
|
|
2978
3205
|
log.debug({ turnId, toolName }, "Live voice turn started tool use");
|
|
2979
3206
|
// Definitive tool use means the turn is guaranteed slow: speak
|
|
2980
3207
|
// the floor-holding ack now instead of waiting out the
|
|
@@ -3008,14 +3235,27 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3008
3235
|
)
|
|
3009
3236
|
: undefined) ??
|
|
3010
3237
|
findLastIncompleteOp(current.progress.ops, event.toolName);
|
|
3238
|
+
// A long-running op finishing is the beat the user has been
|
|
3239
|
+
// waiting through: it narrates immediately rather than waiting for
|
|
3240
|
+
// `opsThreshold` more ops, which on a one-slow-tool turn never
|
|
3241
|
+
// arrive. Short ops stay on the ops trigger — narrating every
|
|
3242
|
+
// quick lookup is the chatter this cadence exists to avoid.
|
|
3243
|
+
let trigger: "ops" | "op_complete" = "ops";
|
|
3011
3244
|
if (op) {
|
|
3012
3245
|
op.completedAtMs = Date.now();
|
|
3013
3246
|
if (event.isError !== undefined) {
|
|
3014
3247
|
op.isError = event.isError;
|
|
3015
3248
|
}
|
|
3016
3249
|
op.resultPreview = event.resultPreview;
|
|
3250
|
+
if (
|
|
3251
|
+
op.completedAtMs - op.startedAtMs >=
|
|
3252
|
+
this.frontModelConfig.progress.longOpMs
|
|
3253
|
+
) {
|
|
3254
|
+
trigger = "op_complete";
|
|
3255
|
+
}
|
|
3017
3256
|
}
|
|
3018
|
-
|
|
3257
|
+
current.progress.stateEpoch += 1;
|
|
3258
|
+
this.maybeNarrateProgress(current, trigger);
|
|
3019
3259
|
},
|
|
3020
3260
|
},
|
|
3021
3261
|
onError: (message) => {
|
|
@@ -3271,10 +3511,10 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3271
3511
|
// the user's ears — not time since launch, so it covers mid-turn silences
|
|
3272
3512
|
// for the whole turn. On expiry with audio still pending, or with the
|
|
3273
3513
|
// silence not yet a full interval old, it re-arms for the remainder; only a
|
|
3274
|
-
// full interval of audible silence
|
|
3275
|
-
//
|
|
3276
|
-
//
|
|
3277
|
-
// turn
|
|
3514
|
+
// full interval of audible silence reaches the narration gatekeeper. The
|
|
3515
|
+
// interval is a polling cadence, not a speaking cadence: most ticks find
|
|
3516
|
+
// nothing new to report and stay quiet, so what the user hears follows the
|
|
3517
|
+
// turn's tool activity (with `maxSilenceMs` as the heartbeat ceiling).
|
|
3278
3518
|
private armProgressIdleTimer(
|
|
3279
3519
|
turn: ActiveAssistantTurn,
|
|
3280
3520
|
delayMs?: number,
|
|
@@ -3302,16 +3542,22 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3302
3542
|
}, delayMs ?? this.frontModelConfig.progress.idleIntervalMs);
|
|
3303
3543
|
}
|
|
3304
3544
|
|
|
3305
|
-
// Wall-clock instant the current audible silence turns a full interval old
|
|
3306
|
-
// measured from the latest of the last emitted segment, the estimated
|
|
3307
|
-
// client playback end, and the last enqueued filler.
|
|
3545
|
+
// Wall-clock instant the current audible silence turns a full interval old.
|
|
3308
3546
|
private progressIdleDeadlineMs(turn: ActiveAssistantTurn): number {
|
|
3309
3547
|
return (
|
|
3310
|
-
|
|
3311
|
-
|
|
3312
|
-
|
|
3313
|
-
|
|
3314
|
-
|
|
3548
|
+
this.progressSilenceSinceMs(turn) +
|
|
3549
|
+
this.frontModelConfig.progress.idleIntervalMs
|
|
3550
|
+
);
|
|
3551
|
+
}
|
|
3552
|
+
|
|
3553
|
+
// When the turn's current audible silence began: the latest of the last
|
|
3554
|
+
// emitted segment, the estimated client playback end, and the last enqueued
|
|
3555
|
+
// filler.
|
|
3556
|
+
private progressSilenceSinceMs(turn: ActiveAssistantTurn): number {
|
|
3557
|
+
return Math.max(
|
|
3558
|
+
turn.progress.lastAudibleAtMs,
|
|
3559
|
+
this.assistantPlaybackTailUntilMs,
|
|
3560
|
+
turn.progress.lastFloorHolderAtMs ?? 0,
|
|
3315
3561
|
);
|
|
3316
3562
|
}
|
|
3317
3563
|
|
|
@@ -3379,17 +3625,39 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3379
3625
|
);
|
|
3380
3626
|
}
|
|
3381
3627
|
|
|
3628
|
+
// The idle tick has something worth saying when the turn's tool activity has
|
|
3629
|
+
// moved since the last narration described it, or when the silence has run
|
|
3630
|
+
// past `maxSilenceMs` — the heartbeat ceiling that proves the assistant is
|
|
3631
|
+
// still alive on a turn with no observable activity at all. Every other tick
|
|
3632
|
+
// stays quiet, so the cadence follows the work rather than the clock.
|
|
3633
|
+
private progressIdleHasSomethingToSay(turn: ActiveAssistantTurn): boolean {
|
|
3634
|
+
const { progress } = turn;
|
|
3635
|
+
if (progress.stateEpoch !== progress.narratedEpoch) {
|
|
3636
|
+
return true;
|
|
3637
|
+
}
|
|
3638
|
+
const silentForMs = Date.now() - this.progressSilenceSinceMs(turn);
|
|
3639
|
+
if (silentForMs >= this.frontModelConfig.progress.maxSilenceMs) {
|
|
3640
|
+
return true;
|
|
3641
|
+
}
|
|
3642
|
+
log.debug(
|
|
3643
|
+
{ turnId: turn.turnId, silentForMs },
|
|
3644
|
+
"Live voice progress narration held — nothing new since the last update",
|
|
3645
|
+
);
|
|
3646
|
+
return false;
|
|
3647
|
+
}
|
|
3648
|
+
|
|
3382
3649
|
// Gatekeeper for spoken progress narration: it speaks only while the turn
|
|
3383
3650
|
// is audibly silent, spaced `minGapMs` from any spoken floor-holder (ack or
|
|
3384
|
-
// narration), one generation at a time, and —
|
|
3385
|
-
//
|
|
3386
|
-
//
|
|
3387
|
-
//
|
|
3388
|
-
//
|
|
3389
|
-
// count, so the next tool event
|
|
3651
|
+
// narration), one generation at a time, and — per trigger — only once the
|
|
3652
|
+
// ops trigger has `opsThreshold` ops accumulated or the idle trigger has
|
|
3653
|
+
// something new to report. No per-turn count cap: the cadence guards bound
|
|
3654
|
+
// the rate, and going quiet deep into a long turn is the failure mode
|
|
3655
|
+
// narration exists to prevent. Every failing guard short-circuits silently;
|
|
3656
|
+
// a skipped ops trigger keeps its accumulated count, so the next tool event
|
|
3657
|
+
// or idle tick retries.
|
|
3390
3658
|
private maybeNarrateProgress(
|
|
3391
3659
|
turn: ActiveAssistantTurn,
|
|
3392
|
-
trigger: "ops" | "idle",
|
|
3660
|
+
trigger: "ops" | "idle" | "op_complete",
|
|
3393
3661
|
): void {
|
|
3394
3662
|
const cfg = this.frontModelConfig.progress;
|
|
3395
3663
|
const { progress } = turn;
|
|
@@ -3406,7 +3674,8 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3406
3674
|
progress.narrationInFlight ||
|
|
3407
3675
|
(progress.lastFloorHolderAtMs !== null &&
|
|
3408
3676
|
Date.now() - progress.lastFloorHolderAtMs < cfg.minGapMs) ||
|
|
3409
|
-
(trigger === "ops" && progress.opsSinceNarration < cfg.opsThreshold)
|
|
3677
|
+
(trigger === "ops" && progress.opsSinceNarration < cfg.opsThreshold) ||
|
|
3678
|
+
(trigger === "idle" && !this.progressIdleHasSomethingToSay(turn))
|
|
3410
3679
|
) {
|
|
3411
3680
|
return;
|
|
3412
3681
|
}
|
|
@@ -3418,18 +3687,22 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3418
3687
|
// on the turn's ordered TTS queue. The decider internally bounds the call
|
|
3419
3688
|
// by `progress.generationTimeoutMs` and resolves null on every failure
|
|
3420
3689
|
// mode; on null the idle trigger falls back to a static phrase (silence is
|
|
3421
|
-
// actively harmful there) while the
|
|
3422
|
-
// filler is not worth it when narration was merely opportunistic.
|
|
3690
|
+
// actively harmful there) while the tool-activity triggers stay silent — a
|
|
3691
|
+
// generic filler is not worth it when narration was merely opportunistic.
|
|
3423
3692
|
private async speakProgressUpdate(
|
|
3424
3693
|
turn: ActiveAssistantTurn,
|
|
3425
3694
|
frontDecider: VoiceFrontDecider,
|
|
3426
|
-
trigger: "ops" | "idle",
|
|
3695
|
+
trigger: "ops" | "idle" | "op_complete",
|
|
3427
3696
|
): Promise<void> {
|
|
3428
3697
|
const { progress } = turn;
|
|
3429
3698
|
progress.narrationInFlight = true;
|
|
3430
3699
|
// Any delta that lands while the decider call is in flight makes the
|
|
3431
3700
|
// generated text stale — and proves the model is speaking again.
|
|
3432
3701
|
const deltaEpochAtLaunch = turn.deltaEpoch;
|
|
3702
|
+
// The activity this update describes. Tool events that land mid-generation
|
|
3703
|
+
// are news the generated text cannot carry, so they must leave the idle
|
|
3704
|
+
// trigger armed rather than count as already narrated.
|
|
3705
|
+
const stateEpochAtLaunch = progress.stateEpoch;
|
|
3433
3706
|
try {
|
|
3434
3707
|
const now = Date.now();
|
|
3435
3708
|
const currentOp = findLastIncompleteOp(progress.ops);
|
|
@@ -3469,7 +3742,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3469
3742
|
// condition may have flipped while generating, and a delta that landed
|
|
3470
3743
|
// mid-generation (deltaEpoch moved) makes the text stale even when the
|
|
3471
3744
|
// resulting audio has already drained by now. The ack-generation and
|
|
3472
|
-
// floor re-checks close the race with a generated ack
|
|
3745
|
+
// floor re-checks close the race with a generated ack that
|
|
3473
3746
|
// STARTED after this generation did — the entry guard rejects a
|
|
3474
3747
|
// narration while an ack generation is already pending, but cannot see
|
|
3475
3748
|
// one that begins mid-generation: while that ack is pending it has not
|
|
@@ -3489,7 +3762,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3489
3762
|
}
|
|
3490
3763
|
let raw = generated;
|
|
3491
3764
|
if (raw === null) {
|
|
3492
|
-
if (trigger
|
|
3765
|
+
if (trigger !== "idle") {
|
|
3493
3766
|
return;
|
|
3494
3767
|
}
|
|
3495
3768
|
raw = pickProgressPhrase(this.progressPhraseCounter++);
|
|
@@ -3498,6 +3771,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3498
3771
|
return;
|
|
3499
3772
|
}
|
|
3500
3773
|
progress.opsSinceNarration = 0;
|
|
3774
|
+
progress.narratedEpoch = stateEpochAtLaunch;
|
|
3501
3775
|
progress.updatesSpoken += 1;
|
|
3502
3776
|
// Like the ack mark, recorded only when narration audio actually
|
|
3503
3777
|
// enqueued (decider text or static fallback alike).
|
|
@@ -3521,30 +3795,22 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3521
3795
|
kind: LiveVoiceSpokenAckKind,
|
|
3522
3796
|
toolName?: string,
|
|
3523
3797
|
): void {
|
|
3524
|
-
|
|
3525
|
-
|
|
3526
|
-
//
|
|
3527
|
-
// any other ack trigger while the generation is in flight; the async
|
|
3528
|
-
// path undoes it if the ack turns out moot.
|
|
3529
|
-
void this.speakGeneratedAck(
|
|
3530
|
-
activeTurn,
|
|
3531
|
-
this.frontDecider,
|
|
3532
|
-
kind,
|
|
3533
|
-
toolName,
|
|
3534
|
-
);
|
|
3798
|
+
if (!this.frontDecider) {
|
|
3799
|
+
// No decider, no ack: a canned line is worse than the silence, which
|
|
3800
|
+
// progress narration already covers once the turn runs long.
|
|
3535
3801
|
return;
|
|
3536
3802
|
}
|
|
3537
|
-
|
|
3538
|
-
|
|
3539
|
-
|
|
3540
|
-
|
|
3541
|
-
|
|
3803
|
+
activeTurn.ackSpoken = true;
|
|
3804
|
+
// Optimistic ackSpoken above keeps the one-per-turn budget honest for
|
|
3805
|
+
// any other ack trigger while the generation is in flight; the async
|
|
3806
|
+
// path undoes it if the ack turns out moot.
|
|
3807
|
+
void this.speakGeneratedAck(activeTurn, this.frontDecider, kind, toolName);
|
|
3542
3808
|
}
|
|
3543
3809
|
|
|
3544
|
-
//
|
|
3545
|
-
// for one short contextual sentence, static phrase on null. The decider
|
|
3810
|
+
// Ask the front model for one short contextual sentence. The decider
|
|
3546
3811
|
// internally bounds the call by `ackGenerationTimeoutMs` and resolves null
|
|
3547
|
-
// on every failure mode, so the enqueue never waits beyond that budget
|
|
3812
|
+
// on every failure mode, so the enqueue never waits beyond that budget; a
|
|
3813
|
+
// null generation speaks nothing.
|
|
3548
3814
|
private async speakGeneratedAck(
|
|
3549
3815
|
activeTurn: ActiveAssistantTurn,
|
|
3550
3816
|
frontDecider: VoiceFrontDecider,
|
|
@@ -3567,23 +3833,22 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3567
3833
|
// The decider contract never rejects; belt-and-braces for a stub.
|
|
3568
3834
|
.catch(() => null);
|
|
3569
3835
|
// Liveness re-check after the await: any turnCanSpeakFiller condition
|
|
3570
|
-
// may have flipped while generating, mooting the ack.
|
|
3571
|
-
//
|
|
3572
|
-
|
|
3836
|
+
// may have flipped while generating, mooting the ack. A null generation
|
|
3837
|
+
// (no provider, timeout, unusable output) speaks nothing. Either way,
|
|
3838
|
+
// undo the optimistic ackSpoken so the flag reflects reality and a
|
|
3839
|
+
// later trigger this turn can still hold the floor.
|
|
3840
|
+
if (generated === null || !this.turnCanSpeakFiller(activeTurn)) {
|
|
3573
3841
|
activeTurn.ackSpoken = false;
|
|
3574
3842
|
return;
|
|
3575
3843
|
}
|
|
3576
|
-
this.enqueueAckPhrase(
|
|
3577
|
-
activeTurn,
|
|
3578
|
-
generated ?? this.pickStaticAckPhrase(kind),
|
|
3579
|
-
kind,
|
|
3580
|
-
);
|
|
3844
|
+
this.enqueueAckPhrase(activeTurn, generated, kind);
|
|
3581
3845
|
} finally {
|
|
3582
3846
|
activeTurn.ackGenerationPending = false;
|
|
3583
3847
|
}
|
|
3584
3848
|
}
|
|
3585
3849
|
|
|
3586
|
-
// Records the ack-spoken metric only when a phrase actually enqueues
|
|
3850
|
+
// Records the ack-spoken metric only when a phrase actually enqueues; a
|
|
3851
|
+
// phrase that sanitizes away releases the turn's ack budget again.
|
|
3587
3852
|
private enqueueAckPhrase(
|
|
3588
3853
|
activeTurn: ActiveAssistantTurn,
|
|
3589
3854
|
raw: string,
|
|
@@ -3591,7 +3856,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3591
3856
|
): void {
|
|
3592
3857
|
if (this.enqueueFillerPhrase(activeTurn, raw)) {
|
|
3593
3858
|
this.markAckSpoken(activeTurn, kind);
|
|
3859
|
+
return;
|
|
3594
3860
|
}
|
|
3861
|
+
activeTurn.ackSpoken = false;
|
|
3595
3862
|
}
|
|
3596
3863
|
|
|
3597
3864
|
// Sanitize and enqueue one filler sentence (spoken ack or progress
|
|
@@ -3664,6 +3931,20 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3664
3931
|
!this.isClosed,
|
|
3665
3932
|
);
|
|
3666
3933
|
|
|
3934
|
+
// Drain-scoped minimize: the latched marker is consumed here, after
|
|
3935
|
+
// the turn's speech has fully drained — never mid-speech, never for
|
|
3936
|
+
// a barged-in turn, at most once per turn.
|
|
3937
|
+
if (
|
|
3938
|
+
currentTurn.minimizeRequested &&
|
|
3939
|
+
!currentTurn.abortController.signal.aborted
|
|
3940
|
+
) {
|
|
3941
|
+
currentTurn.minimizeRequested = false;
|
|
3942
|
+
await this.sendFrame(
|
|
3943
|
+
{ type: "minimize_room", turnId: currentTurn.turnId },
|
|
3944
|
+
() => !this.isClosed,
|
|
3945
|
+
);
|
|
3946
|
+
}
|
|
3947
|
+
|
|
3667
3948
|
if (this.activeAssistantTurn?.token === token) {
|
|
3668
3949
|
if (currentTurn.handle && currentTurn.finalized) {
|
|
3669
3950
|
this.activeAssistantTurn = null;
|
|
@@ -4450,12 +4731,32 @@ async function defaultSpawnBackgroundContinuation(args: {
|
|
|
4450
4731
|
label: string;
|
|
4451
4732
|
signal: AbortSignal;
|
|
4452
4733
|
}): Promise<string> {
|
|
4453
|
-
|
|
4454
|
-
|
|
4455
|
-
|
|
4456
|
-
|
|
4457
|
-
|
|
4734
|
+
// getOrCreateConversation (not a raw registry read): it rebuilds a stale
|
|
4735
|
+
// instance and awaits loadFromDb, so the snapshot below sees the persisted
|
|
4736
|
+
// history. A raw findConversation can return a cold instance whose in-memory
|
|
4737
|
+
// `messages` is empty, which silently forks a continuation with no context.
|
|
4738
|
+
const { getOrCreateConversation } =
|
|
4739
|
+
await import("../daemon/conversation-store.js");
|
|
4740
|
+
const parentConversation = await getOrCreateConversation(
|
|
4741
|
+
args.parentConversationId,
|
|
4742
|
+
);
|
|
4743
|
+
// Belt-and-suspenders for a resident-but-unhydrated instance: the teardown
|
|
4744
|
+
// settle that gated this spawn guarantees the interrupted turn's partial is
|
|
4745
|
+
// persisted, so an empty in-memory history on a conversation that has rows
|
|
4746
|
+
// means the instance is cold — hydrate before snapshotting. A genuinely new
|
|
4747
|
+
// conversation loads zero rows; harmless.
|
|
4748
|
+
if (parentConversation.getMessages().length === 0) {
|
|
4749
|
+
await parentConversation.loadFromDb();
|
|
4458
4750
|
}
|
|
4751
|
+
// The bridge stamps trust per-turn and clears it at teardown, which has
|
|
4752
|
+
// settled by now — inheriting from the parent would read the cleared window
|
|
4753
|
+
// and run the continuation fail-closed as `unknown`, denying every
|
|
4754
|
+
// consequential tool. Resolve the same guardian trust the foreground turn
|
|
4755
|
+
// ran under and pass it explicitly (resolution itself stays fail-closed:
|
|
4756
|
+
// on a miss the continuation runs as `unknown`, exactly as before).
|
|
4757
|
+
const trustContext = await resolveLocalLiveVoiceTrustContext(
|
|
4758
|
+
args.parentConversationId,
|
|
4759
|
+
);
|
|
4459
4760
|
return await getSubagentManager().spawnAndAwait(
|
|
4460
4761
|
{
|
|
4461
4762
|
parentConversationId: args.parentConversationId,
|
|
@@ -4463,18 +4764,34 @@ async function defaultSpawnBackgroundContinuation(args: {
|
|
|
4463
4764
|
objective: args.objective,
|
|
4464
4765
|
fork: true,
|
|
4465
4766
|
sendResultToUser: false,
|
|
4466
|
-
//
|
|
4467
|
-
//
|
|
4468
|
-
//
|
|
4469
|
-
//
|
|
4470
|
-
//
|
|
4471
|
-
|
|
4472
|
-
|
|
4767
|
+
// Full subagent abilities: the continuation runs like any other
|
|
4768
|
+
// background subagent, so it can genuinely finish build-shaped work
|
|
4769
|
+
// (JARVIS-1354). Side effects are governed by the standard
|
|
4770
|
+
// non-interactive permission path under the explicit trust context
|
|
4771
|
+
// resolved above — auto-approved up to the background risk threshold,
|
|
4772
|
+
// auto-denied above it — the same policy the foreground voice turn it
|
|
4773
|
+
// continues ran under. Workspace write races with the user's next
|
|
4774
|
+
// foreground turn are prevented by the session's foreground-wins abort
|
|
4775
|
+
// (a side-effecting tool start on a live turn aborts running
|
|
4776
|
+
// continuations; see the tool_use_start handler).
|
|
4777
|
+
...(trustContext ? { trustContext } : {}),
|
|
4778
|
+
parentMessages: [...parentConversation.getMessages()],
|
|
4473
4779
|
parentSystemPrompt: parentConversation.getCurrentSystemPrompt(),
|
|
4474
4780
|
},
|
|
4475
|
-
//
|
|
4476
|
-
//
|
|
4477
|
-
|
|
4781
|
+
// Broadcast subagent events to clients attached to this conversation so
|
|
4782
|
+
// the continuation appears in the UI like any other handoff — a background
|
|
4783
|
+
// run the user cannot see reads as the assistant silently dropping their
|
|
4784
|
+
// work. "Silent" means never SPOKEN unprompted (live-voice has no unbidden
|
|
4785
|
+
// TTS); it was never meant to mean invisible.
|
|
4786
|
+
//
|
|
4787
|
+
// NOT the conversation's own sender: the voice bridge resets that to a
|
|
4788
|
+
// no-op at turn teardown (see voice-session-bridge's clientCallbackInstalled
|
|
4789
|
+
// reset), and the detach deliberately waits for that teardown before
|
|
4790
|
+
// spawning — so a sender-based route is guaranteed to be dead by the time
|
|
4791
|
+
// these events fire. `broadcastMessage` is the same path the bridge itself
|
|
4792
|
+
// uses to reach an attached web client. The subagent events carry
|
|
4793
|
+
// `parentConversationId`, not `conversationId`, so scope explicitly.
|
|
4794
|
+
(msg) => broadcastMessage(msg, args.parentConversationId),
|
|
4478
4795
|
{ signal: args.signal },
|
|
4479
4796
|
);
|
|
4480
4797
|
}
|