@vellumai/assistant 0.12.1 → 0.12.2-dev.202609162214.308bda0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +2 -0
- package/AGENTS.md +5 -1
- package/ARCHITECTURE.md +20 -21
- package/Dockerfile +7 -7
- package/docs/architecture/integrations.md +7 -0
- package/docs/architecture/memory.md +11 -2
- package/docs/architecture/turn-actor.md +15 -8
- package/docs/credential-execution-service.md +1 -1
- package/docs/desktop-browser-cli.md +35 -0
- package/docs/guardian-request-flow.md +16 -0
- package/node_modules/@vellumai/environments/src/shell.test.ts +21 -0
- package/node_modules/@vellumai/environments/src/shell.ts +24 -0
- package/node_modules/@vellumai/gateway-client/src/inbound-contract.ts +8 -2
- package/node_modules/@vellumai/slack-text/src/index.ts +13 -8
- package/node_modules/@vellumai/slack-text/src/label-resolution-entities.test.ts +95 -0
- package/openapi.yaml +564 -111
- package/package.json +7 -4
- package/scripts/bundled-plugin-packages.ts +154 -0
- package/scripts/generate-bundled-plugin-packages.ts +29 -0
- package/scripts/postinstall.ts +33 -0
- package/scripts/smoke-desktop-browser-cli.ts +280 -0
- package/scripts/smoke-desktop-window-theme.ts +186 -0
- package/scripts/sync-llm-catalog.ts +3 -0
- package/scripts/test.ts +15 -13
- package/scripts/voice-ttft-spike.ts +2 -2
- package/src/__tests__/agent-loop.test.ts +124 -0
- package/src/__tests__/always-loaded-tools-guard.test.ts +18 -10
- package/src/__tests__/anthropic-provider.test.ts +0 -63
- package/src/__tests__/app-compiler.test.ts +6 -0
- package/src/__tests__/app-control-tool-schemas.test.ts +5 -2
- package/src/__tests__/app-dir-path-guard.test.ts +1 -3
- package/src/__tests__/approval-interception-trust-gates.test.ts +40 -0
- package/src/__tests__/assistant-event-hub-machine-name.test.ts +6 -7
- package/src/__tests__/assistant-event-hub.test.ts +44 -0
- package/src/__tests__/attachments-store.test.ts +22 -3
- package/src/__tests__/channel-approval.test.ts +9 -14
- package/src/__tests__/channel-reply-delivery.test.ts +56 -0
- package/src/__tests__/computer-use-screenshot-attachments.test.ts +411 -0
- package/src/__tests__/computer-use-screenshot-selection.test.ts +50 -0
- package/src/__tests__/computer-use-skill-manifest-regression.test.ts +1 -1
- package/src/__tests__/computer-use-skill-proxy-bridge.test.ts +5 -2
- package/src/__tests__/computer-use-tools.test.ts +19 -7
- package/src/__tests__/config-schema-cmd.test.ts +2 -2
- package/src/__tests__/config-schema.test.ts +29 -30
- package/src/__tests__/config-sounds-sync.test.ts +23 -0
- package/src/__tests__/config-watcher.test.ts +23 -8
- package/src/__tests__/connection-model-compat.test.ts +2 -2
- package/src/__tests__/conversation-agent-loop-inference-profile.test.ts +2 -0
- package/src/__tests__/conversation-agent-loop-overflow.test.ts +2 -0
- package/src/__tests__/conversation-agent-loop.test.ts +257 -6
- package/src/__tests__/conversation-attachments.test.ts +106 -0
- package/src/__tests__/conversation-error.test.ts +4 -1
- package/src/__tests__/conversation-fork-crud.test.ts +191 -0
- package/src/__tests__/conversation-key-store-disk-view.test.ts +1 -1
- package/src/__tests__/conversation-pairing.test.ts +26 -0
- package/src/__tests__/conversation-process-app-control-preactivation.test.ts +4 -10
- package/src/__tests__/conversation-queue.test.ts +48 -207
- package/src/__tests__/conversation-retry-discard.test.ts +54 -1
- package/src/__tests__/conversation-runtime-assembly.test.ts +33 -3
- package/src/__tests__/conversation-skill-tools.test.ts +27 -0
- package/src/__tests__/conversation-surfaces-app-control.test.ts +8 -3
- package/src/__tests__/conversation-surfaces-point-at-budget.test.ts +1 -0
- package/src/__tests__/conversation-surfaces-point-at-capability.test.ts +1 -0
- package/src/__tests__/conversation-surfaces-task-progress.test.ts +67 -0
- package/src/__tests__/conversation-title-service.test.ts +14 -10
- package/src/__tests__/conversation-tool-setup-app-refresh.test.ts +3 -5
- package/src/__tests__/conversation-tool-setup-attribution.test.ts +51 -0
- package/src/__tests__/credential-execution-client.test.ts +70 -53
- package/src/__tests__/credential-record-retire.test.ts +171 -1
- package/src/__tests__/credential-routes.test.ts +250 -3
- package/src/__tests__/credential-security-invariants.test.ts +5 -3
- package/src/__tests__/cu-unified-flow.test.ts +14 -4
- package/src/__tests__/db-conversation-tool-surface.test.ts +144 -0
- package/src/__tests__/document-sync-tags.test.ts +25 -0
- package/src/__tests__/drain-kick-guard.test.ts +0 -2
- package/src/__tests__/drain-requeue-on-contention.test.ts +0 -6
- package/src/__tests__/events-tail-route.test.ts +33 -0
- package/src/__tests__/evict-conversations-for-reload.test.ts +1 -1
- package/src/__tests__/external-plugin-loader.test.ts +52 -0
- package/src/__tests__/git-porcelain.test.ts +47 -0
- package/src/__tests__/headless-browser-mode.test.ts +48 -0
- package/src/__tests__/heartbeat-service.test.ts +56 -40
- package/src/__tests__/helpers/set-workspace-mcp.ts +11 -0
- package/src/__tests__/host-bash-proxy.test.ts +26 -45
- package/src/__tests__/host-browser-proxy.test.ts +6 -17
- package/src/__tests__/host-cu-proxy.test.ts +712 -18
- package/src/__tests__/host-file-proxy-targeted.test.ts +23 -0
- package/src/__tests__/host-file-proxy.test.ts +11 -23
- package/src/__tests__/host-transfer-proxy-targeted.test.ts +30 -9
- package/src/__tests__/inference-profile-session-handler.test.ts +26 -0
- package/src/__tests__/list-all-apps.test.ts +40 -8
- package/src/__tests__/list-messages-camera-frame.test.ts +121 -0
- package/src/__tests__/list-messages-tool-merge.test.ts +85 -1
- package/src/__tests__/llm-catalog-parity.test.ts +51 -3
- package/src/__tests__/managed-store.test.ts +121 -0
- package/src/__tests__/mcp-auth-routes.test.ts +104 -8
- package/src/__tests__/mcp-client-auth.test.ts +54 -0
- package/src/__tests__/mcp-config-secret-boundary.test.ts +78 -136
- package/src/__tests__/mcp-health-check.test.ts +40 -43
- package/src/__tests__/mcp-list-plugin-servers.test.ts +85 -42
- package/src/__tests__/mcp-tool-annotations-risk.test.ts +4 -2
- package/src/__tests__/message-deleted-hook-dispatch.test.ts +119 -0
- package/src/__tests__/messages-after-tiebreaker.test.ts +91 -0
- package/src/__tests__/messages-read-boundary-guard.test.ts +1 -1
- package/src/__tests__/messaging-send-tool.test.ts +355 -548
- package/src/__tests__/model-intents.test.ts +3 -0
- package/src/__tests__/notification-vellum-adapter.test.ts +85 -2
- package/src/__tests__/oauth-apps-routes.test.ts +1 -0
- package/src/__tests__/oauth-commands-routes.test.ts +282 -101
- package/src/__tests__/oauth-connect-orchestrator.test.ts +2 -0
- package/src/__tests__/oauth-provider-profiles.test.ts +18 -0
- package/src/__tests__/oauth-provider-serializer.test.ts +23 -0
- package/src/__tests__/oauth-providers-routes.test.ts +3 -0
- package/src/__tests__/openai-provider.test.ts +1 -0
- package/src/__tests__/outlook-messaging-provider.test.ts +6 -6
- package/src/__tests__/persist-media-references.test.ts +50 -0
- package/src/__tests__/platform-callback-registration.test.ts +8 -16
- package/src/__tests__/platform-client-verify-credential.test.ts +2 -0
- package/src/__tests__/plugin-import-boundary-guard.test.ts +13 -0
- package/src/__tests__/qdrant-manager.test.ts +132 -0
- package/src/__tests__/resolve-app.test.ts +135 -0
- package/src/__tests__/run-conversation-turn-persistence.test.ts +138 -1
- package/src/__tests__/scaffold-managed-skill-tool.test.ts +317 -6
- package/src/__tests__/schedule-routes.test.ts +20 -0
- package/src/__tests__/scheduler-result-notification.test.ts +23 -4
- package/src/__tests__/script-proxy-certs.test.ts +35 -0
- package/src/__tests__/secret-routes-acp-guard.test.ts +59 -1
- package/src/__tests__/secret-routes-platform-proxy.test.ts +86 -31
- package/src/__tests__/secret-routes-scrub.test.ts +22 -27
- package/src/__tests__/secure-keys-managed-failover.test.ts +8 -0
- package/src/__tests__/secure-keys.test.ts +7 -3
- package/src/__tests__/service-account-token.test.ts +247 -0
- package/src/__tests__/skill-feature-flags-integration.test.ts +1 -1
- package/src/__tests__/skill-feature-flags.test.ts +2 -2
- package/src/__tests__/skills.test.ts +20 -4
- package/src/__tests__/source-fingerprint.test.ts +184 -0
- package/src/__tests__/subagent-tool-gate-mode.test.ts +239 -3
- package/src/__tests__/subagent-tools.test.ts +3 -2
- package/src/__tests__/telephony-tts-guard.test.ts +38 -3
- package/src/__tests__/terminal-tools.test.ts +8 -0
- package/src/__tests__/test-support/computer-use-skill-harness.ts +3 -2
- package/src/__tests__/tool-result-metadata-plumbing.test.ts +63 -0
- package/src/__tests__/ui-channel-variants.test.ts +1 -56
- package/src/__tests__/unicode.test.ts +49 -0
- package/src/__tests__/user-plugin-loader.test.ts +21 -0
- package/src/__tests__/vellum-self-knowledge-inline-command.test.ts +4 -0
- package/src/__tests__/watch-retro-tool-availability.test.ts +8 -17
- package/src/__tests__/workspace-git-service.test.ts +26 -0
- package/src/__tests__/workspace-migration-154-repair-retired-codex-gpt-5-4-model-ids.test.ts +336 -0
- package/src/acp/__tests__/acp-claude-oauth.test.ts +257 -6
- package/src/acp/__tests__/acp-credentials.test.ts +13 -0
- package/src/acp/__tests__/agent-process.test.ts +258 -4
- package/src/acp/__tests__/auth-required.test.ts +96 -0
- package/src/acp/__tests__/claude-token-refresh.test.ts +257 -0
- package/src/acp/__tests__/client-handler.test.ts +90 -2
- package/src/acp/__tests__/helpers/acp-model-option.ts +51 -0
- package/src/acp/__tests__/prepare-agent-env.test.ts +60 -1
- package/src/acp/__tests__/session-manager-persistence.test.ts +5 -1
- package/src/acp/__tests__/session-manager-resume.test.ts +270 -6
- package/src/acp/__tests__/session-manager-usage.test.ts +4 -2
- package/src/acp/__tests__/session-manager.test.ts +895 -4
- package/src/acp/acp-claude-oauth.ts +328 -14
- package/src/acp/acp-credentials.ts +19 -0
- package/src/acp/agent-process.test.ts +4 -1
- package/src/acp/agent-process.ts +96 -10
- package/src/acp/auth-required.ts +50 -1
- package/src/acp/auto-install.test.ts +9 -7
- package/src/acp/claude-token-refresh.ts +150 -0
- package/src/acp/client-handler.ts +16 -3
- package/src/acp/model-config.test.ts +166 -0
- package/src/acp/model-config.ts +138 -0
- package/src/acp/prepare-agent-env.ts +21 -6
- package/src/acp/resolve-agent.test.ts +220 -6
- package/src/acp/resolve-agent.ts +88 -23
- package/src/acp/session-manager.test.ts +196 -3
- package/src/acp/session-manager.ts +441 -18
- package/src/acp/session-snapshot.ts +260 -0
- package/src/acp/types.ts +50 -5
- package/src/agent/loop.ts +20 -0
- package/src/api/attachment-provenance.test.ts +68 -0
- package/src/api/computer-use-tool.test.ts +55 -0
- package/src/api/computer-use-tool.ts +27 -0
- package/src/api/events/acp-session-model-update.test.ts +88 -0
- package/src/api/events/acp-session-model-update.ts +46 -0
- package/src/api/events/app-files-changed.ts +4 -3
- package/src/api/events/assistant-outbound-attachment.ts +3 -0
- package/src/api/events/desktop-activity-changed.ts +10 -0
- package/src/api/events/notification-intent.ts +2 -0
- package/src/api/events/user-message-echo.ts +7 -1
- package/src/api/index.ts +17 -0
- package/src/api/responses/conversation-message.ts +8 -0
- package/src/approvals/approval-primitive.ts +5 -2
- package/src/approvals/scoped-approval-grants.ts +6 -2
- package/src/apps/app-store.ts +7 -6
- package/src/apps/resolve-app.ts +81 -0
- package/src/apps/source-fingerprint.ts +266 -0
- package/src/avatar/identity-avatar.ts +7 -2
- package/src/browser/operations.ts +2 -1
- package/src/browser/types.ts +7 -0
- package/src/browser/virtual-desktop-target.ts +34 -0
- package/src/bundler/app-compiler.ts +15 -0
- package/src/calls/__tests__/barge-in-guard.test.ts +81 -0
- package/src/{__tests__ → calls/__tests__}/call-controller.test.ts +503 -82
- package/src/{__tests__ → calls/__tests__}/media-stream-audio-transcode.test.ts +1 -1
- package/src/{__tests__ → calls/__tests__}/media-stream-output.test.ts +37 -8
- package/src/{__tests__ → calls/__tests__}/media-stream-parser.test.ts +1 -1
- package/src/{__tests__ → calls/__tests__}/media-stream-server-integration.test.ts +150 -57
- package/src/{__tests__ → calls/__tests__}/media-stream-stt-session.test.ts +5 -5
- package/src/{__tests__ → calls/__tests__}/media-turn-detector.test.ts +1 -1
- package/src/{live-voice → calls}/__tests__/progress-narration.test.ts +27 -4
- package/src/{live-voice → calls}/__tests__/progress-phrases.test.ts +1 -1
- package/src/calls/__tests__/voice-control-protocol.test.ts +151 -27
- package/src/calls/__tests__/voice-leg-coordinator.test.ts +229 -0
- package/src/calls/__tests__/voice-progress-cadence.test.ts +274 -0
- package/src/{__tests__/voice-session-bridge.test.ts → calls/__tests__/voice-session-bridge-turn-flow.test.ts} +20 -12
- package/src/calls/__tests__/voice-session-bridge.test.ts +140 -3
- package/src/calls/__tests__/voice-triage-escalate.test.ts +105 -0
- package/src/calls/barge-in-guard.ts +118 -0
- package/src/calls/call-constants.ts +24 -4
- package/src/calls/call-controller.ts +499 -199
- package/src/calls/call-transport.ts +16 -0
- package/src/calls/media-stream-output.ts +32 -0
- package/src/calls/media-stream-server.ts +82 -30
- package/src/calls/media-stream-stt-session.ts +15 -0
- package/src/{live-voice → calls}/progress-narration.ts +17 -4
- package/src/calls/resolve-call-tts-provider.ts +33 -2
- package/src/calls/tts-call-strategy.ts +9 -5
- package/src/calls/types.ts +2 -0
- package/src/calls/voice-control-protocol.ts +194 -21
- package/src/calls/voice-leg-coordinator.ts +189 -0
- package/src/calls/voice-progress-cadence.ts +423 -0
- package/src/calls/voice-session-bridge.ts +140 -76
- package/src/calls/voice-triage-escalate.ts +178 -59
- package/src/cli/commands/__tests__/apps.test.ts +245 -0
- package/src/cli/commands/__tests__/bash.test.ts +53 -0
- package/src/cli/commands/__tests__/browser.test.ts +84 -3
- package/src/cli/commands/__tests__/cli-test-harness.ts +21 -3
- package/src/cli/commands/__tests__/plugins.test.ts +85 -0
- package/src/cli/commands/__tests__/roadmap.help.test.ts +19 -0
- package/src/cli/commands/__tests__/schedules.test.ts +14 -0
- package/src/cli/commands/apps.help.ts +67 -13
- package/src/cli/commands/apps.ts +183 -0
- package/src/cli/commands/bash.help.ts +9 -13
- package/src/cli/commands/bash.ts +7 -25
- package/src/cli/commands/browser.help.ts +43 -0
- package/src/cli/commands/browser.ts +52 -8
- package/src/cli/commands/channels/__tests__/send.test.ts +218 -0
- package/src/cli/commands/channels/index.help.ts +70 -2
- package/src/cli/commands/channels/index.ts +16 -3
- package/src/cli/commands/channels/send.ts +128 -0
- package/src/cli/commands/credentials.help.ts +9 -5
- package/src/cli/commands/credentials.ts +2 -2
- package/src/cli/commands/mcp.help.ts +8 -8
- package/src/cli/commands/mcp.ts +1 -1
- package/src/cli/commands/oauth/index.help.ts +10 -0
- package/src/cli/commands/oauth/providers.ts +12 -0
- package/src/cli/commands/oauth/request.test.ts +179 -2
- package/src/cli/commands/oauth/request.ts +41 -10
- package/src/cli/commands/oauth/status.ts +5 -0
- package/src/cli/commands/plugins.ts +73 -23
- package/src/cli/commands/roadmap.help.ts +5 -1
- package/src/cli/commands/roadmap.ts +1 -1
- package/src/cli/commands/schedules.ts +2 -0
- package/src/cli/lib/__tests__/inspect-plugin.test.ts +54 -0
- package/src/cli/lib/__tests__/install-from-github.test.ts +219 -0
- package/src/cli/lib/__tests__/install-from-platform.test.ts +77 -2
- package/src/cli/lib/__tests__/list-installed-plugins.test.ts +45 -3
- package/src/cli/lib/__tests__/local-plugin-upgrade.test.ts +251 -0
- package/src/cli/lib/__tests__/plugin-catalog-cache.test.ts +65 -2
- package/src/cli/lib/__tests__/plugin-catalog-platform.test.ts +14 -0
- package/src/cli/lib/__tests__/plugin-catalog-resolve.test.ts +27 -1
- package/src/cli/lib/__tests__/plugin-details.test.ts +35 -2
- package/src/cli/lib/__tests__/plugin-marketplace.test.ts +20 -0
- package/src/cli/lib/__tests__/plugins-install-offline.test.ts +43 -0
- package/src/cli/lib/__tests__/search-plugins.test.ts +31 -0
- package/src/cli/lib/__tests__/uninstall-plugin.test.ts +122 -5
- package/src/cli/lib/bundled-marketplace.json +962 -81
- package/src/cli/lib/bundled-plugin-packages.json +786 -0
- package/src/cli/lib/bundled-plugin-packages.ts +87 -0
- package/src/cli/lib/diff-plugin.ts +1 -1
- package/src/cli/lib/inspect-plugin.ts +87 -12
- package/src/cli/lib/install-from-github.ts +150 -94
- package/src/cli/lib/install-from-platform.ts +14 -9
- package/src/cli/lib/list-installed-plugins.ts +21 -35
- package/src/cli/lib/plugin-catalog-cache.ts +22 -3
- package/src/cli/lib/plugin-catalog-local.ts +13 -3
- package/src/cli/lib/plugin-catalog-platform.ts +6 -1
- package/src/cli/lib/plugin-catalog-resolve.ts +20 -0
- package/src/cli/lib/plugin-details.ts +33 -8
- package/src/cli/lib/plugin-marketplace.ts +121 -21
- package/src/cli/lib/plugin-pin-history.ts +5 -2
- package/src/cli/lib/search-plugins.ts +58 -16
- package/src/cli/lib/uninstall-plugin.ts +59 -1
- package/src/cli/lib/upgrade-plugin.ts +61 -8
- package/src/cli/output.ts +18 -4
- package/src/config/__tests__/default-profile-catalog.test.ts +22 -0
- package/src/config/__tests__/default-provider.test.ts +4 -0
- package/src/config/__tests__/memory-retrospective-schema.test.ts +15 -1
- package/src/config/__tests__/platform-identity.test.ts +255 -0
- package/src/config/__tests__/platform-rehydration.test.ts +99 -9
- package/src/config/__tests__/plugin-resident-skill-discovery.test.ts +21 -0
- package/src/config/acp-agent-write.ts +43 -0
- package/src/config/acp-defaults.test.ts +13 -3
- package/src/config/acp-defaults.ts +23 -6
- package/src/config/acp-schema.test.ts +81 -0
- package/src/config/acp-schema.ts +30 -2
- package/src/config/bundled-skills/acp/SKILL.md +23 -15
- package/src/config/bundled-skills/acp/TOOLS.json +7 -3
- package/src/config/bundled-skills/app-builder/SKILL.md +12 -9
- package/src/config/bundled-skills/app-builder/references/CUSTOM_ROUTES.md +1 -1
- package/src/config/bundled-skills/computer-use/SKILL.md +59 -2
- package/src/config/bundled-skills/computer-use/TOOLS.json +101 -2
- package/src/config/bundled-skills/computer-use/tools/computer-use-sequence.ts +12 -0
- package/src/config/bundled-skills/media-processing/services/preprocess.ts +2 -1
- package/src/config/bundled-skills/media-processing/services/processing-pipeline.ts +2 -1
- package/src/config/bundled-skills/media-processing/tools/generate-clip.ts +2 -1
- package/src/config/bundled-skills/messaging/SKILL.md +3 -3
- package/src/config/bundled-skills/messaging/TOOLS.json +3 -3
- package/src/config/bundled-skills/messaging/tools/messaging-send.ts +67 -139
- package/src/config/bundled-skills/phone-calls/references/TRANSCRIPTS.md +1 -1
- package/src/config/bundled-skills/playbooks/tools/playbook-create.ts +2 -1
- package/src/config/bundled-skills/playbooks/tools/playbook-update.ts +2 -1
- package/src/config/bundled-skills/schedule/SKILL.md +7 -8
- package/src/config/bundled-skills/screen-annotation/SKILL.md +10 -3
- package/src/config/bundled-skills/screen-annotation/tools/screen-annotation.test.ts +4 -1
- package/src/config/bundled-skills/skill-management/SKILL.md +1 -1
- package/src/config/bundled-skills/skill-management/TOOLS.json +13 -9
- package/src/config/bundled-skills/transcribe/tools/transcribe-media.ts +5 -2
- package/src/config/default-profile-catalog.ts +1 -1
- package/src/config/env.ts +2 -2
- package/src/config/feature-flag-registry.json +33 -13
- package/src/config/loader.ts +11 -17
- package/src/config/memory-v3-gate.test.ts +31 -0
- package/src/config/memory-v3-gate.ts +20 -16
- package/src/config/platform-identity.ts +201 -0
- package/src/config/platform-rehydration.ts +11 -27
- package/src/config/profile-text-generation.test.ts +51 -0
- package/src/config/profile-text-generation.ts +51 -0
- package/src/config/schema.ts +2 -2
- package/src/config/schemas/__tests__/live-voice.test.ts +8 -158
- package/src/config/schemas/__tests__/voice.test.ts +148 -0
- package/src/config/schemas/calls.ts +1 -28
- package/src/config/schemas/live-voice.ts +1 -173
- package/src/config/schemas/llm.ts +7 -2
- package/src/config/schemas/mcp.ts +15 -9
- package/src/config/schemas/memory-retrospective.ts +10 -0
- package/src/config/schemas/monitoring.ts +9 -0
- package/src/config/schemas/services.ts +18 -0
- package/src/config/schemas/skills.ts +0 -14
- package/src/config/schemas/voice.ts +150 -0
- package/src/config/skills.ts +13 -21
- package/src/credential-execution/ces-runtime.ts +2 -2
- package/src/credential-execution/executable-discovery.ts +36 -134
- package/src/credential-execution/process-manager.test.ts +37 -42
- package/src/credential-execution/process-manager.ts +24 -19
- package/src/credential-health/credential-health-service.ts +11 -17
- package/src/daemon/__tests__/conversation-tool-setup-exclude.test.ts +47 -0
- package/src/daemon/__tests__/conversation-tool-setup.test.ts +109 -121
- package/src/daemon/__tests__/plugin-mcp-reconcile.test.ts +5 -1
- package/src/daemon/__tests__/skill-memory-refresh-mtime.test.ts +133 -0
- package/src/daemon/__tests__/turn-tail-deleted-conversation.test.ts +36 -0
- package/src/daemon/app-change-notify.ts +45 -0
- package/src/daemon/assistant-attachments.ts +20 -12
- package/src/daemon/config-watcher.ts +13 -194
- package/src/daemon/conversation-agent-loop-handlers.ts +74 -3
- package/src/daemon/conversation-agent-loop.ts +12 -0
- package/src/daemon/conversation-attachments.ts +93 -12
- package/src/daemon/conversation-client-surface.ts +26 -0
- package/src/daemon/conversation-error.ts +3 -2
- package/src/daemon/conversation-media-retry.ts +2 -1
- package/src/daemon/conversation-process.ts +35 -64
- package/src/daemon/conversation-runtime-assembly.ts +21 -8
- package/src/daemon/conversation-skill-tools.ts +13 -31
- package/src/daemon/conversation-store.ts +1 -1
- package/src/daemon/conversation-surfaces.ts +21 -10
- package/src/daemon/conversation-tool-setup.ts +135 -109
- package/src/daemon/conversation-turn-finalize.ts +26 -13
- package/src/daemon/conversation.ts +17 -3
- package/src/daemon/daemon-control.ts +3 -7
- package/src/daemon/dictation-profile-store.ts +2 -1
- package/src/daemon/handlers/shared.ts +2 -1
- package/src/daemon/handlers/skills.ts +6 -3
- package/src/daemon/host-bash-proxy.ts +8 -12
- package/src/daemon/host-browser-proxy.ts +15 -1
- package/src/daemon/host-cu-proxy.ts +275 -45
- package/src/daemon/host-file-proxy.ts +8 -10
- package/src/daemon/host-proxy-preactivation.ts +4 -1
- package/src/daemon/host-transfer-proxy.ts +14 -18
- package/src/daemon/lifecycle.ts +4 -9
- package/src/daemon/mcp-reload-service.ts +13 -6
- package/src/daemon/message-types/shared.ts +1 -0
- package/src/daemon/message-types/sync.ts +2 -0
- package/src/daemon/orphan-reaper.ts +4 -3
- package/src/daemon/persist-media-references.ts +20 -3
- package/src/daemon/providers-setup.ts +5 -1
- package/src/daemon/reaction-history-render.ts +2 -1
- package/src/daemon/shutdown-handlers.ts +1 -9
- package/src/daemon/skill-memory-refresh.ts +64 -0
- package/src/daemon/tool-setup-types.ts +6 -0
- package/src/daemon/tool-side-effects.ts +7 -18
- package/src/daemon/wake-conversation-ops.ts +38 -15
- package/src/desktop/__tests__/fake-desktop.ts +123 -0
- package/src/desktop/desktop-automation-lease.test.ts +351 -0
- package/src/desktop/desktop-automation-lease.ts +306 -0
- package/src/desktop/desktop-browser-client.test.ts +413 -0
- package/src/desktop/desktop-browser-client.ts +430 -0
- package/src/desktop/desktop-browser-cursor.ts +40 -0
- package/src/desktop/desktop-browser-endpoint.test.ts +78 -0
- package/src/desktop/desktop-browser-endpoint.ts +145 -0
- package/src/desktop/desktop-browser-operations.ts +130 -0
- package/src/desktop/desktop-chrome-policy.test.ts +79 -0
- package/src/desktop/desktop-chrome-policy.ts +47 -0
- package/src/desktop/desktop-chrome-session.test.ts +48 -0
- package/src/desktop/desktop-chrome-session.ts +87 -0
- package/src/desktop/desktop-dependencies.test.ts +131 -0
- package/src/desktop/desktop-dependencies.ts +329 -0
- package/src/desktop/desktop-display.ts +9 -0
- package/src/desktop/desktop-panel-config.test.ts +135 -0
- package/src/desktop/desktop-panel-config.ts +162 -0
- package/src/desktop/desktop-session-manager.test.ts +813 -0
- package/src/desktop/desktop-session-manager.ts +970 -0
- package/src/desktop/desktop-stream-bridge.test.ts +395 -0
- package/src/desktop/desktop-stream-bridge.ts +214 -0
- package/src/desktop/desktop-wallpaper.test.ts +191 -0
- package/src/desktop/desktop-wallpaper.ts +113 -0
- package/src/desktop/desktop-window-drag.ts +177 -0
- package/src/desktop/desktop-window-manager-config.test.ts +230 -0
- package/src/desktop/desktop-window-manager-config.ts +238 -0
- package/src/desktop/desktop-window-theme.ts +170 -0
- package/src/desktop/virtual-desktop-feature.ts +18 -0
- package/src/desktop/virtual-desktop-platform.test.ts +61 -0
- package/src/heartbeat/heartbeat-service.ts +21 -3
- package/src/home/__tests__/feed-writer.test.ts +22 -0
- package/src/home/feed-writer.ts +11 -2
- package/src/hooks/types.ts +33 -0
- package/src/i18n/AGENTS.md +22 -0
- package/src/i18n/__tests__/i18n.test.ts +112 -0
- package/src/i18n/index.ts +16 -0
- package/src/i18n/locales.ts +93 -0
- package/src/i18n/messages.ts +70 -0
- package/src/i18n/title.ts +25 -0
- package/src/inbound/platform-callback-registration.ts +4 -5
- package/src/ipc/__tests__/browser-ipc.test.ts +230 -2
- package/src/ipc/__tests__/cancel-on-disconnect.test.ts +54 -0
- package/src/ipc/assistant-server.ts +15 -1
- package/src/ipc/cli-client.ts +13 -5
- package/src/live-voice/__tests__/live-voice-agent-turn.test.ts +143 -0
- package/src/live-voice/__tests__/live-voice-events.test.ts +9 -7
- package/src/live-voice/__tests__/live-voice-look-follow-up.test.ts +374 -0
- package/src/live-voice/__tests__/live-voice-metrics.test.ts +0 -3
- package/src/live-voice/__tests__/live-voice-progress.test.ts +94 -15
- package/src/live-voice/__tests__/live-voice-session-telemetry.test.ts +19 -0
- package/src/live-voice/__tests__/live-voice-sight-frame-echo.test.ts +238 -0
- package/src/live-voice/__tests__/live-voice-vad.test.ts +660 -5
- package/src/live-voice/__tests__/protocol.test.ts +95 -0
- package/src/live-voice/__tests__/runtime-websocket-shell.test.ts +13 -69
- package/src/live-voice/__tests__/session-controls.test.ts +156 -0
- package/src/live-voice/live-voice-metrics.ts +9 -0
- package/src/live-voice/live-voice-photo.ts +25 -6
- package/src/live-voice/live-voice-session.ts +1081 -760
- package/src/live-voice/protocol.ts +89 -2
- package/src/live-voice/session-controls.ts +197 -0
- package/src/mcp/__tests__/credential-target.test.ts +59 -0
- package/src/mcp/__tests__/effective-config.test.ts +13 -2
- package/src/mcp/__tests__/manager-state.test.ts +128 -0
- package/src/mcp/__tests__/manager-tool-caps.test.ts +111 -0
- package/src/mcp/__tests__/mcp-auth-orchestrator.test.ts +20 -6
- package/src/mcp/__tests__/mcp-auth-state-target.test.ts +49 -0
- package/src/mcp/__tests__/plugin-mcp-oauth-cleanup.test.ts +107 -0
- package/src/mcp/__tests__/plugin-mcp-oauth-provider-isolation.test.ts +89 -0
- package/src/mcp/__tests__/plugin-server-credential-isolation.test.ts +63 -11
- package/src/mcp/__tests__/reload-signal-emission.test.ts +39 -2
- package/src/mcp/__tests__/startup.test.ts +30 -7
- package/src/mcp/__tests__/tool-caps.test.ts +107 -0
- package/src/mcp/__tests__/workspace-mcp-config.test.ts +135 -0
- package/src/mcp/client.ts +64 -16
- package/src/mcp/credential-target.ts +105 -0
- package/src/mcp/effective-config.ts +45 -18
- package/src/mcp/manager.ts +201 -81
- package/src/mcp/mcp-auth-orchestrator.ts +17 -4
- package/src/mcp/mcp-auth-state.ts +38 -6
- package/src/mcp/mcp-header-store.ts +29 -45
- package/src/mcp/mcp-oauth-provider.ts +101 -33
- package/src/mcp/spec-schema.ts +111 -0
- package/src/mcp/startup.ts +41 -17
- package/src/mcp/tool-caps.ts +176 -0
- package/src/mcp/workspace-mcp-config.ts +188 -0
- package/src/messaging/provider-message-metadata.ts +3 -3
- package/src/messaging/provider.ts +10 -3
- package/src/messaging/providers/__tests__/proactive-address.test.ts +95 -0
- package/src/messaging/providers/callback-routing.ts +18 -0
- package/src/messaging/providers/channel-transport.ts +56 -0
- package/src/messaging/providers/discord/transport.ts +17 -0
- package/src/messaging/providers/slack/__tests__/adapter-token-routing.test.ts +6 -17
- package/src/messaging/providers/slack/adapter.ts +7 -44
- package/src/messaging/providers/slack/transport.ts +16 -0
- package/src/messaging/providers/telegram-bot/adapter.ts +1 -48
- package/src/messaging/providers/telegram-bot/client.ts +0 -26
- package/src/messaging/providers/telegram-bot/transport.ts +20 -0
- package/src/messaging/providers/whatsapp/adapter.ts +1 -48
- package/src/messaging/providers/whatsapp/transport.ts +18 -0
- package/src/monitoring/__tests__/mount-watch.test.ts +236 -0
- package/src/monitoring/__tests__/workspace-git-heartbeat.test.ts +8 -0
- package/src/monitoring/mount-watch.ts +348 -0
- package/src/monitoring/plugin-auto-update.ts +6 -0
- package/src/monitoring/worker.ts +9 -0
- package/src/notifications/__tests__/assistant-reply-producer.test.ts +55 -0
- package/src/notifications/__tests__/copy-composer.test.ts +70 -0
- package/src/notifications/__tests__/decision-engine.test.ts +175 -0
- package/src/notifications/__tests__/guardian-feed-projection.test.ts +158 -2
- package/src/notifications/__tests__/home-feed-side-effect.test.ts +153 -3
- package/src/notifications/__tests__/schedule-result-producer.test.ts +99 -3
- package/src/notifications/adapters/macos.ts +3 -0
- package/src/notifications/assistant-reply-producer.ts +8 -3
- package/src/notifications/conversation-pairing.ts +36 -16
- package/src/notifications/copy-composer.ts +11 -3
- package/src/notifications/decision-engine.ts +43 -8
- package/src/notifications/delivered-post-record.ts +11 -1
- package/src/notifications/guardian-feed-projection.ts +125 -5
- package/src/notifications/home-feed-side-effect.ts +55 -7
- package/src/notifications/schedule-result-producer.ts +64 -22
- package/src/oauth/AGENTS.md +4 -0
- package/src/oauth/__tests__/identity-verifier.test.ts +36 -1
- package/src/oauth/__tests__/seed-providers-managed.test.ts +149 -0
- package/src/oauth/byo-connection.test.ts +105 -0
- package/src/oauth/byo-connection.ts +36 -6
- package/src/oauth/connection-resolver.test.ts +100 -0
- package/src/oauth/connection-resolver.ts +120 -3
- package/src/oauth/identity-verifier.ts +24 -0
- package/src/oauth/oauth-store.ts +11 -1
- package/src/oauth/provider-serializer.ts +10 -0
- package/src/oauth/scope-utils.ts +24 -0
- package/src/oauth/seed-providers.ts +367 -27
- package/src/onboarding/checkin-event.ts +2 -1
- package/src/outbound-proxy/certs.ts +3 -5
- package/src/permissions/checker.test.ts +11 -6
- package/src/permissions/confirmation-guardian-request.test.ts +16 -0
- package/src/permissions/confirmation-guardian-request.ts +2 -3
- package/src/persistence/attachments-store.ts +84 -44
- package/src/persistence/compaction-log-store-clickhouse.ts +3 -3
- package/src/persistence/conversation-crud.ts +182 -83
- package/src/persistence/conversation-lineage.ts +12 -4
- package/src/persistence/conversation-plugin-facade.ts +13 -0
- package/src/persistence/conversation-queries.ts +2 -2
- package/src/persistence/conversation-title-placeholders.ts +23 -0
- package/src/persistence/conversation-title-service.ts +9 -26
- package/src/persistence/conversation-tool-surface.ts +86 -0
- package/src/persistence/conversation-types.test.ts +32 -0
- package/src/persistence/conversation-types.ts +26 -4
- package/src/persistence/embeddings/qdrant-manager.ts +163 -18
- package/src/persistence/llm-request-log-sink-clickhouse.ts +3 -3
- package/src/persistence/llm-request-log-source-clickhouse.ts +3 -3
- package/src/persistence/message-cursor.ts +78 -0
- package/src/persistence/message-reads.ts +30 -0
- package/src/persistence/migrations/157-invite-contact-id.ts +18 -6
- package/src/persistence/migrations/378-create-conversation-tool-surfaces.test.ts +78 -0
- package/src/persistence/migrations/378-create-conversation-tool-surfaces.ts +29 -0
- package/src/persistence/migrations/379-oauth-providers-response-ok-field.test.ts +92 -0
- package/src/persistence/migrations/379-oauth-providers-response-ok-field.ts +32 -0
- package/src/persistence/migrations/__tests__/157-invite-contact-id.test.ts +149 -0
- package/src/persistence/schema/conversation-tool-surfaces.ts +22 -0
- package/src/persistence/schema/index.ts +1 -0
- package/src/persistence/schema/memory-core.ts +5 -0
- package/src/persistence/schema/oauth.ts +1 -0
- package/src/persistence/steps.ts +15 -3
- package/src/platform/client.test.ts +4 -2
- package/src/platform/client.ts +2 -10
- package/src/plugin-api/constants.ts +2 -0
- package/src/plugin-api/conversation-turn.ts +31 -7
- package/src/plugin-api/index.ts +10 -1
- package/src/plugin-api/plugin-channel-turn-trust.test.ts +133 -0
- package/src/plugin-api/plugin-channel-turn-trust.ts +71 -0
- package/src/plugin-api/types.ts +2 -0
- package/src/plugins/__tests__/installed-plugin-dirs.test.ts +14 -1
- package/src/plugins/__tests__/mcp-servers.test.ts +38 -8
- package/src/plugins/defaults/index.ts +2 -0
- package/src/plugins/defaults/memory/AGENTS.md +14 -2
- package/src/plugins/defaults/memory/__tests__/buffer-file.test.ts +365 -0
- package/src/plugins/defaults/memory/__tests__/buffer-format.test.ts +43 -0
- package/src/plugins/defaults/memory/__tests__/fixtures/buffer-appender.ts +17 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-accounting.test.ts +46 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-cursor-backfill.test.ts +118 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-cursor-preserve.test.ts +83 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-enqueue.test.ts +9 -5
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-job.test.ts +66 -10
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-prompt.test.ts +20 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-state.test.ts +219 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-sweep.test.ts +31 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-trigger-check.test.ts +63 -0
- package/src/plugins/defaults/memory/__tests__/memory-run-evidence.test.ts +161 -0
- package/src/plugins/defaults/memory/__tests__/shutdown-hook.test.ts +4 -0
- package/src/plugins/defaults/memory/buffer-file.ts +354 -0
- package/src/plugins/defaults/memory/buffer-format.ts +40 -0
- package/src/plugins/defaults/memory/context-search/agent-runner.ts +3 -3
- package/src/plugins/defaults/memory/context-search/format.ts +4 -2
- package/src/plugins/defaults/memory/context-search/sources/memory-v2.ts +7 -3
- package/src/plugins/defaults/memory/context-search/sources/workspace.ts +4 -2
- package/src/plugins/defaults/memory/graph/capability-seed.ts +6 -8
- package/src/plugins/defaults/memory/graph/tool-handlers.ts +1 -42
- package/src/plugins/defaults/memory/hooks/init.ts +10 -0
- package/src/plugins/defaults/memory/hooks/message-deleted.ts +21 -0
- package/src/plugins/defaults/memory/hooks/shutdown.ts +2 -0
- package/src/plugins/defaults/memory/injectors.ts +4 -3
- package/src/plugins/defaults/memory/jobs-worker.ts +13 -0
- package/src/plugins/defaults/memory/memory-retrospective-accounting.ts +34 -60
- package/src/plugins/defaults/memory/memory-retrospective-cursor-backfill.ts +126 -0
- package/src/plugins/defaults/memory/memory-retrospective-cursor-preserve.ts +79 -0
- package/src/plugins/defaults/memory/memory-retrospective-cursor.ts +28 -0
- package/src/plugins/defaults/memory/memory-retrospective-enqueue.ts +3 -5
- package/src/plugins/defaults/memory/memory-retrospective-job.ts +67 -184
- package/src/plugins/defaults/memory/memory-retrospective-prompt.ts +1 -1
- package/src/plugins/defaults/memory/memory-retrospective-state.ts +312 -71
- package/src/plugins/defaults/memory/memory-retrospective-sweep.ts +2 -1
- package/src/plugins/defaults/memory/memory-retrospective-trigger-check.ts +5 -3
- package/src/plugins/defaults/memory/memory-run-evidence.ts +213 -0
- package/src/plugins/defaults/memory/skill-md-mtime-poll.ts +9 -0
- package/src/plugins/defaults/memory/src/memory-item-routes.test.ts +1 -1
- package/src/plugins/defaults/memory/startup.ts +28 -2
- package/src/plugins/defaults/memory/substrate/__tests__/consolidation-job.test.ts +407 -99
- package/src/plugins/defaults/memory/substrate/__tests__/consolidation-prompt-flag-gating-guard.test.ts +10 -0
- package/src/plugins/defaults/memory/substrate/__tests__/prompts-consolidation.test.ts +107 -7
- package/src/plugins/defaults/memory/substrate/__tests__/skill-content.test.ts +7 -9
- package/src/plugins/defaults/memory/substrate/consolidation-job.ts +307 -86
- package/src/plugins/defaults/memory/substrate/consolidation-tool-surface.ts +34 -0
- package/src/plugins/defaults/memory/substrate/page-index.ts +7 -1
- package/src/plugins/defaults/memory/substrate/prompts/consolidation.ts +89 -49
- package/src/plugins/defaults/memory/substrate/skill-content.ts +2 -2
- package/src/plugins/defaults/memory/substrate/sweep-job.ts +1 -1
- package/src/plugins/defaults/memory/tools.ts +1 -1
- package/src/plugins/defaults/memory/v1/graph/consolidation.ts +4 -2
- package/src/plugins/defaults/memory/v1/graph/extraction.ts +4 -1
- package/src/plugins/defaults/memory/v1/graph/retriever.ts +4 -1
- package/src/plugins/defaults/memory/v2/__tests__/migration.test.ts +5 -0
- package/src/plugins/defaults/memory/v2/__tests__/reranker.test.ts +5 -2
- package/src/plugins/defaults/memory/v2/reranker.ts +7 -1
- package/src/plugins/defaults/memory/v3/__tests__/card.test.ts +24 -0
- package/src/plugins/defaults/memory/v3/__tests__/injection.test.ts +81 -1
- package/src/plugins/defaults/memory/v3/__tests__/orchestrate.test.ts +87 -0
- package/src/plugins/defaults/memory/v3/__tests__/pool-select.test.ts +125 -0
- package/src/plugins/defaults/memory/v3/__tests__/sections.test.ts +26 -0
- package/src/plugins/defaults/memory/v3/__tests__/shadow-plugin.test.ts +51 -0
- package/src/plugins/defaults/memory/v3/card.ts +4 -2
- package/src/plugins/defaults/memory/v3/injector.ts +212 -178
- package/src/plugins/defaults/memory/v3/orchestrate.ts +86 -22
- package/src/plugins/defaults/memory/v3/pool-select.ts +13 -8
- package/src/plugins/defaults/memory/v3/sections.ts +10 -3
- package/src/plugins/defaults/memory/v3/shadow-plugin.ts +13 -1
- package/src/plugins/defaults/memory/worker.ts +6 -0
- package/src/plugins/defaults/tool-result-truncate/terminal.ts +1 -46
- package/src/plugins/external-plugin-loader.ts +49 -86
- package/src/plugins/installed-plugin-dirs.ts +6 -4
- package/src/plugins/mcp-servers.ts +24 -90
- package/src/plugins/mtime-cache.ts +2 -1
- package/src/plugins/user-loader.ts +2 -1
- package/src/providers/__tests__/connection-model-compat.test.ts +3 -3
- package/src/providers/__tests__/retry-callsite.test.ts +41 -0
- package/src/providers/__tests__/retry-outbound-sanitize.test.ts +176 -0
- package/src/providers/anthropic/client.ts +0 -47
- package/src/providers/gemini/client.ts +9 -23
- package/src/providers/gemini-thought-signature.test.ts +63 -0
- package/src/providers/gemini-thought-signature.ts +44 -0
- package/src/providers/inference/__tests__/endpoint-probe.test.ts +87 -3
- package/src/providers/inference/adapter-factory.ts +6 -0
- package/src/providers/inference/auth.ts +2 -3
- package/src/providers/inference/connection-availability.ts +27 -8
- package/src/providers/inference/endpoint-probe.ts +36 -12
- package/src/providers/inference/resolve-auth.ts +22 -4
- package/src/providers/inference/service-account-token.ts +248 -0
- package/src/providers/jev/client.test.ts +260 -0
- package/src/providers/jev/client.ts +518 -0
- package/src/providers/model-catalog.ts +92 -3
- package/src/providers/model-intents.ts +10 -5
- package/src/providers/openai/__tests__/chat-completions-output-token-limit.test.ts +141 -0
- package/src/providers/openai/__tests__/chat-completions-provider-reasoning.test.ts +146 -18
- package/src/providers/openai/__tests__/chat-completions-provider-thought-signature.test.ts +613 -0
- package/src/providers/openai/__tests__/google-thought-signature.test.ts +235 -0
- package/src/providers/openai/__tests__/tool-choice-mapping.test.ts +83 -1
- package/src/providers/openai/chat-completions-provider.ts +212 -79
- package/src/providers/openai/codex-models.ts +0 -4
- package/src/providers/openai/google-thought-signature.ts +320 -0
- package/src/providers/openai/responses-provider.ts +9 -2
- package/src/providers/opencode/client.test.ts +240 -4
- package/src/providers/opencode/client.ts +129 -3
- package/src/providers/openrouter/client.ts +20 -0
- package/src/providers/outbound-request-sanitize.ts +96 -0
- package/src/providers/retry.ts +30 -19
- package/src/providers/unparseable-tool-args.ts +5 -1
- package/src/runtime/AGENTS.md +9 -3
- package/src/runtime/__tests__/agent-wake.test.ts +86 -1
- package/src/runtime/__tests__/channel-send.test.ts +443 -0
- package/src/runtime/__tests__/desktop-stream-upgrade.test.ts +132 -0
- package/src/runtime/__tests__/host-observe.test.ts +40 -1
- package/src/runtime/__tests__/live-voice-upgrade.test.ts +58 -0
- package/src/runtime/__tests__/runtime-ws-test-utils.ts +72 -0
- package/src/runtime/__tests__/watch-stream-upgrade.test.ts +59 -0
- package/src/runtime/agent-wake.ts +22 -5
- package/src/runtime/assistant-event-hub.ts +21 -3
- package/src/runtime/auth/same-actor.ts +38 -3
- package/src/runtime/channel-send.ts +316 -0
- package/src/runtime/guardian-action-service.ts +2 -17
- package/src/runtime/guardian-reply-router.ts +1 -8
- package/src/runtime/host-observe.ts +11 -1
- package/src/runtime/http-server.ts +160 -151
- package/src/runtime/migrations/__tests__/vbundle-import-policy.test.ts +89 -0
- package/src/runtime/migrations/vbundle-import-policy.ts +33 -11
- package/src/runtime/routes/__tests__/acp-claude-auth-routes.test.ts +16 -5
- package/src/runtime/routes/__tests__/acp-routes.test.ts +178 -4
- package/src/runtime/routes/__tests__/apps-refresh-route.test.ts +275 -0
- package/src/runtime/routes/__tests__/channel-send-routes.test.ts +127 -0
- package/src/runtime/routes/__tests__/conversation-query-routes.test.ts +108 -1
- package/src/runtime/routes/__tests__/default-provider-routes.test.ts +2 -2
- package/src/runtime/routes/__tests__/dictation-cleanup-guard.test.ts +107 -0
- package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +24 -2
- package/src/runtime/routes/__tests__/mcp-add-default-risk.test.ts +16 -12
- package/src/runtime/routes/__tests__/plugins-routes.test.ts +127 -3
- package/src/runtime/routes/acp-claude-auth-routes.ts +11 -3
- package/src/runtime/routes/acp-routes.test.ts +15 -0
- package/src/runtime/routes/acp-routes.ts +84 -165
- package/src/runtime/routes/app-management-routes.ts +90 -2
- package/src/runtime/routes/app-routes.ts +1 -1
- package/src/runtime/routes/browser-context.ts +39 -0
- package/src/runtime/routes/browser-routes.ts +16 -47
- package/src/runtime/routes/browser-tabs-routes.ts +72 -29
- package/src/runtime/routes/channel-route-shared.ts +1 -9
- package/src/runtime/routes/channel-send-routes.ts +85 -0
- package/src/runtime/routes/conversation-list-routes.ts +14 -3
- package/src/runtime/routes/conversation-query-routes.ts +40 -0
- package/src/runtime/routes/conversation-routes.ts +45 -7
- package/src/runtime/routes/credential-routes.ts +20 -14
- package/src/runtime/routes/desktop-setup-routes.test.ts +89 -0
- package/src/runtime/routes/desktop-setup-routes.ts +41 -0
- package/src/runtime/routes/diagnostics-routes.ts +61 -4
- package/src/runtime/routes/events-routes.ts +4 -0
- package/src/runtime/routes/guardian-approval-interception.ts +24 -0
- package/src/runtime/routes/host-cu-routes.ts +6 -0
- package/src/runtime/routes/inbound-message-handler.ts +2 -3
- package/src/runtime/routes/inbound-stages/background-dispatch.test.ts +1 -1
- package/src/runtime/routes/inbound-stages/background-dispatch.ts +7 -4
- package/src/runtime/routes/index.ts +4 -2
- package/src/runtime/routes/inference-profile-session-handler.ts +11 -1
- package/src/runtime/routes/inference-profiles-routes.ts +9 -0
- package/src/runtime/routes/log-export-routes.ts +2 -0
- package/src/runtime/routes/mcp-auth-routes.ts +141 -137
- package/src/runtime/routes/migration-routes.ts +6 -20
- package/src/runtime/routes/oauth-commands-routes.ts +98 -45
- package/src/runtime/routes/oauth-providers.ts +11 -0
- package/src/runtime/routes/oauth-request-hints.test.ts +246 -0
- package/src/runtime/routes/oauth-request-hints.ts +149 -0
- package/src/runtime/routes/platform-routes.ts +18 -15
- package/src/runtime/routes/plugins-routes.ts +83 -50
- package/src/runtime/routes/schedule-routes.ts +2 -0
- package/src/runtime/routes/secret-routes.ts +99 -119
- package/src/runtime/routes/workspace-routes.test.ts +55 -26
- package/src/runtime/routes/workspace-routes.ts +46 -63
- package/src/runtime/routes/workspace-tree-walk.test.ts +244 -0
- package/src/runtime/routes/workspace-tree-walk.ts +240 -0
- package/src/runtime/routes/workspace-utils.ts +17 -0
- package/src/runtime/services/__tests__/conversation-serializer.test.ts +28 -0
- package/src/runtime/services/conversation-serializer.ts +16 -3
- package/src/runtime/sync/resource-sync-events.ts +6 -1
- package/src/schedule/__tests__/worker-mcp-bootstrap.test.ts +107 -0
- package/src/schedule/__tests__/worker-mcp-readiness.test.ts +173 -0
- package/src/schedule/__tests__/worker-mcp-tools.test.ts +13 -47
- package/src/schedule/run-script.ts +4 -3
- package/src/schedule/schedule-store.ts +9 -2
- package/src/schedule/scheduler.ts +47 -11
- package/src/schedule/tool-surface-readiness.ts +65 -0
- package/src/schedule/worker-mcp.ts +85 -0
- package/src/schedule/worker.ts +9 -31
- package/src/security/__tests__/ces-http-record-backend.test.ts +137 -0
- package/src/security/ces-http-record-backend.ts +120 -0
- package/src/security/ces-rpc-record-backend.ts +7 -0
- package/src/security/secure-keys.ts +60 -48
- package/src/security/untrusted-content.ts +3 -1
- package/src/skills/__tests__/workspace-skill-md-signature.test.ts +60 -0
- package/src/skills/managed-store.ts +153 -31
- package/src/skills/workspace-skill-md-signature.ts +50 -0
- package/src/subagent/consult-context.ts +12 -25
- package/src/tools/__tests__/side-effect-abort-guard.test.ts +1 -0
- package/src/tools/acp/list-agents.test.ts +3 -1
- package/src/tools/acp/spawn.test.ts +146 -11
- package/src/tools/acp/spawn.ts +21 -3
- package/src/tools/acp/status.test.ts +349 -0
- package/src/tools/acp/status.ts +112 -9
- package/src/tools/app-control/skill-proxy-bridge.ts +2 -1
- package/src/tools/browser/browser-execution.ts +45 -14
- package/src/tools/browser/cdp-client/__tests__/factory.test.ts +69 -0
- package/src/tools/browser/cdp-client/accessibility-snapshot.ts +9 -3
- package/src/tools/browser/cdp-client/cdp-inspect/__tests__/ws-transport.test.ts +1 -0
- package/src/tools/browser/cdp-client/cdp-inspect/discovery.ts +2 -1
- package/src/tools/browser/cdp-client/cdp-inspect/ws-transport.ts +5 -0
- package/src/tools/browser/cdp-client/extension-cdp-client.ts +3 -2
- package/src/tools/browser/network-recorder.ts +2 -1
- package/src/tools/capability-offer.test.ts +22 -2
- package/src/tools/capability-offer.ts +16 -1
- package/src/tools/computer-use/definitions.ts +123 -5
- package/src/tools/computer-use/skill-proxy-bridge.ts +2 -1
- package/src/tools/credentials/metadata-store.ts +100 -0
- package/src/tools/filesystem/search.ts +2 -1
- package/src/tools/host-filesystem/edit.test.ts +2 -1
- package/src/tools/host-filesystem/edit.ts +2 -2
- package/src/tools/host-filesystem/read.test.ts +2 -1
- package/src/tools/host-filesystem/read.ts +2 -2
- package/src/tools/host-filesystem/transfer.test.ts +2 -1
- package/src/tools/host-filesystem/transfer.ts +2 -2
- package/src/tools/host-filesystem/write.test.ts +2 -1
- package/src/tools/host-filesystem/write.ts +2 -2
- package/src/tools/host-terminal/host-shell.test.ts +16 -0
- package/src/tools/host-terminal/host-shell.ts +16 -26
- package/src/tools/network/web-search.ts +6 -1
- package/src/tools/policy-context.ts +6 -7
- package/src/tools/shared/filesystem/file-ops-service.ts +1 -31
- package/src/tools/shared/shell-output.test.ts +10 -0
- package/src/tools/shared/shell-output.ts +14 -2
- package/src/tools/skills/find-similar-skills.test.ts +214 -2
- package/src/tools/skills/find-similar-skills.ts +62 -9
- package/src/tools/skills/resolve-execute-invocation.ts +24 -0
- package/src/tools/skills/sandbox-runner.ts +13 -2
- package/src/tools/skills/scaffold-managed.ts +78 -16
- package/src/tools/subagent/spawn.ts +5 -8
- package/src/tools/terminal/__tests__/safe-env.test.ts +33 -4
- package/src/tools/terminal/__tests__/sanitized-bash.test.ts +43 -0
- package/src/tools/terminal/safe-env.ts +35 -24
- package/src/tools/terminal/sanitized-bash.ts +118 -0
- package/src/tools/terminal/shell-launch.test.ts +162 -0
- package/src/tools/terminal/shell.test.ts +29 -0
- package/src/tools/terminal/shell.ts +13 -7
- package/src/tools/ui-surface/channel-variants.ts +10 -59
- package/src/tools/watch/watch-retro-report.ts +0 -8
- package/src/types/host-capabilities.ts +5 -3
- package/src/util/__tests__/language-subtag.test.ts +23 -1
- package/src/util/abort-reasons.ts +12 -1
- package/src/util/ca-bundle.ts +13 -0
- package/src/util/host-process.test.ts +17 -1
- package/src/util/host-process.ts +24 -0
- package/src/util/language-subtag.ts +16 -0
- package/src/util/platform.ts +8 -0
- package/src/util/plugin-manifest.ts +203 -0
- package/src/util/unicode.ts +44 -1
- package/src/watch/watch-retro.ts +6 -8
- package/src/watch/watch-timeline.ts +2 -2
- package/src/workflows/run-manager.ts +2 -1
- package/src/workspace/git-porcelain.ts +66 -0
- package/src/workspace/git-service.ts +6 -55
- package/src/workspace/migrations/154-repair-retired-codex-gpt-5-4-model-ids.ts +258 -0
- package/src/workspace/migrations/155-move-front-model-config-to-voice.ts +78 -0
- package/src/workspace/migrations/156-extract-workspace-mcp-json.ts +214 -0
- package/src/workspace/migrations/__tests__/155-move-front-model-config-to-voice.test.ts +102 -0
- package/src/workspace/migrations/__tests__/156-extract-workspace-mcp-json.test.ts +199 -0
- package/src/workspace/migrations/registry.ts +6 -0
- package/src/__tests__/app-source-watcher.test.ts +0 -218
- package/src/__tests__/config-watcher-skill-reseed.test.ts +0 -359
- package/src/__tests__/speaker-identification.test.ts +0 -56
- package/src/calls/speaker-identification.ts +0 -233
- package/src/daemon/app-source-watcher.ts +0 -218
- package/src/messaging/providers/whatsapp/client.ts +0 -29
- package/src/runtime/routes/debug-bash-routes.ts +0 -173
- package/src/util/fs-watcher-error.ts +0 -36
- /package/src/{live-voice → calls}/progress-phrases.ts +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { mkdirSync, writeFileSync } from "node:fs";
|
|
2
2
|
import { join } from "node:path";
|
|
3
|
-
import { beforeAll, describe, expect, mock, test } from "bun:test";
|
|
3
|
+
import { afterAll, beforeAll, describe, expect, mock, test } from "bun:test";
|
|
4
4
|
|
|
5
5
|
import type { TurnDetectorConfig } from "../../calls/media-turn-detector.js";
|
|
6
6
|
import type {
|
|
@@ -12,12 +12,21 @@ import {
|
|
|
12
12
|
loadRawConfig,
|
|
13
13
|
saveRawConfig,
|
|
14
14
|
} from "../../config/loader.js";
|
|
15
|
-
import type {
|
|
15
|
+
import type { VoiceFrontModelConfig } from "../../config/schemas/voice.js";
|
|
16
16
|
import type {
|
|
17
17
|
StreamingTranscriber,
|
|
18
18
|
SttStreamServerEvent,
|
|
19
19
|
} from "../../stt/types.js";
|
|
20
|
-
import {
|
|
20
|
+
import {
|
|
21
|
+
computerUseKeyTool,
|
|
22
|
+
computerUseObserveTool,
|
|
23
|
+
} from "../../tools/computer-use/definitions.js";
|
|
24
|
+
import {
|
|
25
|
+
__resetRegistryForTesting,
|
|
26
|
+
registerSkillTools,
|
|
27
|
+
unregisterSkillTools,
|
|
28
|
+
} from "../../tools/registry.js";
|
|
29
|
+
import { finalizeTool } from "../../tools/tool-defaults.js";
|
|
21
30
|
import { getWorkspaceSkillsDir } from "../../util/platform.js";
|
|
22
31
|
import type { LiveVoiceContinuationLabeler } from "../continuation-label.js";
|
|
23
32
|
import type { LiveVoiceAudioArchiveResult } from "../live-voice-archive.js";
|
|
@@ -183,7 +192,7 @@ function createHarness(options: {
|
|
|
183
192
|
echoBargeInMargin?: number;
|
|
184
193
|
echoEmaHalfLifeMs?: number;
|
|
185
194
|
echoDrainSlackMs?: number;
|
|
186
|
-
frontModelConfig?: Partial<
|
|
195
|
+
frontModelConfig?: Partial<VoiceFrontModelConfig>;
|
|
187
196
|
emitMetrics?: boolean;
|
|
188
197
|
metricsClock?: () => number;
|
|
189
198
|
// Return a promise to hold a frame's transport write open (a backed-up
|
|
@@ -202,6 +211,9 @@ function createHarness(options: {
|
|
|
202
211
|
getTurnTeardown?: (conversationId: string) => Promise<void> | undefined;
|
|
203
212
|
detachTeardownSettleTimeoutMs?: number;
|
|
204
213
|
continuationAnnounceSilenceMs?: number;
|
|
214
|
+
foregroundTaskResumeSilenceMs?: number;
|
|
215
|
+
foregroundTaskMaxSuspendedMs?: number;
|
|
216
|
+
foregroundTaskMaxInterveningTurns?: number;
|
|
205
217
|
}) {
|
|
206
218
|
const sequencer = createLiveVoiceServerFrameSequencer();
|
|
207
219
|
const frames: LiveVoiceServerFrame[] = [];
|
|
@@ -290,6 +302,18 @@ function createHarness(options: {
|
|
|
290
302
|
continuationAnnounceSilenceMs: options.continuationAnnounceSilenceMs,
|
|
291
303
|
}
|
|
292
304
|
: {}),
|
|
305
|
+
...(options.foregroundTaskResumeSilenceMs !== undefined
|
|
306
|
+
? { foregroundTaskResumeSilenceMs: options.foregroundTaskResumeSilenceMs }
|
|
307
|
+
: {}),
|
|
308
|
+
...(options.foregroundTaskMaxSuspendedMs !== undefined
|
|
309
|
+
? { foregroundTaskMaxSuspendedMs: options.foregroundTaskMaxSuspendedMs }
|
|
310
|
+
: {}),
|
|
311
|
+
...(options.foregroundTaskMaxInterveningTurns !== undefined
|
|
312
|
+
? {
|
|
313
|
+
foregroundTaskMaxInterveningTurns:
|
|
314
|
+
options.foregroundTaskMaxInterveningTurns,
|
|
315
|
+
}
|
|
316
|
+
: {}),
|
|
293
317
|
};
|
|
294
318
|
const session = options.viaFactory
|
|
295
319
|
? createLiveVoiceSession(context, {
|
|
@@ -310,6 +334,26 @@ function countType(frames: LiveVoiceServerFrame[], type: string): number {
|
|
|
310
334
|
return frames.filter((frame) => frame.type === type).length;
|
|
311
335
|
}
|
|
312
336
|
|
|
337
|
+
function foregroundTaskStateOf(session: LiveVoiceSession): {
|
|
338
|
+
phase: string;
|
|
339
|
+
request?: string;
|
|
340
|
+
interveningTurns?: number;
|
|
341
|
+
resumePending?: boolean;
|
|
342
|
+
hostToolStarted?: boolean;
|
|
343
|
+
} | null {
|
|
344
|
+
return (
|
|
345
|
+
session as unknown as {
|
|
346
|
+
foregroundTaskState: {
|
|
347
|
+
phase: string;
|
|
348
|
+
request?: string;
|
|
349
|
+
interveningTurns?: number;
|
|
350
|
+
resumePending?: boolean;
|
|
351
|
+
hostToolStarted?: boolean;
|
|
352
|
+
} | null;
|
|
353
|
+
}
|
|
354
|
+
).foregroundTaskState;
|
|
355
|
+
}
|
|
356
|
+
|
|
313
357
|
function makeTtsChunk(text: string): LiveVoiceTtsAudioChunk {
|
|
314
358
|
return {
|
|
315
359
|
type: "tts_audio",
|
|
@@ -463,11 +507,88 @@ async function startForegroundWinsScenario(): Promise<{
|
|
|
463
507
|
};
|
|
464
508
|
}
|
|
465
509
|
|
|
510
|
+
async function startForegroundTaskBargeInScenario(options?: {
|
|
511
|
+
foregroundTaskMaxInterveningTurns?: number;
|
|
512
|
+
foregroundTaskMaxSuspendedMs?: number;
|
|
513
|
+
foregroundTaskResumeSilenceMs?: number;
|
|
514
|
+
skillExecuteInput?: Record<string, unknown>;
|
|
515
|
+
skillExecuteAllowedToolNames?: ReadonlySet<string>;
|
|
516
|
+
startHostTool?: boolean;
|
|
517
|
+
}): Promise<{
|
|
518
|
+
calls: VoiceTurnOptions[];
|
|
519
|
+
frames: LiveVoiceServerFrame[];
|
|
520
|
+
session: LiveVoiceSession;
|
|
521
|
+
spawnBackgroundContinuation: ReturnType<typeof mock>;
|
|
522
|
+
}> {
|
|
523
|
+
const calls: VoiceTurnOptions[] = [];
|
|
524
|
+
const startVoiceTurn = mock(async (turn: VoiceTurnOptions) => {
|
|
525
|
+
calls.push(turn);
|
|
526
|
+
return {
|
|
527
|
+
turnId: `bridge-turn-${calls.length}`,
|
|
528
|
+
abort: mock(),
|
|
529
|
+
discard: mock(async () => {}),
|
|
530
|
+
};
|
|
531
|
+
});
|
|
532
|
+
const spawnBackgroundContinuation = mock(async () => "");
|
|
533
|
+
const { frames, session } = createHarness({
|
|
534
|
+
finals: ["change the title", "what title is there now"],
|
|
535
|
+
startVoiceTurn,
|
|
536
|
+
streamTtsAudio: mock(async (tts: LiveVoiceTtsOptions) => {
|
|
537
|
+
tts.onAudioChunk(makeTtsChunk("assistant audio"));
|
|
538
|
+
return makeTtsResult("assistant audio");
|
|
539
|
+
}),
|
|
540
|
+
spawnBackgroundContinuation,
|
|
541
|
+
foregroundTaskResumeSilenceMs: options?.foregroundTaskResumeSilenceMs ?? 20,
|
|
542
|
+
...(options?.foregroundTaskMaxInterveningTurns !== undefined
|
|
543
|
+
? {
|
|
544
|
+
foregroundTaskMaxInterveningTurns:
|
|
545
|
+
options.foregroundTaskMaxInterveningTurns,
|
|
546
|
+
}
|
|
547
|
+
: {}),
|
|
548
|
+
...(options?.foregroundTaskMaxSuspendedMs !== undefined
|
|
549
|
+
? { foregroundTaskMaxSuspendedMs: options.foregroundTaskMaxSuspendedMs }
|
|
550
|
+
: {}),
|
|
551
|
+
});
|
|
552
|
+
|
|
553
|
+
await session.start();
|
|
554
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
555
|
+
await waitFor(() => calls.length === 1);
|
|
556
|
+
|
|
557
|
+
calls[0]?.callbacks?.assistant_text_delta?.(makeTextDelta("[1] One moment."));
|
|
558
|
+
calls[0]?.callbacks?.message_complete?.(makeMessageComplete());
|
|
559
|
+
await waitFor(() => calls.length === 2);
|
|
560
|
+
|
|
561
|
+
if (options?.startHostTool !== false) {
|
|
562
|
+
calls[1]?.callbacks?.tool_use_start?.("skill_execute", {
|
|
563
|
+
toolUseId: "tool-1",
|
|
564
|
+
input: options?.skillExecuteInput ?? {
|
|
565
|
+
tool: "computer_use_observe",
|
|
566
|
+
input: {},
|
|
567
|
+
},
|
|
568
|
+
...(options?.skillExecuteAllowedToolNames !== undefined
|
|
569
|
+
? { allowedToolNames: options.skillExecuteAllowedToolNames }
|
|
570
|
+
: {}),
|
|
571
|
+
});
|
|
572
|
+
}
|
|
573
|
+
await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
|
|
574
|
+
await waitFor(() => calls.length === 3);
|
|
575
|
+
|
|
576
|
+
return { calls, frames, session, spawnBackgroundContinuation };
|
|
577
|
+
}
|
|
578
|
+
|
|
466
579
|
describe("LiveVoiceSession server VAD", () => {
|
|
467
580
|
// The foreground-wins classification consults the tool registry's owner map
|
|
468
581
|
// (a tool is only provably read-only when it is the trusted built-in);
|
|
469
582
|
// register the core baseline so built-in names resolve like in the daemon.
|
|
470
|
-
beforeAll(() =>
|
|
583
|
+
beforeAll(() => {
|
|
584
|
+
__resetRegistryForTesting();
|
|
585
|
+
registerSkillTools("computer-use-test", [
|
|
586
|
+
finalizeTool(computerUseKeyTool),
|
|
587
|
+
finalizeTool(computerUseObserveTool),
|
|
588
|
+
finalizeTool({ ...computerUseObserveTool, name: "read_file" }),
|
|
589
|
+
]);
|
|
590
|
+
});
|
|
591
|
+
afterAll(() => unregisterSkillTools("computer-use-test"));
|
|
471
592
|
|
|
472
593
|
test("ready echoes turnDetection server_vad", async () => {
|
|
473
594
|
const { frames, session } = createHarness({});
|
|
@@ -580,6 +701,506 @@ describe("LiveVoiceSession server VAD", () => {
|
|
|
580
701
|
expect(countType(frames, "utterance_end")).toBe(2);
|
|
581
702
|
});
|
|
582
703
|
|
|
704
|
+
test("an escalated task interrupted before its first tool stays on the parent", async () => {
|
|
705
|
+
const { calls, session, spawnBackgroundContinuation } =
|
|
706
|
+
await startForegroundTaskBargeInScenario({ startHostTool: false });
|
|
707
|
+
|
|
708
|
+
expect(spawnBackgroundContinuation).not.toHaveBeenCalled();
|
|
709
|
+
expect(foregroundTaskStateOf(session)).toMatchObject({
|
|
710
|
+
phase: "suspended",
|
|
711
|
+
hostToolStarted: false,
|
|
712
|
+
});
|
|
713
|
+
|
|
714
|
+
const frontDoorAnswer = calls[2];
|
|
715
|
+
frontDoorAnswer?.callbacks?.assistant_text_delta?.(
|
|
716
|
+
makeTextDelta("The title is Project Theta."),
|
|
717
|
+
);
|
|
718
|
+
frontDoorAnswer?.callbacks?.message_complete?.(makeMessageComplete());
|
|
719
|
+
await waitFor(() => calls.length === 4);
|
|
720
|
+
|
|
721
|
+
const resume = calls[3];
|
|
722
|
+
expect(resume).toMatchObject({
|
|
723
|
+
routingLeg: "escalated",
|
|
724
|
+
directEscalated: true,
|
|
725
|
+
hiddenSyntheticPrompt: true,
|
|
726
|
+
});
|
|
727
|
+
expect(resume?.content).toContain("change the title");
|
|
728
|
+
resume?.callbacks?.tool_use_start?.("skill_execute", {
|
|
729
|
+
toolUseId: "tool-2",
|
|
730
|
+
input: { tool: "computer_use_observe", input: {} },
|
|
731
|
+
});
|
|
732
|
+
expect(foregroundTaskStateOf(session)).toMatchObject({
|
|
733
|
+
phase: "owned",
|
|
734
|
+
hostToolStarted: true,
|
|
735
|
+
});
|
|
736
|
+
|
|
737
|
+
resume?.callbacks?.assistant_text_delta?.(makeTextDelta("Done."));
|
|
738
|
+
resume?.callbacks?.message_complete?.(makeMessageComplete());
|
|
739
|
+
await waitFor(() => foregroundTaskStateOf(session) === null);
|
|
740
|
+
});
|
|
741
|
+
|
|
742
|
+
test("a new barged-in task becomes the resume anchor", async () => {
|
|
743
|
+
const calls: VoiceTurnOptions[] = [];
|
|
744
|
+
const startVoiceTurn = mock(async (turn: VoiceTurnOptions) => {
|
|
745
|
+
calls.push(turn);
|
|
746
|
+
return {
|
|
747
|
+
turnId: `bridge-turn-${calls.length}`,
|
|
748
|
+
abort: mock(),
|
|
749
|
+
discard: mock(async () => {}),
|
|
750
|
+
};
|
|
751
|
+
});
|
|
752
|
+
let releaseFirstCompletionTts: (() => void) | undefined;
|
|
753
|
+
const { session } = createHarness({
|
|
754
|
+
finals: ["change the title", "add three bullets", "what is the title"],
|
|
755
|
+
startVoiceTurn,
|
|
756
|
+
streamTtsAudio: mock(async (tts: LiveVoiceTtsOptions) => {
|
|
757
|
+
tts.onAudioChunk(makeTtsChunk("assistant audio"));
|
|
758
|
+
if (tts.text.includes("First task done")) {
|
|
759
|
+
await new Promise<void>((resolve) => {
|
|
760
|
+
releaseFirstCompletionTts = resolve;
|
|
761
|
+
});
|
|
762
|
+
}
|
|
763
|
+
return makeTtsResult("assistant audio");
|
|
764
|
+
}),
|
|
765
|
+
foregroundTaskResumeSilenceMs: 20,
|
|
766
|
+
});
|
|
767
|
+
|
|
768
|
+
await session.start();
|
|
769
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
770
|
+
await waitFor(() => calls.length === 1);
|
|
771
|
+
calls[0]?.callbacks?.assistant_text_delta?.(
|
|
772
|
+
makeTextDelta("[1] One moment."),
|
|
773
|
+
);
|
|
774
|
+
calls[0]?.callbacks?.message_complete?.(makeMessageComplete());
|
|
775
|
+
await waitFor(() => calls.length === 2);
|
|
776
|
+
calls[1]?.callbacks?.tool_use_start?.("skill_execute", {
|
|
777
|
+
toolUseId: "tool-1",
|
|
778
|
+
input: { tool: "computer_use_observe", input: {} },
|
|
779
|
+
});
|
|
780
|
+
calls[1]?.callbacks?.assistant_text_delta?.(
|
|
781
|
+
makeTextDelta("First task done."),
|
|
782
|
+
);
|
|
783
|
+
calls[1]?.callbacks?.message_complete?.(makeMessageComplete());
|
|
784
|
+
await waitFor(() => releaseFirstCompletionTts !== undefined);
|
|
785
|
+
|
|
786
|
+
await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
|
|
787
|
+
await waitFor(() => calls.length === 3);
|
|
788
|
+
await waitFor(() => foregroundTaskStateOf(session) === null);
|
|
789
|
+
releaseFirstCompletionTts?.();
|
|
790
|
+
calls[2]?.callbacks?.assistant_text_delta?.(
|
|
791
|
+
makeTextDelta("[1] One moment."),
|
|
792
|
+
);
|
|
793
|
+
calls[2]?.callbacks?.message_complete?.(makeMessageComplete());
|
|
794
|
+
await waitFor(() => calls.length === 4);
|
|
795
|
+
expect(foregroundTaskStateOf(session)).toMatchObject({
|
|
796
|
+
phase: "owned",
|
|
797
|
+
request: "add three bullets",
|
|
798
|
+
hostToolStarted: false,
|
|
799
|
+
});
|
|
800
|
+
|
|
801
|
+
await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
|
|
802
|
+
await waitFor(() => calls.length === 5);
|
|
803
|
+
calls[4]?.callbacks?.assistant_text_delta?.(
|
|
804
|
+
makeTextDelta("The title is Project Alpha."),
|
|
805
|
+
);
|
|
806
|
+
calls[4]?.callbacks?.message_complete?.(makeMessageComplete());
|
|
807
|
+
await waitFor(() => calls.length === 6);
|
|
808
|
+
|
|
809
|
+
const resume = calls[5];
|
|
810
|
+
expect(resume).toMatchObject({
|
|
811
|
+
routingLeg: "escalated",
|
|
812
|
+
directEscalated: true,
|
|
813
|
+
hiddenSyntheticPrompt: true,
|
|
814
|
+
});
|
|
815
|
+
expect(resume?.content).toContain("add three bullets");
|
|
816
|
+
expect(resume?.content).not.toContain("change the title");
|
|
817
|
+
resume?.callbacks?.assistant_text_delta?.(makeTextDelta("[TASK:STOP]"));
|
|
818
|
+
resume?.callbacks?.message_complete?.(makeMessageComplete());
|
|
819
|
+
await waitFor(() => foregroundTaskStateOf(session) === null);
|
|
820
|
+
});
|
|
821
|
+
|
|
822
|
+
test("an escalated task retries when its first tool-capable leg fails", async () => {
|
|
823
|
+
const calls: VoiceTurnOptions[] = [];
|
|
824
|
+
const startVoiceTurn = mock(async (turn: VoiceTurnOptions) => {
|
|
825
|
+
calls.push(turn);
|
|
826
|
+
return {
|
|
827
|
+
turnId: `bridge-turn-${calls.length}`,
|
|
828
|
+
abort: mock(),
|
|
829
|
+
discard: mock(async () => {}),
|
|
830
|
+
};
|
|
831
|
+
});
|
|
832
|
+
const { session } = createHarness({
|
|
833
|
+
finals: ["change the title"],
|
|
834
|
+
startVoiceTurn,
|
|
835
|
+
streamTtsAudio: mock(async (tts: LiveVoiceTtsOptions) => {
|
|
836
|
+
tts.onAudioChunk(makeTtsChunk("assistant audio"));
|
|
837
|
+
return makeTtsResult("assistant audio");
|
|
838
|
+
}),
|
|
839
|
+
foregroundTaskResumeSilenceMs: 20,
|
|
840
|
+
});
|
|
841
|
+
|
|
842
|
+
await session.start();
|
|
843
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
844
|
+
await waitFor(() => calls.length === 1);
|
|
845
|
+
calls[0]?.callbacks?.assistant_text_delta?.(
|
|
846
|
+
makeTextDelta("[1] One moment."),
|
|
847
|
+
);
|
|
848
|
+
calls[0]?.callbacks?.message_complete?.(makeMessageComplete());
|
|
849
|
+
await waitFor(() => calls.length === 2);
|
|
850
|
+
expect(foregroundTaskStateOf(session)).toMatchObject({
|
|
851
|
+
phase: "owned",
|
|
852
|
+
hostToolStarted: false,
|
|
853
|
+
});
|
|
854
|
+
|
|
855
|
+
calls[1]?.onError?.("strong leg failed");
|
|
856
|
+
await waitFor(() => calls.length === 3);
|
|
857
|
+
expect(calls[2]).toMatchObject({
|
|
858
|
+
routingLeg: "escalated",
|
|
859
|
+
directEscalated: true,
|
|
860
|
+
hiddenSyntheticPrompt: true,
|
|
861
|
+
});
|
|
862
|
+
calls[2]?.callbacks?.assistant_text_delta?.(makeTextDelta("[TASK:STOP]"));
|
|
863
|
+
calls[2]?.callbacks?.message_complete?.(makeMessageComplete());
|
|
864
|
+
await waitFor(() => foregroundTaskStateOf(session) === null);
|
|
865
|
+
});
|
|
866
|
+
|
|
867
|
+
test("a barged-in skill host tool stays on the parent and resumes after the front door answers", async () => {
|
|
868
|
+
const { calls, frames, session, spawnBackgroundContinuation } =
|
|
869
|
+
await startForegroundTaskBargeInScenario();
|
|
870
|
+
expect(spawnBackgroundContinuation).not.toHaveBeenCalled();
|
|
871
|
+
|
|
872
|
+
const frontDoorAnswer = calls[2];
|
|
873
|
+
expect(frontDoorAnswer?.routingLeg).toBe("front-door");
|
|
874
|
+
expect(frontDoorAnswer?.voiceControlPrompt).toContain("[TASK:STOP]");
|
|
875
|
+
expect(frontDoorAnswer?.voiceControlPrompt).toContain("change the title");
|
|
876
|
+
|
|
877
|
+
// A direct answer cannot accidentally end the task. Once its speech has
|
|
878
|
+
// drained, the task resumes on the strong, tool-capable leg.
|
|
879
|
+
frontDoorAnswer?.callbacks?.assistant_text_delta?.(
|
|
880
|
+
makeTextDelta("The title is Project Alpha."),
|
|
881
|
+
);
|
|
882
|
+
frontDoorAnswer?.callbacks?.message_complete?.(makeMessageComplete());
|
|
883
|
+
await waitFor(() => calls.length === 4);
|
|
884
|
+
|
|
885
|
+
const resume = calls[3];
|
|
886
|
+
expect(resume).toMatchObject({
|
|
887
|
+
routingLeg: "escalated",
|
|
888
|
+
directEscalated: true,
|
|
889
|
+
hiddenSyntheticPrompt: true,
|
|
890
|
+
});
|
|
891
|
+
expect(resume?.content).toContain("change the title");
|
|
892
|
+
expect(resume?.content).toContain("[TASK:STOP]");
|
|
893
|
+
expect(
|
|
894
|
+
frames.filter((frame) => frame.type === "thinking").length,
|
|
895
|
+
).toBeGreaterThanOrEqual(3);
|
|
896
|
+
|
|
897
|
+
// The strong leg can decline a stale resume silently and clear ownership.
|
|
898
|
+
const spokenBeforeStop = countType(frames, "assistant_text_delta");
|
|
899
|
+
resume?.callbacks?.assistant_text_delta?.(makeTextDelta("[TASK:STOP]"));
|
|
900
|
+
resume?.callbacks?.message_complete?.(makeMessageComplete());
|
|
901
|
+
await flushAsyncCallbacks();
|
|
902
|
+
expect(countType(frames, "assistant_text_delta")).toBe(spokenBeforeStop);
|
|
903
|
+
expect(foregroundTaskStateOf(session)).toBeNull();
|
|
904
|
+
});
|
|
905
|
+
|
|
906
|
+
test.each([
|
|
907
|
+
[
|
|
908
|
+
"a provider-wrapped inner tool",
|
|
909
|
+
{
|
|
910
|
+
_raw: JSON.stringify({
|
|
911
|
+
tool: "computer_use_observe",
|
|
912
|
+
input: {},
|
|
913
|
+
}),
|
|
914
|
+
},
|
|
915
|
+
],
|
|
916
|
+
[
|
|
917
|
+
"an aliased inner tool",
|
|
918
|
+
{
|
|
919
|
+
tool: "computer_use_press_key",
|
|
920
|
+
input: { key: "Enter", reasoning: "Continue the task" },
|
|
921
|
+
},
|
|
922
|
+
],
|
|
923
|
+
])("recognizes host ownership for %s", async (_label, skillExecuteInput) => {
|
|
924
|
+
const { session, spawnBackgroundContinuation } =
|
|
925
|
+
await startForegroundTaskBargeInScenario({ skillExecuteInput });
|
|
926
|
+
|
|
927
|
+
expect(spawnBackgroundContinuation).not.toHaveBeenCalled();
|
|
928
|
+
expect(foregroundTaskStateOf(session)).toMatchObject({
|
|
929
|
+
phase: "suspended",
|
|
930
|
+
});
|
|
931
|
+
await session.close("client_end");
|
|
932
|
+
});
|
|
933
|
+
|
|
934
|
+
test("preserves an allowed host tool whose name is also an alias", async () => {
|
|
935
|
+
const { session, spawnBackgroundContinuation } =
|
|
936
|
+
await startForegroundTaskBargeInScenario({
|
|
937
|
+
skillExecuteInput: { tool: "read_file", input: {} },
|
|
938
|
+
skillExecuteAllowedToolNames: new Set(["read_file"]),
|
|
939
|
+
});
|
|
940
|
+
|
|
941
|
+
expect(spawnBackgroundContinuation).not.toHaveBeenCalled();
|
|
942
|
+
expect(foregroundTaskStateOf(session)).toMatchObject({
|
|
943
|
+
phase: "suspended",
|
|
944
|
+
});
|
|
945
|
+
await session.close("client_end");
|
|
946
|
+
});
|
|
947
|
+
|
|
948
|
+
test("retries suspended host work when the handed-off leg fails to start", async () => {
|
|
949
|
+
const calls: VoiceTurnOptions[] = [];
|
|
950
|
+
const startVoiceTurn = mock(async (options: VoiceTurnOptions) => {
|
|
951
|
+
calls.push(options);
|
|
952
|
+
if (calls.length === 4) {
|
|
953
|
+
throw new Error("strong leg unavailable");
|
|
954
|
+
}
|
|
955
|
+
return {
|
|
956
|
+
turnId: `bridge-turn-${calls.length}`,
|
|
957
|
+
abort: mock(),
|
|
958
|
+
discard: mock(async () => {}),
|
|
959
|
+
};
|
|
960
|
+
});
|
|
961
|
+
const { session } = createHarness({
|
|
962
|
+
finals: ["change the title", "make it shorter"],
|
|
963
|
+
startVoiceTurn,
|
|
964
|
+
streamTtsAudio: mock(async (tts: LiveVoiceTtsOptions) => {
|
|
965
|
+
tts.onAudioChunk(makeTtsChunk("assistant audio"));
|
|
966
|
+
return makeTtsResult("assistant audio");
|
|
967
|
+
}),
|
|
968
|
+
foregroundTaskResumeSilenceMs: 20,
|
|
969
|
+
});
|
|
970
|
+
|
|
971
|
+
await session.start();
|
|
972
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
973
|
+
await waitFor(() => calls.length === 1);
|
|
974
|
+
calls[0]?.callbacks?.assistant_text_delta?.(
|
|
975
|
+
makeTextDelta("[1] One moment."),
|
|
976
|
+
);
|
|
977
|
+
calls[0]?.callbacks?.message_complete?.(makeMessageComplete());
|
|
978
|
+
await waitFor(() => calls.length === 2);
|
|
979
|
+
calls[1]?.callbacks?.tool_use_start?.("skill_execute", {
|
|
980
|
+
toolUseId: "tool-1",
|
|
981
|
+
input: { tool: "computer_use_observe", input: {} },
|
|
982
|
+
});
|
|
983
|
+
|
|
984
|
+
await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
|
|
985
|
+
await waitFor(() => calls.length === 3);
|
|
986
|
+
calls[2]?.callbacks?.assistant_text_delta?.(
|
|
987
|
+
makeTextDelta("[1] I will continue."),
|
|
988
|
+
);
|
|
989
|
+
calls[2]?.callbacks?.message_complete?.(makeMessageComplete());
|
|
990
|
+
await waitFor(() => calls.length === 5);
|
|
991
|
+
|
|
992
|
+
expect(calls[3]).toMatchObject({ routingLeg: "escalated" });
|
|
993
|
+
expect(calls[4]).toMatchObject({
|
|
994
|
+
routingLeg: "escalated",
|
|
995
|
+
directEscalated: true,
|
|
996
|
+
hiddenSyntheticPrompt: true,
|
|
997
|
+
});
|
|
998
|
+
calls[4]?.callbacks?.assistant_text_delta?.(makeTextDelta("[TASK:STOP]"));
|
|
999
|
+
calls[4]?.callbacks?.message_complete?.(makeMessageComplete());
|
|
1000
|
+
await waitFor(() => foregroundTaskStateOf(session) === null);
|
|
1001
|
+
});
|
|
1002
|
+
|
|
1003
|
+
test("retries suspended host work when the handed-off leg fails asynchronously", async () => {
|
|
1004
|
+
const { calls, session } = await startForegroundTaskBargeInScenario();
|
|
1005
|
+
|
|
1006
|
+
calls[2]?.callbacks?.assistant_text_delta?.(
|
|
1007
|
+
makeTextDelta("[1] I will continue."),
|
|
1008
|
+
);
|
|
1009
|
+
calls[2]?.callbacks?.message_complete?.(makeMessageComplete());
|
|
1010
|
+
await waitFor(() => calls.length === 4);
|
|
1011
|
+
await waitFor(() => foregroundTaskStateOf(session)?.phase === "owned");
|
|
1012
|
+
|
|
1013
|
+
calls[3]?.onError?.("strong leg failed");
|
|
1014
|
+
await waitFor(() => calls.length === 5);
|
|
1015
|
+
|
|
1016
|
+
expect(calls[4]).toMatchObject({
|
|
1017
|
+
routingLeg: "escalated",
|
|
1018
|
+
directEscalated: true,
|
|
1019
|
+
hiddenSyntheticPrompt: true,
|
|
1020
|
+
});
|
|
1021
|
+
calls[4]?.callbacks?.assistant_text_delta?.(makeTextDelta("[TASK:STOP]"));
|
|
1022
|
+
calls[4]?.callbacks?.message_complete?.(makeMessageComplete());
|
|
1023
|
+
await waitFor(() => foregroundTaskStateOf(session) === null);
|
|
1024
|
+
});
|
|
1025
|
+
|
|
1026
|
+
test("a front-door task-stop answer abandons the suspended foreground task", async () => {
|
|
1027
|
+
const { calls, frames, session } =
|
|
1028
|
+
await startForegroundTaskBargeInScenario();
|
|
1029
|
+
|
|
1030
|
+
const frontDoorAnswer = calls[2];
|
|
1031
|
+
frontDoorAnswer?.callbacks?.assistant_text_delta?.(
|
|
1032
|
+
makeTextDelta("Okay, stopping. [TASK:STOP]"),
|
|
1033
|
+
);
|
|
1034
|
+
frontDoorAnswer?.callbacks?.message_complete?.(makeMessageComplete());
|
|
1035
|
+
await waitFor(() => frames.some((frame) => frame.type === "tts_done"));
|
|
1036
|
+
await new Promise((resolve) => setTimeout(resolve, 80));
|
|
1037
|
+
|
|
1038
|
+
expect(calls).toHaveLength(3);
|
|
1039
|
+
expect(foregroundTaskStateOf(session)).toBeNull();
|
|
1040
|
+
expect(
|
|
1041
|
+
frames.filter((frame) => frame.type === "assistant_text_delta").at(-1),
|
|
1042
|
+
).toMatchObject({ text: "Okay, stopping. " });
|
|
1043
|
+
});
|
|
1044
|
+
|
|
1045
|
+
test("barging into completed host-task speech does not revive the task", async () => {
|
|
1046
|
+
const calls: VoiceTurnOptions[] = [];
|
|
1047
|
+
const startVoiceTurn = mock(async (options: VoiceTurnOptions) => {
|
|
1048
|
+
calls.push(options);
|
|
1049
|
+
return { turnId: `bridge-turn-${calls.length}`, abort: mock() };
|
|
1050
|
+
});
|
|
1051
|
+
let releaseDoneTts: (() => void) | undefined;
|
|
1052
|
+
const streamTtsAudio = mock(async (options: LiveVoiceTtsOptions) => {
|
|
1053
|
+
options.onAudioChunk(makeTtsChunk("assistant audio"));
|
|
1054
|
+
if (options.text.includes("Done")) {
|
|
1055
|
+
await new Promise<void>((resolve) => {
|
|
1056
|
+
releaseDoneTts = resolve;
|
|
1057
|
+
});
|
|
1058
|
+
}
|
|
1059
|
+
return makeTtsResult("assistant audio");
|
|
1060
|
+
});
|
|
1061
|
+
const { session } = createHarness({
|
|
1062
|
+
finals: ["change the title", "one more thing"],
|
|
1063
|
+
startVoiceTurn,
|
|
1064
|
+
streamTtsAudio,
|
|
1065
|
+
foregroundTaskResumeSilenceMs: 20,
|
|
1066
|
+
});
|
|
1067
|
+
|
|
1068
|
+
await session.start();
|
|
1069
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
1070
|
+
await waitFor(() => calls.length === 1);
|
|
1071
|
+
calls[0]?.callbacks?.assistant_text_delta?.(
|
|
1072
|
+
makeTextDelta("[1] One moment."),
|
|
1073
|
+
);
|
|
1074
|
+
calls[0]?.callbacks?.message_complete?.(makeMessageComplete());
|
|
1075
|
+
await waitFor(() => calls.length === 2);
|
|
1076
|
+
calls[1]?.callbacks?.tool_use_start?.("skill_execute", {
|
|
1077
|
+
toolUseId: "tool-1",
|
|
1078
|
+
input: { tool: "computer_use_observe", input: {} },
|
|
1079
|
+
});
|
|
1080
|
+
calls[1]?.callbacks?.assistant_text_delta?.(makeTextDelta("Done."));
|
|
1081
|
+
calls[1]?.callbacks?.message_complete?.(makeMessageComplete());
|
|
1082
|
+
await waitFor(() => releaseDoneTts !== undefined);
|
|
1083
|
+
|
|
1084
|
+
await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
|
|
1085
|
+
await waitFor(() => calls.length === 3);
|
|
1086
|
+
await waitFor(() => foregroundTaskStateOf(session) === null);
|
|
1087
|
+
calls[2]?.callbacks?.assistant_text_delta?.(makeTextDelta("Sure."));
|
|
1088
|
+
calls[2]?.callbacks?.message_complete?.(makeMessageComplete());
|
|
1089
|
+
releaseDoneTts?.();
|
|
1090
|
+
await new Promise((resolve) => setTimeout(resolve, 80));
|
|
1091
|
+
|
|
1092
|
+
expect(calls).toHaveLength(3);
|
|
1093
|
+
});
|
|
1094
|
+
|
|
1095
|
+
test("too many intervening answers abandon a suspended foreground task", async () => {
|
|
1096
|
+
const { calls, session } = await startForegroundTaskBargeInScenario({
|
|
1097
|
+
foregroundTaskMaxInterveningTurns: 1,
|
|
1098
|
+
foregroundTaskResumeSilenceMs: 60_000,
|
|
1099
|
+
});
|
|
1100
|
+
|
|
1101
|
+
calls[2]?.callbacks?.assistant_text_delta?.(makeTextDelta("First answer."));
|
|
1102
|
+
calls[2]?.callbacks?.message_complete?.(makeMessageComplete());
|
|
1103
|
+
await waitFor(() => foregroundTaskStateOf(session)?.interveningTurns === 1);
|
|
1104
|
+
|
|
1105
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
1106
|
+
await waitFor(() => calls.length === 4);
|
|
1107
|
+
calls[3]?.callbacks?.assistant_text_delta?.(
|
|
1108
|
+
makeTextDelta("Second answer."),
|
|
1109
|
+
);
|
|
1110
|
+
calls[3]?.callbacks?.message_complete?.(makeMessageComplete());
|
|
1111
|
+
await waitFor(() => foregroundTaskStateOf(session) === null);
|
|
1112
|
+
|
|
1113
|
+
expect(calls).toHaveLength(4);
|
|
1114
|
+
await session.close("client_end");
|
|
1115
|
+
});
|
|
1116
|
+
|
|
1117
|
+
test("an old suspended foreground task is not revived", async () => {
|
|
1118
|
+
const { calls, session } = await startForegroundTaskBargeInScenario({
|
|
1119
|
+
foregroundTaskMaxSuspendedMs: 1,
|
|
1120
|
+
foregroundTaskResumeSilenceMs: 20,
|
|
1121
|
+
});
|
|
1122
|
+
await new Promise((resolve) => setTimeout(resolve, 5));
|
|
1123
|
+
|
|
1124
|
+
calls[2]?.callbacks?.assistant_text_delta?.(makeTextDelta("Answer."));
|
|
1125
|
+
calls[2]?.callbacks?.message_complete?.(makeMessageComplete());
|
|
1126
|
+
await waitFor(() => foregroundTaskStateOf(session) === null);
|
|
1127
|
+
await new Promise((resolve) => setTimeout(resolve, 60));
|
|
1128
|
+
|
|
1129
|
+
expect(calls).toHaveLength(3);
|
|
1130
|
+
});
|
|
1131
|
+
|
|
1132
|
+
test.each(["hold_verdict", "speech_resumed"] as const)(
|
|
1133
|
+
"a discarded speculative front-door turn leaves host ownership suspended: %s",
|
|
1134
|
+
async (discardReason) => {
|
|
1135
|
+
const calls: VoiceTurnOptions[] = [];
|
|
1136
|
+
const discards: Array<ReturnType<typeof mock>> = [];
|
|
1137
|
+
const startVoiceTurn = mock(async (options: VoiceTurnOptions) => {
|
|
1138
|
+
calls.push(options);
|
|
1139
|
+
const discard = mock(async () => {});
|
|
1140
|
+
discards.push(discard);
|
|
1141
|
+
if (
|
|
1142
|
+
discardReason === "hold_verdict" &&
|
|
1143
|
+
options.unifiedVerdict === true
|
|
1144
|
+
) {
|
|
1145
|
+
setTimeout(() => {
|
|
1146
|
+
options.callbacks?.assistant_text_delta?.(makeTextDelta("[0]"));
|
|
1147
|
+
options.callbacks?.message_complete?.(makeMessageComplete());
|
|
1148
|
+
}, 0);
|
|
1149
|
+
}
|
|
1150
|
+
return {
|
|
1151
|
+
turnId: `bridge-turn-${calls.length}`,
|
|
1152
|
+
abort: mock(),
|
|
1153
|
+
discard,
|
|
1154
|
+
};
|
|
1155
|
+
});
|
|
1156
|
+
const spawnBackgroundContinuation = mock(async () => "");
|
|
1157
|
+
const { session, transcribers } = createHarness({
|
|
1158
|
+
finals: ["change the title", "actually just change it"],
|
|
1159
|
+
startVoiceTurn,
|
|
1160
|
+
streamTtsAudio: mock(async (tts: LiveVoiceTtsOptions) => {
|
|
1161
|
+
tts.onAudioChunk(makeTtsChunk("assistant audio"));
|
|
1162
|
+
return makeTtsResult("assistant audio");
|
|
1163
|
+
}),
|
|
1164
|
+
spawnBackgroundContinuation,
|
|
1165
|
+
holdStopEventsFor: [1],
|
|
1166
|
+
turnDetectorConfig: { silenceThresholdMs: 120 },
|
|
1167
|
+
foregroundTaskResumeSilenceMs: 60_000,
|
|
1168
|
+
});
|
|
1169
|
+
|
|
1170
|
+
await session.start();
|
|
1171
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
1172
|
+
await waitFor(() => calls.length === 1);
|
|
1173
|
+
calls[0]?.callbacks?.assistant_text_delta?.(
|
|
1174
|
+
makeTextDelta("[1] One moment."),
|
|
1175
|
+
);
|
|
1176
|
+
calls[0]?.callbacks?.message_complete?.(makeMessageComplete());
|
|
1177
|
+
await waitFor(() => calls.length === 2);
|
|
1178
|
+
calls[1]?.callbacks?.tool_use_start?.("skill_execute", {
|
|
1179
|
+
toolUseId: "tool-1",
|
|
1180
|
+
input: { tool: "computer_use_observe", input: {} },
|
|
1181
|
+
});
|
|
1182
|
+
|
|
1183
|
+
await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
|
|
1184
|
+
await waitFor(() => transcribers.length === 2);
|
|
1185
|
+
transcribers[1]?.emit({ type: "partial", text: "actually" });
|
|
1186
|
+
await waitFor(() => calls.length === 3);
|
|
1187
|
+
expect(calls[2]?.unifiedVerdict).toBe(true);
|
|
1188
|
+
|
|
1189
|
+
if (discardReason === "speech_resumed") {
|
|
1190
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
1191
|
+
}
|
|
1192
|
+
await waitFor(() => (discards[2]?.mock.calls.length ?? 0) === 1);
|
|
1193
|
+
|
|
1194
|
+
expect(spawnBackgroundContinuation).not.toHaveBeenCalled();
|
|
1195
|
+
expect(foregroundTaskStateOf(session)).toMatchObject({
|
|
1196
|
+
phase: "suspended",
|
|
1197
|
+
interveningTurns: 0,
|
|
1198
|
+
resumePending: false,
|
|
1199
|
+
});
|
|
1200
|
+
await session.close("client_end");
|
|
1201
|
+
},
|
|
1202
|
+
);
|
|
1203
|
+
|
|
583
1204
|
test("speech while the first tts_audio send is stuck in the queue does not cancel; barge-in works once it lands", async () => {
|
|
584
1205
|
let callbacks: VoiceTurnCallbacks | undefined;
|
|
585
1206
|
const abort = mock();
|
|
@@ -1873,6 +2494,40 @@ describe("LiveVoiceSession server VAD", () => {
|
|
|
1873
2494
|
expect(announcement?.voiceControlPrompt).toContain("first question");
|
|
1874
2495
|
});
|
|
1875
2496
|
|
|
2497
|
+
// Deepgram Flux sends interim updates through silence, each an empty
|
|
2498
|
+
// partial. A call nobody is talking on is still idle.
|
|
2499
|
+
test("an empty partial on an idle call does not hold the announcement back", async () => {
|
|
2500
|
+
const continuation = makeControlledContinuation();
|
|
2501
|
+
const { startVoiceTurn, calls } = makeResurfaceTurnStarter();
|
|
2502
|
+
const { frames, session, transcribers } = createHarness({
|
|
2503
|
+
finals: ["first question", ""],
|
|
2504
|
+
startVoiceTurn,
|
|
2505
|
+
streamTtsAudio: makeImmediateTts(),
|
|
2506
|
+
spawnBackgroundContinuation: continuation.spawnBackgroundContinuation,
|
|
2507
|
+
continuationAnnounceSilenceMs: 20,
|
|
2508
|
+
});
|
|
2509
|
+
|
|
2510
|
+
await session.start();
|
|
2511
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
2512
|
+
await waitFor(() => frames.some((frame) => frame.type === "thinking"));
|
|
2513
|
+
await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
|
|
2514
|
+
await waitFor(
|
|
2515
|
+
() => continuation.spawnBackgroundContinuation.mock.calls.length === 1,
|
|
2516
|
+
);
|
|
2517
|
+
await waitFor(() =>
|
|
2518
|
+
frames.some((frame) => frame.type === "utterance_discarded"),
|
|
2519
|
+
);
|
|
2520
|
+
await waitFor(() => transcribers.some((t) => !t.stopped));
|
|
2521
|
+
for (const transcriber of transcribers) {
|
|
2522
|
+
if (!transcriber.stopped) {
|
|
2523
|
+
transcriber.emit({ type: "partial", text: "" });
|
|
2524
|
+
}
|
|
2525
|
+
}
|
|
2526
|
+
|
|
2527
|
+
continuation.finish("THE_RESULT");
|
|
2528
|
+
await waitFor(() => announcementOf(calls) !== undefined);
|
|
2529
|
+
});
|
|
2530
|
+
|
|
1876
2531
|
test("an announcement persists hidden and is never delivered twice", async () => {
|
|
1877
2532
|
const continuation = makeControlledContinuation();
|
|
1878
2533
|
const { startVoiceTurn, calls } = makeResurfaceTurnStarter();
|