@vellumai/assistant 0.6.2 → 0.6.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +273 -10
- package/Dockerfile +2 -3
- package/bun.lock +41 -49
- package/bunfig.toml +3 -0
- package/docs/architecture/memory.md +1 -1
- package/docs/backup-troubleshooting.md +52 -0
- package/docs/browser-use-architecture-phase2.md +174 -0
- package/docs/stt-provider-onboarding.md +120 -0
- package/knip.json +12 -2
- package/node_modules/@vellumai/ces-contracts/bun.lock +8 -6
- package/node_modules/@vellumai/ces-contracts/package.json +3 -3
- package/node_modules/@vellumai/ces-contracts/src/rpc.ts +42 -0
- package/openapi.yaml +1111 -86
- package/package.json +40 -42
- package/scripts/generate-openapi.ts +0 -2
- package/scripts/test.sh +73 -18
- package/src/__tests__/acp-session.test.ts +43 -0
- package/src/__tests__/agent-image-optimize.test.ts +28 -0
- package/src/__tests__/agent-loop.test.ts +123 -0
- package/src/__tests__/anthropic-provider.test.ts +263 -10
- package/src/__tests__/app-builder-tool-scripts.test.ts +1 -0
- package/src/__tests__/app-executors.test.ts +1 -0
- package/src/__tests__/app-source-watcher.test.ts +37 -11
- package/src/__tests__/approval-routes-http.test.ts +178 -1
- package/src/__tests__/auto-analysis-end-to-end.test.ts +550 -0
- package/src/__tests__/auto-analysis-prompt.test.ts +50 -0
- package/src/__tests__/browser-fill-credential.test.ts +240 -94
- package/src/__tests__/browser-manager.test.ts +40 -27
- package/src/__tests__/browser-skill-baseline-tool-payload.test.ts +2 -2
- package/src/__tests__/browser-skill-endstate.test.ts +31 -7
- package/src/__tests__/btw-routes.test.ts +7 -0
- package/src/__tests__/call-controller.test.ts +581 -20
- package/src/__tests__/catalog-files.test.ts +1000 -0
- package/src/__tests__/channel-approvals.test.ts +53 -0
- package/src/__tests__/channel-invite-transport.test.ts +2 -2
- package/src/__tests__/channel-readiness-routes.test.ts +16 -20
- package/src/__tests__/channel-readiness-service.test.ts +12 -7
- package/src/__tests__/checker.test.ts +157 -10
- package/src/__tests__/clawhub-files.test.ts +347 -0
- package/src/__tests__/commit-message-enrichment-service.test.ts +36 -19
- package/src/__tests__/config-analysis.test.ts +100 -0
- package/src/__tests__/config-managed-gemini-defaults.test.ts +326 -0
- package/src/__tests__/config-schema-cmd.test.ts +2 -2
- package/src/__tests__/config-schema.test.ts +1248 -224
- package/src/__tests__/config-watcher-cleanup-throttle.test.ts +339 -0
- package/src/__tests__/config-watcher.test.ts +43 -8
- package/src/__tests__/confirmation-request-guardian-bridge.test.ts +23 -0
- package/src/__tests__/contact-store-user-file.test.ts +512 -0
- package/src/__tests__/contacts-write.test.ts +197 -0
- package/src/__tests__/context-overflow-approval.test.ts +16 -1
- package/src/__tests__/context-window-manager.test.ts +88 -0
- package/src/__tests__/conversation-abort-tool-results.test.ts +2 -0
- package/src/__tests__/conversation-agent-loop-overflow.test.ts +2 -1
- package/src/__tests__/conversation-agent-loop.test.ts +99 -3
- package/src/__tests__/conversation-analysis-routes.test.ts +2 -2
- package/src/__tests__/conversation-attachments.test.ts +80 -4
- package/src/__tests__/conversation-confirmation-signals.test.ts +290 -0
- package/src/__tests__/conversation-error.test.ts +70 -0
- package/src/__tests__/conversation-fork-crud.test.ts +17 -0
- package/src/__tests__/conversation-history-web-search.test.ts +12 -4
- package/src/__tests__/conversation-host-access-routes.test.ts +229 -0
- package/src/__tests__/conversation-init.benchmark.test.ts +6 -1
- package/src/__tests__/conversation-inject-context.test.ts +103 -0
- package/src/__tests__/conversation-launcher-skill-regression.test.ts +51 -0
- package/src/__tests__/conversation-list-source.test.ts +145 -0
- package/src/__tests__/conversation-pre-run-repair.test.ts +2 -0
- package/src/__tests__/conversation-provider-retry-repair.test.ts +2 -0
- package/src/__tests__/conversation-queue.test.ts +946 -62
- package/src/__tests__/conversation-routes-disk-view.test.ts +275 -0
- package/src/__tests__/conversation-routes-guardian-reply.test.ts +16 -0
- package/src/__tests__/conversation-routes-slash-commands.test.ts +1 -0
- package/src/__tests__/conversation-runtime-assembly.test.ts +324 -46
- package/src/__tests__/conversation-skill-tools.test.ts +7 -4
- package/src/__tests__/conversation-slash-commands.test.ts +33 -0
- package/src/__tests__/conversation-slash-queue.test.ts +89 -18
- package/src/__tests__/conversation-slash-unknown.test.ts +2 -0
- package/src/__tests__/conversation-starter-routes.test.ts +126 -0
- package/src/__tests__/conversation-starters-cadence.test.ts +161 -0
- package/src/__tests__/conversation-store.test.ts +195 -0
- package/src/__tests__/conversation-tool-setup-batch-authorized.test.ts +226 -0
- package/src/__tests__/conversation-workspace-cache-state.test.ts +193 -0
- package/src/__tests__/conversation-workspace-injection.test.ts +2 -0
- package/src/__tests__/conversation-workspace-tool-tracking.test.ts +2 -0
- package/src/__tests__/credential-execution-approval-bridge.test.ts +32 -1
- package/src/__tests__/credential-health-service.test.ts +352 -0
- package/src/__tests__/credential-security-invariants.test.ts +6 -3
- package/src/__tests__/credential-vault-unit.test.ts +383 -7
- package/src/__tests__/credential-vault.test.ts +152 -13
- package/src/__tests__/credentials-cli.test.ts +42 -18
- package/src/__tests__/cross-provider-web-search.test.ts +146 -35
- package/src/__tests__/date-context.test.ts +4 -4
- package/src/__tests__/deterministic-verification-control-plane.test.ts +10 -1
- package/src/__tests__/device-id.test.ts +112 -0
- package/src/__tests__/docker-signing-key-bootstrap.test.ts +167 -4
- package/src/__tests__/dynamic-skill-workflow-prompt.test.ts +1 -3
- package/src/__tests__/email-html-renderer.test.ts +71 -0
- package/src/__tests__/email-invite-adapter.test.ts +36 -32
- package/src/__tests__/embedding-managed-proxy-selection.test.ts +256 -0
- package/src/__tests__/emit-event-signal.test.ts +71 -0
- package/src/__tests__/extension-id-sync-guard.test.ts +222 -0
- package/src/__tests__/fixtures/mock-chrome-extension.ts +386 -0
- package/src/__tests__/gateway-only-enforcement.test.ts +206 -1
- package/src/__tests__/gateway-only-guard.test.ts +2 -0
- package/src/__tests__/gemini-provider.test.ts +66 -2
- package/src/__tests__/get-skill-detail-audit.test.ts +325 -0
- package/src/__tests__/gmail-archive-fallback.test.ts +193 -0
- package/src/__tests__/gmail-archive-gate.test.ts +246 -0
- package/src/__tests__/gmail-preferences.test.ts +117 -0
- package/src/__tests__/guardian-routing-invariants.test.ts +70 -2
- package/src/__tests__/headless-browser-interactions.test.ts +738 -359
- package/src/__tests__/headless-browser-mode.test.ts +614 -0
- package/src/__tests__/headless-browser-navigate.test.ts +528 -49
- package/src/__tests__/headless-browser-read-tools.test.ts +274 -100
- package/src/__tests__/headless-browser-snapshot.test.ts +250 -77
- package/src/__tests__/heartbeat-service.test.ts +70 -17
- package/src/__tests__/home-state-routes.test.ts +162 -0
- package/src/__tests__/host-bash-proxy.test.ts +145 -1
- package/src/__tests__/host-browser-e2e-cloud.test.ts +596 -0
- package/src/__tests__/host-browser-e2e-self-hosted-capability.test.ts +286 -0
- package/src/__tests__/host-browser-e2e-self-hosted.test.ts +374 -0
- package/src/__tests__/host-browser-event-routes.test.ts +350 -0
- package/src/__tests__/host-browser-proxy.test.ts +444 -0
- package/src/__tests__/host-browser-routes.test.ts +198 -0
- package/src/__tests__/host-browser-ws-events-e2e.test.ts +423 -0
- package/src/__tests__/host-cu-proxy.test.ts +166 -1
- package/src/__tests__/host-file-proxy.test.ts +185 -1
- package/src/__tests__/host-file-read-tool.test.ts +52 -0
- package/src/__tests__/host-proxy-interface.test.ts +165 -0
- package/src/__tests__/host-shell-tool.test.ts +1 -11
- package/src/__tests__/http-user-message-parity.test.ts +1 -0
- package/src/__tests__/identity-intro-cache.test.ts +40 -10
- package/src/__tests__/init-feature-flag-overrides.test.ts +38 -112
- package/src/__tests__/integration-status.test.ts +6 -7
- package/src/__tests__/jobs-store-upsert-debounced.test.ts +141 -0
- package/src/__tests__/list-messages-tool-merge.test.ts +37 -12
- package/src/__tests__/llm-context-normalization.test.ts +488 -0
- package/src/__tests__/llm-context-route-provider.test.ts +86 -5
- package/src/__tests__/llm-usage-store.test.ts +363 -0
- package/src/__tests__/mcp-client-auth.test.ts +40 -4
- package/src/__tests__/mcp-health-check.test.ts +10 -3
- package/src/__tests__/media-stream-output.test.ts +555 -0
- package/src/__tests__/media-stream-parser.test.ts +374 -0
- package/src/__tests__/media-stream-server-integration.test.ts +1234 -0
- package/src/__tests__/media-stream-stt-session.test.ts +588 -0
- package/src/__tests__/media-turn-detector.test.ts +440 -0
- package/src/__tests__/message-queue.test.ts +125 -0
- package/src/__tests__/migration-cross-version-compatibility.test.ts +3 -1
- package/src/__tests__/migration-export-http.test.ts +67 -8
- package/src/__tests__/migration-export-streaming.test.ts +66 -0
- package/src/__tests__/migration-import-commit-http.test.ts +109 -7
- package/src/__tests__/migration-import-preflight-http.test.ts +6 -5
- package/src/__tests__/migration-validate-http.test.ts +3 -3
- package/src/__tests__/mock-gateway-ipc.ts +151 -0
- package/src/__tests__/model-intents.test.ts +2 -2
- package/src/__tests__/native-host-marker-sync-guard.test.ts +157 -0
- package/src/__tests__/oauth-apps-routes.test.ts +18 -12
- package/src/__tests__/oauth-cli.test.ts +709 -60
- package/src/__tests__/oauth-connect-orchestrator.test.ts +118 -24
- package/src/__tests__/oauth-provider-seed-logos.test.ts +23 -0
- package/src/__tests__/oauth-provider-serializer.test.ts +147 -10
- package/src/__tests__/oauth-provider-visibility.test.ts +19 -21
- package/src/__tests__/oauth-providers-routes.test.ts +52 -14
- package/src/__tests__/oauth-store.test.ts +1465 -176
- package/src/__tests__/oauth2-gateway-transport.test.ts +460 -26
- package/src/__tests__/onboarding-template-contract.test.ts +81 -70
- package/src/__tests__/openai-provider.test.ts +178 -2
- package/src/__tests__/openai-responses-cutover-guard.test.ts +184 -0
- package/src/__tests__/openai-responses-provider.test.ts +1105 -0
- package/src/__tests__/openrouter-token-estimation.test.ts +100 -0
- package/src/__tests__/outlook-categories.test.ts +1 -1
- package/src/__tests__/outlook-client-automation.test.ts +1 -1
- package/src/__tests__/outlook-compose-tools.test.ts +1 -1
- package/src/__tests__/outlook-email-watcher.test.ts +1 -1
- package/src/__tests__/outlook-follow-up.test.ts +1 -1
- package/src/__tests__/outlook-messaging-provider.test.ts +2 -2
- package/src/__tests__/outlook-trash.test.ts +1 -1
- package/src/__tests__/outlook-unsubscribe.test.ts +32 -3
- package/src/__tests__/permission-checker-host-gate.test.ts +74 -14
- package/src/__tests__/permission-mode.test.ts +28 -56
- package/src/__tests__/persona-resolver.test.ts +251 -0
- package/src/__tests__/platform-bash-auto-approve.test.ts +4 -0
- package/src/__tests__/platform-callback-registration.test.ts +19 -0
- package/src/__tests__/platform.test.ts +92 -1
- package/src/__tests__/post-turn-tool-result-truncation.test.ts +343 -0
- package/src/__tests__/prechat-onboarding-contract.test.ts +267 -0
- package/src/__tests__/pricing.test.ts +174 -0
- package/src/__tests__/proxy-approval-callback.test.ts +18 -0
- package/src/__tests__/qdrant-manager.test.ts +29 -8
- package/src/__tests__/regenerate-fire-and-forget-trace.test.ts +194 -0
- package/src/__tests__/relationship-state-contract.test.ts +175 -0
- package/src/__tests__/relay-server.test.ts +423 -5
- package/src/__tests__/require-fresh-approval.test.ts +40 -1
- package/src/__tests__/sanitize-config-for-transfer.test.ts +132 -0
- package/src/__tests__/schedule-routes.test.ts +162 -0
- package/src/__tests__/search-skills-unified.test.ts +118 -0
- package/src/__tests__/secret-detection-handler.test.ts +84 -0
- package/src/__tests__/secret-ingress-http.test.ts +1 -0
- package/src/__tests__/secret-scanner-executor.test.ts +4 -0
- package/src/__tests__/secure-keys.test.ts +107 -0
- package/src/__tests__/send-endpoint-busy.test.ts +8 -1
- package/src/__tests__/sequence-store.test.ts +1 -1
- package/src/__tests__/server-history-render.test.ts +49 -0
- package/src/__tests__/set-permission-mode.test.ts +13 -250
- package/src/__tests__/settings-routes.test.ts +201 -0
- package/src/__tests__/skill-load-feature-flag.test.ts +1 -0
- package/src/__tests__/skills-file-content-endpoint.test.ts +801 -0
- package/src/__tests__/skills-files-catalog-fallback.test.ts +738 -0
- package/src/__tests__/skills.test.ts +5 -2
- package/src/__tests__/skillssh-files.test.ts +446 -0
- package/src/__tests__/slack-block-formatting.test.ts +110 -0
- package/src/__tests__/slack-channel-config.test.ts +576 -16
- package/src/__tests__/stt-catalog-parity.test.ts +282 -0
- package/src/__tests__/stt-stream-session.test.ts +535 -0
- package/src/__tests__/subagent-detail.test.ts +44 -2
- package/src/__tests__/subagent-disposal.test.ts +1 -0
- package/src/__tests__/subagent-fork-notifications.test.ts +291 -0
- package/src/__tests__/subagent-fork-spawn.test.ts +384 -0
- package/src/__tests__/subagent-manager-notify.test.ts +1 -0
- package/src/__tests__/subagent-notify-parent.test.ts +1 -0
- package/src/__tests__/subagent-spawn-tool-fork.test.ts +411 -0
- package/src/__tests__/subagent-tools.test.ts +1 -0
- package/src/__tests__/subagent-types.test.ts +1 -0
- package/src/__tests__/system-prompt-ask-mode.test.ts +27 -71
- package/src/__tests__/system-prompt.test.ts +184 -27
- package/src/__tests__/task-scheduler.test.ts +32 -6
- package/src/__tests__/telegram-config.test.ts +10 -13
- package/src/__tests__/telephony-stt-routing.test.ts +329 -0
- package/src/__tests__/terminal-tools.test.ts +25 -5
- package/src/__tests__/test-preload.ts +18 -0
- package/src/__tests__/test-support/browser-skill-harness.ts +4 -1
- package/src/__tests__/tool-approval-handler.test.ts +73 -0
- package/src/__tests__/tool-executor-lifecycle-events.test.ts +9 -5
- package/src/__tests__/tool-executor-shell-integration.test.ts +4 -0
- package/src/__tests__/tool-executor.test.ts +33 -24
- package/src/__tests__/tool-result-truncation.test.ts +36 -0
- package/src/__tests__/tool-side-effects-slack-dm.test.ts +22 -0
- package/src/__tests__/top-level-renderer.test.ts +73 -1
- package/src/__tests__/transport-hints-queue.test.ts +14 -29
- package/src/__tests__/trust-store.test.ts +7 -1
- package/src/__tests__/trusted-contact-approval-notifier.test.ts +1 -1
- package/src/__tests__/trusted-contact-inline-approval-integration.test.ts +109 -0
- package/src/__tests__/tts-catalog-parity.test.ts +345 -0
- package/src/__tests__/twilio-routes-twiml.test.ts +512 -114
- package/src/__tests__/twilio-routes.test.ts +376 -0
- package/src/__tests__/unicode.test.ts +293 -0
- package/src/__tests__/update-bulletin-format.test.ts +59 -0
- package/src/__tests__/update-bulletin.test.ts +206 -5
- package/src/__tests__/usage-routes.test.ts +25 -4
- package/src/__tests__/user-reference.test.ts +46 -61
- package/src/__tests__/v2-consent-policy.test.ts +103 -0
- package/src/__tests__/verification-control-plane-policy.test.ts +4 -0
- package/src/__tests__/voice-config-update.test.ts +403 -0
- package/src/__tests__/voice-quality.test.ts +434 -19
- package/src/__tests__/workspace-heartbeat-service.test.ts +7 -0
- package/src/__tests__/workspace-migration-033-stt-service-explicit-config.test.ts +547 -0
- package/src/__tests__/workspace-migration-034-remove-calls-voice-transcription-provider.test.ts +596 -0
- package/src/__tests__/workspace-migration-drop-user-md.test.ts +368 -0
- package/src/__tests__/workspace-migration-meets.test.ts +244 -0
- package/src/__tests__/workspace-migration-seed-device-id.test.ts +14 -20
- package/src/__tests__/workspace-policy.test.ts +2 -0
- package/src/acp/client-handler.ts +30 -4
- package/src/agent/image-optimize.ts +24 -12
- package/src/agent/loop.ts +55 -9
- package/src/approvals/guardian-request-resolvers.ts +21 -15
- package/src/backup/__tests__/backup-key.test.ts +152 -0
- package/src/backup/__tests__/backup-worker.test.ts +767 -0
- package/src/backup/__tests__/list-snapshots.test.ts +87 -0
- package/src/backup/__tests__/local-writer.test.ts +218 -0
- package/src/backup/__tests__/offsite-writer.test.ts +641 -0
- package/src/backup/__tests__/paths.test.ts +300 -0
- package/src/backup/__tests__/restore.test.ts +498 -0
- package/src/backup/__tests__/snapshot-lock.test.ts +352 -0
- package/src/backup/__tests__/stream-crypt.test.ts +228 -0
- package/src/backup/backup-key.ts +137 -0
- package/src/backup/backup-worker.ts +459 -0
- package/src/backup/list-snapshots.ts +147 -0
- package/src/backup/local-writer.ts +133 -0
- package/src/backup/offsite-writer.ts +222 -0
- package/src/backup/paths.ts +226 -0
- package/src/backup/restore.ts +322 -0
- package/src/backup/snapshot-lock.ts +431 -0
- package/src/backup/stream-crypt.ts +263 -0
- package/src/browser-session/__tests__/manager.test.ts +297 -0
- package/src/browser-session/backends/cdp-inspect.ts +30 -0
- package/src/browser-session/backends/extension.ts +26 -0
- package/src/browser-session/backends/local.ts +24 -0
- package/src/browser-session/events.ts +164 -0
- package/src/browser-session/index.ts +27 -0
- package/src/browser-session/manager.ts +159 -0
- package/src/browser-session/types.ts +28 -0
- package/src/bundler/package-resolver.ts +4 -0
- package/src/calls/audio-store.ts +11 -5
- package/src/calls/call-controller.ts +226 -71
- package/src/calls/call-domain.ts +9 -0
- package/src/calls/call-speech-output.ts +190 -0
- package/src/calls/call-transport.ts +77 -0
- package/src/calls/media-stream-audio-transcode.ts +173 -0
- package/src/calls/media-stream-output.ts +660 -0
- package/src/calls/media-stream-parser.ts +300 -0
- package/src/calls/media-stream-protocol.ts +166 -0
- package/src/calls/media-stream-server.ts +592 -0
- package/src/calls/media-stream-stt-session.ts +460 -0
- package/src/calls/media-turn-detector.ts +230 -0
- package/src/calls/relay-server.ts +90 -75
- package/src/calls/resolve-call-tts-provider.ts +136 -0
- package/src/calls/telephony-stt-routing.ts +145 -0
- package/src/calls/tts-call-strategy.ts +161 -0
- package/src/calls/tts-text-sanitizer.ts +32 -16
- package/src/calls/twilio-routes.ts +281 -17
- package/src/calls/voice-quality.ts +78 -35
- package/src/calls/voice-session-bridge.ts +8 -1
- package/src/channels/__tests__/types.test.ts +134 -0
- package/src/channels/types.ts +69 -3
- package/src/cli/__tests__/run-assistant-command.ts +11 -1
- package/src/cli/commands/__tests__/backup.test.ts +1165 -0
- package/src/cli/commands/__tests__/domain-register.test.ts +234 -0
- package/src/cli/commands/__tests__/domain-status.test.ts +132 -0
- package/src/cli/commands/__tests__/email-attachment.test.ts +422 -0
- package/src/cli/commands/__tests__/email-download.test.ts +16 -1
- package/src/cli/commands/__tests__/email-list.test.ts +22 -4
- package/src/cli/commands/__tests__/email-register.test.ts +4 -4
- package/src/cli/commands/__tests__/email-send.test.ts +37 -4
- package/src/cli/commands/__tests__/email-status.test.ts +5 -1
- package/src/cli/commands/__tests__/email-unregister.test.ts +34 -5
- package/src/cli/commands/backup.ts +993 -0
- package/src/cli/commands/conversations.ts +77 -0
- package/src/cli/commands/credentials.ts +3 -4
- package/src/cli/commands/domain.ts +210 -0
- package/src/cli/commands/email.ts +273 -16
- package/src/cli/commands/mcp.ts +16 -4
- package/src/cli/commands/oauth/__tests__/connect.test.ts +56 -44
- package/src/cli/commands/oauth/__tests__/disconnect.test.ts +21 -21
- package/src/cli/commands/oauth/__tests__/mode.test.ts +17 -17
- package/src/cli/commands/oauth/__tests__/ping.test.ts +16 -16
- package/src/cli/commands/oauth/__tests__/providers-delete.test.ts +32 -33
- package/src/cli/commands/oauth/__tests__/providers-register.test.ts +330 -0
- package/src/cli/commands/oauth/__tests__/providers-update.test.ts +117 -12
- package/src/cli/commands/oauth/__tests__/status.test.ts +10 -10
- package/src/cli/commands/oauth/__tests__/token.test.ts +7 -7
- package/src/cli/commands/oauth/apps.ts +7 -4
- package/src/cli/commands/oauth/connect.ts +6 -3
- package/src/cli/commands/oauth/disconnect.ts +1 -1
- package/src/cli/commands/oauth/mode.ts +12 -3
- package/src/cli/commands/oauth/providers.ts +215 -36
- package/src/cli/commands/oauth/shared.ts +7 -6
- package/src/cli/commands/platform/__tests__/callback-routes-list.test.ts +254 -0
- package/src/cli/commands/platform/__tests__/connect.test.ts +6 -0
- package/src/cli/commands/platform/__tests__/disconnect.test.ts +7 -1
- package/src/cli/commands/platform/__tests__/status.test.ts +6 -0
- package/src/cli/commands/platform/index.ts +107 -10
- package/src/cli/commands/usage.ts +10 -9
- package/src/cli/lib/daemon-credential-client.ts +4 -0
- package/src/cli/program.ts +30 -4
- package/src/config/__tests__/backup-schema.test.ts +134 -0
- package/src/config/assistant-feature-flags.ts +61 -62
- package/src/config/bundled-skills/app-builder/SKILL.md +26 -249
- package/src/config/bundled-skills/app-builder/references/CUSTOM_ROUTES.md +141 -0
- package/src/config/bundled-skills/app-builder/references/INTERACTION_HOOKS.md +56 -0
- package/src/config/bundled-skills/app-builder/references/WIDGETS.md +125 -0
- package/src/config/bundled-skills/browser/SKILL.md +30 -5
- package/src/config/bundled-skills/browser/TOOLS.json +123 -0
- package/src/config/bundled-skills/browser/tools/browser-attach.ts +12 -0
- package/src/config/bundled-skills/browser/tools/browser-detach.ts +12 -0
- package/src/config/bundled-skills/browser/tools/browser-status.ts +12 -0
- package/src/config/bundled-skills/browser/tools/browser-wait-for-download.ts +17 -0
- package/src/config/bundled-skills/contacts/SKILL.md +5 -2
- package/src/config/bundled-skills/document/SKILL.md +4 -0
- package/src/config/bundled-skills/gmail/SKILL.md +54 -8
- package/src/config/bundled-skills/gmail/TOOLS.json +33 -3
- package/src/config/bundled-skills/gmail/tools/gmail-archive.ts +116 -9
- package/src/config/bundled-skills/gmail/tools/gmail-outreach-scan.ts +138 -11
- package/src/config/bundled-skills/gmail/tools/gmail-preferences-tool.ts +59 -0
- package/src/config/bundled-skills/gmail/tools/gmail-preferences.ts +82 -0
- package/src/config/bundled-skills/gmail/tools/gmail-sender-digest.ts +113 -17
- package/src/config/bundled-skills/gmail/tools/gmail-unsubscribe.ts +2 -2
- package/src/config/bundled-skills/media-processing/SKILL.md +3 -9
- package/src/config/bundled-skills/media-processing/TOOLS.json +1 -6
- package/src/config/bundled-skills/media-processing/__tests__/audio-transcribe.test.ts +125 -0
- package/src/config/bundled-skills/media-processing/__tests__/extract-keyframes.test.ts +181 -0
- package/src/config/bundled-skills/media-processing/__tests__/preprocess-audio.test.ts +141 -0
- package/src/config/bundled-skills/media-processing/services/audio-transcribe.ts +32 -87
- package/src/config/bundled-skills/media-processing/services/preprocess.ts +8 -4
- package/src/config/bundled-skills/media-processing/tools/extract-keyframes.ts +0 -10
- package/src/config/bundled-skills/messaging/SKILL.md +3 -3
- package/src/config/bundled-skills/messaging/tools/messaging-archive-by-sender.ts +2 -2
- package/src/config/bundled-skills/outlook/SKILL.md +9 -2
- package/src/config/bundled-skills/outlook/tools/outlook-unsubscribe.ts +2 -2
- package/src/config/bundled-skills/phone-calls/SKILL.md +2 -2
- package/src/config/bundled-skills/phone-calls/references/CONFIG.md +27 -18
- package/src/config/bundled-skills/phone-calls/references/TROUBLESHOOTING.md +3 -3
- package/src/config/bundled-skills/settings/TOOLS.json +3 -3
- package/src/config/bundled-skills/settings/tools/voice-config-update.ts +26 -22
- package/src/config/bundled-skills/slack/SKILL.md +1 -0
- package/src/config/bundled-skills/subagent/SKILL.md +21 -0
- package/src/config/bundled-skills/subagent/TOOLS.json +8 -4
- package/src/config/bundled-skills/tasks/SKILL.md +5 -0
- package/src/config/bundled-skills/transcribe/SKILL.md +9 -14
- package/src/config/bundled-skills/transcribe/TOOLS.json +2 -7
- package/src/config/bundled-skills/transcribe/tools/transcribe-media.test.ts +256 -0
- package/src/config/bundled-skills/transcribe/tools/transcribe-media.ts +38 -188
- package/src/config/bundled-tool-registry.ts +8 -0
- package/src/config/env-registry.ts +38 -0
- package/src/config/env.ts +49 -4
- package/src/config/feature-flag-registry.json +85 -14
- package/src/config/loader.ts +82 -13
- package/src/config/sanitize-for-transfer.ts +47 -0
- package/src/config/schema.ts +81 -15
- package/src/config/schemas/__tests__/stt.test.ts +43 -0
- package/src/config/schemas/analysis.ts +51 -0
- package/src/config/schemas/backup.ts +72 -0
- package/src/config/schemas/calls.ts +1 -26
- package/src/config/schemas/elevenlabs.ts +0 -59
- package/src/config/schemas/filing.ts +47 -7
- package/src/config/schemas/heartbeat.ts +27 -5
- package/src/config/schemas/host-browser.ts +112 -0
- package/src/config/schemas/inference.ts +1 -1
- package/src/config/schemas/memory-lifecycle.ts +14 -2
- package/src/config/schemas/memory-retrieval.ts +103 -0
- package/src/config/schemas/security.ts +0 -6
- package/src/config/schemas/services.ts +52 -0
- package/src/config/schemas/stt.ts +59 -0
- package/src/config/schemas/tts.ts +230 -0
- package/src/config/schemas/updates.ts +14 -0
- package/src/config/skills.ts +4 -0
- package/src/config/types.ts +4 -1
- package/src/contacts/contact-store.ts +56 -11
- package/src/contacts/contacts-write.ts +38 -1
- package/src/context/post-turn-tool-result-truncation.ts +177 -0
- package/src/context/tool-result-truncation.ts +2 -1
- package/src/context/window-manager.ts +61 -10
- package/src/credential-execution/approval-bridge.ts +49 -15
- package/src/credential-execution/executable-discovery.ts +12 -2
- package/src/credential-execution/process-manager.ts +33 -2
- package/src/credential-health/credential-health-service.ts +366 -0
- package/src/daemon/__tests__/conversation-lifecycle-auto-analyze.test.ts +324 -0
- package/src/daemon/__tests__/conversation-surfaces-launch.test.ts +497 -0
- package/src/daemon/__tests__/conversation-tool-setup.test.ts +195 -0
- package/src/daemon/__tests__/lifecycle-startup-ordering.test.ts +127 -0
- package/src/daemon/app-source-watcher.ts +35 -0
- package/src/daemon/config-watcher.ts +99 -5
- package/src/daemon/context-overflow-approval.ts +5 -0
- package/src/daemon/conversation-agent-loop-handlers.ts +23 -2
- package/src/daemon/conversation-agent-loop.ts +153 -42
- package/src/daemon/conversation-attachments.ts +40 -0
- package/src/daemon/conversation-error.ts +11 -0
- package/src/daemon/conversation-history.ts +40 -6
- package/src/daemon/conversation-launch.ts +220 -0
- package/src/daemon/conversation-lifecycle.ts +59 -9
- package/src/daemon/conversation-messaging.ts +37 -3
- package/src/daemon/conversation-notifiers.ts +5 -0
- package/src/daemon/conversation-process.ts +622 -13
- package/src/daemon/conversation-queue-manager.ts +24 -0
- package/src/daemon/conversation-runtime-assembly.ts +128 -36
- package/src/daemon/conversation-slash.ts +36 -0
- package/src/daemon/conversation-surfaces.ts +131 -40
- package/src/daemon/conversation-tool-setup.ts +99 -8
- package/src/daemon/conversation-usage.ts +7 -4
- package/src/daemon/conversation-workspace.ts +12 -0
- package/src/daemon/conversation.ts +292 -16
- package/src/daemon/date-context.ts +10 -10
- package/src/daemon/first-greeting.ts +3 -2
- package/src/daemon/handlers/config-slack-channel.ts +269 -94
- package/src/daemon/handlers/conversations.ts +13 -141
- package/src/daemon/handlers/shared.ts +80 -0
- package/src/daemon/handlers/skills.ts +483 -44
- package/src/daemon/host-bash-proxy.ts +48 -13
- package/src/daemon/host-browser-proxy.ts +192 -0
- package/src/daemon/host-cu-proxy.ts +36 -11
- package/src/daemon/host-file-proxy.ts +57 -9
- package/src/daemon/lifecycle.ts +179 -28
- package/src/daemon/message-protocol.ts +13 -0
- package/src/daemon/message-types/conversations.ts +89 -14
- package/src/daemon/message-types/home.ts +40 -0
- package/src/daemon/message-types/host-browser.ts +100 -0
- package/src/daemon/message-types/meet.ts +143 -0
- package/src/daemon/message-types/messages.ts +19 -5
- package/src/daemon/message-types/schedules.ts +34 -2
- package/src/daemon/message-types/skills.ts +26 -0
- package/src/daemon/message-types/subagents.ts +2 -0
- package/src/daemon/message-types/surfaces.ts +2 -0
- package/src/daemon/server.ts +439 -14
- package/src/daemon/shutdown-handlers.ts +32 -4
- package/src/daemon/shutdown-registry.ts +40 -0
- package/src/daemon/tool-side-effects.ts +15 -0
- package/src/daemon/transport-hints.ts +5 -24
- package/src/email/html-renderer.ts +76 -0
- package/src/heartbeat/heartbeat-service.ts +93 -7
- package/src/home/__tests__/assistant-feed-authoring.test.ts +156 -0
- package/src/home/__tests__/emit-feed-event.test.ts +169 -0
- package/src/home/__tests__/feed-scheduler.test.ts +194 -0
- package/src/home/__tests__/feed-types.test.ts +275 -0
- package/src/home/__tests__/feed-writer.test.ts +688 -0
- package/src/home/__tests__/phase5-exit-criteria.test.ts +212 -0
- package/src/home/__tests__/platform-gmail-digest.test.ts +222 -0
- package/src/home/__tests__/progress-formula.test.ts +213 -0
- package/src/home/__tests__/relationship-state-writer.test.ts +740 -0
- package/src/home/__tests__/rollup-producer.test.ts +398 -0
- package/src/home/assistant-feed-authoring.ts +124 -0
- package/src/home/emit-feed-event.ts +158 -0
- package/src/home/feed-scheduler.ts +247 -0
- package/src/home/feed-types.ts +181 -0
- package/src/home/feed-writer.ts +469 -0
- package/src/home/platform-gmail-digest.ts +163 -0
- package/src/home/progress-formula.ts +86 -0
- package/src/home/relationship-state-writer.ts +824 -0
- package/src/home/relationship-state.ts +143 -0
- package/src/home/rollup-producer.ts +384 -0
- package/src/hooks/runner.ts +7 -0
- package/src/inbound/platform-callback-registration.ts +30 -20
- package/src/inbound/public-ingress-urls.ts +12 -0
- package/src/instrument.ts +1 -1
- package/src/ipc/__tests__/cli-ipc.test.ts +200 -0
- package/src/ipc/cli-client.ts +151 -0
- package/src/ipc/cli-server.ts +234 -0
- package/src/ipc/gateway-client.ts +180 -0
- package/src/ipc/routes/index.ts +5 -0
- package/src/ipc/routes/wake-conversation.ts +19 -0
- package/src/mcp/client.ts +59 -24
- package/src/memory/__tests__/auto-analysis-enqueue.test.ts +356 -0
- package/src/memory/__tests__/auto-analysis-guard.test.ts +57 -0
- package/src/memory/__tests__/conversation-analyze-job.test.ts +232 -0
- package/src/memory/__tests__/find-analysis-conversation.test.ts +196 -0
- package/src/memory/app-store.ts +31 -1
- package/src/memory/attachments-store.ts +70 -0
- package/src/memory/auto-analysis-enqueue.ts +127 -0
- package/src/memory/auto-analysis-guard.ts +27 -0
- package/src/memory/cleanup-schedule-state.ts +37 -0
- package/src/memory/conversation-analyze-job.ts +73 -0
- package/src/memory/conversation-crud.ts +122 -0
- package/src/memory/conversation-disk-view.ts +7 -0
- package/src/memory/conversation-group-migration.ts +34 -2
- package/src/memory/conversation-queries.ts +6 -5
- package/src/memory/conversation-starters-cadence.ts +76 -0
- package/src/memory/conversation-title-service.ts +5 -2
- package/src/memory/db-init.ts +18 -0
- package/src/memory/db-maintenance.ts +108 -0
- package/src/memory/db.ts +1 -0
- package/src/memory/embedding-backend.test.ts +75 -0
- package/src/memory/embedding-backend.ts +131 -5
- package/src/memory/embedding-gemini.test.ts +54 -0
- package/src/memory/embedding-gemini.ts +20 -9
- package/src/memory/embedding-local.ts +176 -17
- package/src/memory/graph/consolidation.ts +10 -23
- package/src/memory/graph/conversation-graph-memory.ts +15 -0
- package/src/memory/graph/extraction-job.ts +15 -0
- package/src/memory/graph/extraction.test.ts +23 -0
- package/src/memory/graph/extraction.ts +8 -0
- package/src/memory/graph/retriever.ts +67 -40
- package/src/memory/graph/scoring.test.ts +186 -0
- package/src/memory/graph/scoring.ts +31 -1
- package/src/memory/graph/store.test.ts +7 -3
- package/src/memory/graph/store.ts +47 -12
- package/src/memory/graph/tools.ts +1 -1
- package/src/memory/group-crud.ts +6 -1
- package/src/memory/indexer.ts +95 -16
- package/src/memory/job-handlers/cleanup.ts +11 -8
- package/src/memory/job-handlers/conversation-starters.ts +16 -10
- package/src/memory/jobs-store.ts +64 -4
- package/src/memory/jobs-worker.ts +22 -9
- package/src/memory/llm-usage-store.ts +137 -60
- package/src/memory/migrations/213-oauth-providers-scope-separator.ts +13 -0
- package/src/memory/migrations/214-oauth-providers-refresh-url.ts +11 -0
- package/src/memory/migrations/215-oauth-providers-revoke.ts +14 -0
- package/src/memory/migrations/216-oauth-providers-token-auth-method.ts +30 -0
- package/src/memory/migrations/217-conversation-host-access.ts +40 -0
- package/src/memory/migrations/218-oauth-providers-logo-url.ts +11 -0
- package/src/memory/migrations/219-oauth-providers-token-exchange-body-format.ts +15 -0
- package/src/memory/migrations/220-normalize-user-file-by-principal.ts +190 -0
- package/src/memory/migrations/221-conversations-archived-at.ts +16 -0
- package/src/memory/migrations/index.ts +12 -0
- package/src/memory/migrations/registry.ts +16 -0
- package/src/memory/qdrant-manager.ts +43 -16
- package/src/memory/schema/conversations.ts +3 -0
- package/src/memory/schema/oauth.ts +21 -13
- package/src/memory/usage-buckets.ts +396 -0
- package/src/messaging/providers/gmail/client.ts +57 -6
- package/src/messaging/providers/slack/__tests__/adapter-token-routing.test.ts +282 -0
- package/src/messaging/providers/slack/adapter.ts +143 -38
- package/src/messaging/providers/slack/client.ts +16 -0
- package/src/messaging/providers/slack/types.ts +4 -0
- package/src/notifications/decision-engine.ts +3 -3
- package/src/notifications/signal.ts +5 -0
- package/src/oauth/AGENTS.md +76 -0
- package/src/oauth/__tests__/identity-verifier.test.ts +25 -19
- package/src/oauth/__tests__/seed-providers-managed.test.ts +32 -0
- package/src/oauth/byo-connection.test.ts +26 -9
- package/src/oauth/byo-connection.ts +10 -8
- package/src/oauth/connect-orchestrator.ts +25 -21
- package/src/oauth/connect-types.ts +3 -3
- package/src/oauth/connection-resolver.test.ts +17 -4
- package/src/oauth/connection-resolver.ts +22 -18
- package/src/oauth/connection.ts +3 -1
- package/src/oauth/manual-token-connection.ts +13 -13
- package/src/oauth/oauth-store.ts +223 -100
- package/src/oauth/platform-connection.test.ts +101 -3
- package/src/oauth/platform-connection.ts +56 -35
- package/src/oauth/provider-serializer.ts +31 -5
- package/src/oauth/revoke.ts +76 -0
- package/src/oauth/seed-providers.ts +133 -87
- package/src/oauth/token-persistence.ts +1 -1
- package/src/permissions/checker.ts +16 -6
- package/src/permissions/defaults.ts +49 -1
- package/src/permissions/permission-mode.ts +4 -11
- package/src/permissions/prompter.ts +13 -1
- package/src/permissions/trust-store.ts +3 -3
- package/src/permissions/v2-consent-policy.ts +87 -0
- package/src/permissions/workspace-policy.ts +3 -0
- package/src/platform/client.test.ts +10 -0
- package/src/platform/sync-identity.ts +129 -0
- package/src/prompts/persona-resolver.ts +126 -2
- package/src/prompts/system-prompt.ts +76 -38
- package/src/prompts/templates/BOOTSTRAP-REFERENCE.md +3 -65
- package/src/prompts/templates/BOOTSTRAP.md +59 -105
- package/src/prompts/templates/SOUL.md +3 -1
- package/src/prompts/templates/UPDATES.md +12 -0
- package/src/prompts/templates/channels/slack.md +20 -0
- package/src/prompts/update-bulletin-format.ts +26 -9
- package/src/prompts/update-bulletin.ts +34 -23
- package/src/prompts/user-reference.ts +20 -17
- package/src/providers/__tests__/provider-secret-catalog.test.ts +42 -0
- package/src/providers/anthropic/client.ts +157 -60
- package/src/providers/fireworks/client.ts +2 -2
- package/src/providers/gemini/client.ts +9 -1
- package/src/providers/model-catalog.ts +6 -0
- package/src/providers/model-intents.ts +4 -4
- package/src/providers/ollama/client.ts +2 -2
- package/src/providers/openai/chat-completions-provider.ts +474 -0
- package/src/providers/openai/client.ts +25 -440
- package/src/providers/openai/responses-provider.ts +502 -0
- package/src/providers/openrouter/client.ts +101 -4
- package/src/providers/provider-secret-catalog.ts +139 -0
- package/src/providers/registry.ts +2 -2
- package/src/providers/retry.ts +14 -3
- package/src/providers/speech-to-text/__tests__/provider-catalog.test.ts +251 -0
- package/src/providers/speech-to-text/__tests__/resolve.test.ts +828 -0
- package/src/providers/speech-to-text/deepgram-realtime.test.ts +980 -0
- package/src/providers/speech-to-text/deepgram-realtime.ts +767 -0
- package/src/providers/speech-to-text/deepgram.test.ts +332 -0
- package/src/providers/speech-to-text/deepgram.ts +115 -0
- package/src/providers/speech-to-text/google-gemini-live-stream.test.ts +743 -0
- package/src/providers/speech-to-text/google-gemini-live-stream.ts +625 -0
- package/src/providers/speech-to-text/google-gemini.test.ts +226 -0
- package/src/providers/speech-to-text/google-gemini.ts +101 -0
- package/src/providers/speech-to-text/openai-whisper-stream.test.ts +564 -0
- package/src/providers/speech-to-text/openai-whisper-stream.ts +381 -0
- package/src/providers/speech-to-text/openai-whisper.test.ts +1 -37
- package/src/providers/speech-to-text/openai-whisper.ts +63 -33
- package/src/providers/speech-to-text/provider-catalog.ts +306 -0
- package/src/providers/speech-to-text/resolve.ts +386 -6
- package/src/providers/types.ts +10 -1
- package/src/runtime/AGENTS.md +65 -0
- package/src/runtime/__tests__/agent-wake.test.ts +831 -0
- package/src/runtime/__tests__/browser-extension-pair-routes.test.ts +715 -0
- package/src/runtime/__tests__/capability-tokens.test.ts +258 -0
- package/src/runtime/__tests__/chrome-extension-registry.test.ts +518 -0
- package/src/runtime/__tests__/runtime-mode.test.ts +62 -0
- package/src/runtime/__tests__/slack-block-formatting.test.ts +481 -0
- package/src/runtime/agent-wake.ts +512 -0
- package/src/runtime/assistant-event-hub.ts +2 -2
- package/src/runtime/auth/__tests__/guard-tests.test.ts +1 -0
- package/src/runtime/auth/__tests__/middleware.test.ts +116 -1
- package/src/runtime/auth/__tests__/route-policy.test.ts +48 -0
- package/src/runtime/auth/middleware.ts +98 -0
- package/src/runtime/auth/route-policy.ts +33 -9
- package/src/runtime/auth/token-service.ts +56 -1
- package/src/runtime/btw-sidechain.ts +2 -0
- package/src/runtime/capability-tokens.ts +414 -0
- package/src/runtime/channel-approvals.ts +18 -5
- package/src/runtime/channel-invite-transport.ts +1 -1
- package/src/runtime/channel-invite-transports/email.ts +14 -6
- package/src/runtime/channel-readiness-service.ts +12 -22
- package/src/runtime/chrome-extension-registry.ts +368 -0
- package/src/runtime/confirmation-request-guardian-bridge.ts +6 -0
- package/src/runtime/guardian-decision-types.ts +7 -0
- package/src/runtime/http-server.ts +815 -75
- package/src/runtime/http-types.ts +6 -2
- package/src/runtime/migrations/__tests__/rebind-secrets-credentials.test.ts +172 -0
- package/src/runtime/migrations/__tests__/vbundle-builder-credentials.test.ts +276 -0
- package/src/runtime/migrations/__tests__/vbundle-import-credentials.test.ts +198 -0
- package/src/runtime/migrations/__tests__/vbundle-legacy-user-md.test.ts +360 -0
- package/src/runtime/migrations/migration-transport.ts +7 -0
- package/src/runtime/migrations/migration-wizard.ts +23 -2
- package/src/runtime/migrations/rebind-secrets-screen.ts +76 -15
- package/src/runtime/migrations/vbundle-builder.ts +145 -38
- package/src/runtime/migrations/vbundle-import-analyzer.ts +96 -1
- package/src/runtime/migrations/vbundle-importer.ts +89 -5
- package/src/runtime/pending-interactions.ts +18 -13
- package/src/runtime/routes/__tests__/backup-routes.test.ts +967 -0
- package/src/runtime/routes/__tests__/home-feed-routes.test.ts +507 -0
- package/src/runtime/routes/__tests__/migration-import-credential-filter.test.ts +208 -0
- package/src/runtime/routes/__tests__/stt-routes.test.ts +406 -0
- package/src/runtime/routes/__tests__/tts-routes.test.ts +474 -0
- package/src/runtime/routes/__tests__/user-route-dispatcher.test.ts +148 -17
- package/src/runtime/routes/app-management-routes.ts +12 -18
- package/src/runtime/routes/approval-routes.ts +90 -16
- package/src/runtime/routes/attachment-routes.test.ts +9 -3
- package/src/runtime/routes/attachment-routes.ts +216 -17
- package/src/runtime/routes/backup-routes.ts +519 -0
- package/src/runtime/routes/browser-extension-pair-routes.ts +556 -0
- package/src/runtime/routes/btw-routes.ts +8 -6
- package/src/runtime/routes/contact-routes.test.ts +298 -0
- package/src/runtime/routes/contact-routes.ts +132 -5
- package/src/runtime/routes/conversation-analysis-routes.ts +22 -141
- package/src/runtime/routes/conversation-management-routes.ts +223 -0
- package/src/runtime/routes/conversation-routes.ts +598 -103
- package/src/runtime/routes/conversation-starter-routes.ts +78 -16
- package/src/runtime/routes/filing-routes.ts +93 -0
- package/src/runtime/routes/guardian-action-routes.ts +24 -13
- package/src/runtime/routes/home-feed-routes.ts +334 -0
- package/src/runtime/routes/home-state-routes.ts +138 -0
- package/src/runtime/routes/host-browser-routes.ts +268 -0
- package/src/runtime/routes/host-file-routes.ts +9 -1
- package/src/runtime/routes/identity-intro-cache.ts +7 -3
- package/src/runtime/routes/identity-routes.ts +262 -33
- package/src/runtime/routes/inbound-stages/transcribe-audio.test.ts +46 -39
- package/src/runtime/routes/inbound-stages/transcribe-audio.ts +15 -15
- package/src/runtime/routes/integrations/slack/__tests__/channel.test.ts +137 -0
- package/src/runtime/routes/integrations/slack/__tests__/share.test.ts +179 -0
- package/src/runtime/routes/integrations/slack/channel.ts +11 -3
- package/src/runtime/routes/integrations/slack/share.ts +45 -7
- package/src/runtime/routes/llm-context-normalization.ts +303 -0
- package/src/runtime/routes/log-export-routes.ts +42 -22
- package/src/runtime/routes/memory-item-routes.test.ts +3 -2
- package/src/runtime/routes/memory-item-routes.ts +1 -7
- package/src/runtime/routes/migration-routes.ts +122 -2
- package/src/runtime/routes/oauth-apps.ts +15 -17
- package/src/runtime/routes/oauth-providers.ts +4 -0
- package/src/runtime/routes/schedule-routes.ts +24 -11
- package/src/runtime/routes/settings-routes.ts +31 -102
- package/src/runtime/routes/skills-routes.ts +128 -9
- package/src/runtime/routes/stt-routes.ts +233 -0
- package/src/runtime/routes/subagents-routes.ts +14 -10
- package/src/runtime/routes/surface-action-routes.ts +41 -2
- package/src/runtime/routes/tts-routes.ts +108 -24
- package/src/runtime/routes/usage-routes.ts +38 -9
- package/src/runtime/routes/user-route-dispatcher.ts +50 -5
- package/src/runtime/routes/user-routes.ts +13 -1
- package/src/runtime/routes/work-items-routes.ts +8 -1
- package/src/runtime/routes/workspace-routes.test.ts +22 -0
- package/src/runtime/routes/workspace-routes.ts +8 -1
- package/src/runtime/routes/workspace-utils.ts +2 -0
- package/src/runtime/runtime-mode.ts +33 -0
- package/src/runtime/services/__tests__/analyze-conversation.test.ts +444 -0
- package/src/runtime/services/__tests__/analyze-deps-singleton.test.ts +67 -0
- package/src/runtime/services/__tests__/auto-analysis-prompt.test.ts +53 -0
- package/src/runtime/services/__tests__/manual-analysis-prompt.test.ts +41 -0
- package/src/runtime/services/analyze-conversation.ts +344 -0
- package/src/runtime/services/analyze-deps-singleton.ts +32 -0
- package/src/runtime/services/auto-analysis-prompt.ts +55 -0
- package/src/runtime/skill-route-registry.ts +49 -0
- package/src/runtime/slack-block-formatting.ts +437 -10
- package/src/schedule/scheduler.ts +57 -5
- package/src/security/ces-credential-client.ts +20 -0
- package/src/security/ces-rpc-credential-backend.ts +17 -0
- package/src/security/credential-backend.ts +5 -0
- package/src/security/oauth2.ts +68 -29
- package/src/security/secure-keys.ts +143 -27
- package/src/security/token-manager.ts +31 -10
- package/src/sequence/engine.ts +23 -0
- package/src/sequence/types.ts +1 -1
- package/src/skills/catalog-files.ts +554 -0
- package/src/skills/category-inference.ts +122 -0
- package/src/skills/clawhub-files.ts +213 -0
- package/src/skills/clawhub.ts +84 -23
- package/src/skills/skill-file-provider.ts +40 -0
- package/src/skills/skillssh-files.ts +395 -0
- package/src/skills/skillssh-registry.ts +4 -4
- package/src/stt/__tests__/daemon-batch-transcriber.test.ts +392 -0
- package/src/stt/__tests__/types.test.ts +89 -0
- package/src/stt/daemon-batch-transcriber.ts +195 -0
- package/src/stt/stt-stream-session.ts +499 -0
- package/src/stt/types.ts +330 -0
- package/src/stt/wav-encoder.test.ts +373 -0
- package/src/stt/wav-encoder.ts +175 -0
- package/src/subagent/manager.ts +169 -40
- package/src/subagent/types.ts +19 -0
- package/src/tools/apps/executors.ts +11 -2
- package/src/tools/browser/__tests__/auth-detector.test.ts +202 -108
- package/src/tools/browser/__tests__/browser-mode.test.ts +119 -0
- package/src/tools/browser/__tests__/browser-status.test.ts +123 -0
- package/src/tools/browser/auth-detector.ts +43 -12
- package/src/tools/browser/browser-execution.ts +1787 -342
- package/src/tools/browser/browser-manager.ts +81 -12
- package/src/tools/browser/browser-mode-constants.ts +12 -0
- package/src/tools/browser/browser-mode.ts +92 -0
- package/src/tools/browser/browser-status-constants.ts +33 -0
- package/src/tools/browser/cdp-client/__tests__/accessibility-snapshot.test.ts +318 -0
- package/src/tools/browser/cdp-client/__tests__/cdp-dom-helpers.test.ts +1175 -0
- package/src/tools/browser/cdp-client/__tests__/cdp-inspect-client.test.ts +1263 -0
- package/src/tools/browser/cdp-client/__tests__/extension-cdp-client.test.ts +359 -0
- package/src/tools/browser/cdp-client/__tests__/factory.test.ts +1993 -0
- package/src/tools/browser/cdp-client/__tests__/fixtures/ax-tree-nested-frames.json +64 -0
- package/src/tools/browser/cdp-client/__tests__/fixtures/ax-tree-simple.json +69 -0
- package/src/tools/browser/cdp-client/__tests__/local-cdp-client.test.ts +310 -0
- package/src/tools/browser/cdp-client/__tests__/types.test.ts +96 -0
- package/src/tools/browser/cdp-client/accessibility-snapshot.ts +387 -0
- package/src/tools/browser/cdp-client/cdp-dom-helpers.ts +695 -0
- package/src/tools/browser/cdp-client/cdp-inspect/__tests__/discovery.test.ts +1007 -0
- package/src/tools/browser/cdp-client/cdp-inspect/__tests__/ws-transport.test.ts +580 -0
- package/src/tools/browser/cdp-client/cdp-inspect/discovery.ts +744 -0
- package/src/tools/browser/cdp-client/cdp-inspect/ws-transport.ts +579 -0
- package/src/tools/browser/cdp-client/cdp-inspect-client.ts +868 -0
- package/src/tools/browser/cdp-client/errors.ts +49 -0
- package/src/tools/browser/cdp-client/extension-cdp-client.ts +148 -0
- package/src/tools/browser/cdp-client/factory.ts +914 -0
- package/src/tools/browser/cdp-client/index.ts +28 -0
- package/src/tools/browser/cdp-client/local-cdp-client.ts +187 -0
- package/src/tools/browser/cdp-client/types.ts +120 -0
- package/src/tools/credentials/vault.ts +35 -6
- package/src/tools/filesystem/edit.ts +1 -1
- package/src/tools/filesystem/list.ts +1 -1
- package/src/tools/filesystem/read.ts +1 -1
- package/src/tools/filesystem/write.ts +2 -1
- package/src/tools/host-filesystem/edit.ts +1 -1
- package/src/tools/host-filesystem/read.ts +12 -15
- package/src/tools/host-filesystem/write.ts +1 -1
- package/src/tools/host-terminal/host-shell.ts +21 -16
- package/src/tools/network/web-fetch.ts +5 -2
- package/src/tools/network/web-search.ts +5 -2
- package/src/tools/permission-checker.ts +77 -82
- package/src/tools/registry.ts +0 -2
- package/src/tools/secret-detection-handler.ts +34 -0
- package/src/tools/shared/filesystem/image-read.ts +61 -40
- package/src/tools/shared/shell-output.ts +3 -1
- package/src/tools/side-effects.ts +2 -0
- package/src/tools/skills/sandbox-runner.ts +3 -2
- package/src/tools/subagent/spawn.ts +47 -3
- package/src/tools/subagent/status.ts +2 -0
- package/src/tools/system/register.ts +2 -16
- package/src/tools/terminal/safe-env.ts +15 -0
- package/src/tools/terminal/shell.ts +36 -20
- package/src/tools/tool-approval-handler.ts +48 -2
- package/src/tools/tool-manifest.ts +21 -0
- package/src/tools/types.ts +19 -0
- package/src/tools/ui-surface/definitions.ts +6 -1
- package/src/tts/__tests__/provider-adapters.test.ts +834 -0
- package/src/tts/__tests__/provider-catalog-consistency.test.ts +196 -0
- package/src/tts/__tests__/provider-catalog.test.ts +183 -0
- package/src/tts/__tests__/provider-registry.test.ts +90 -0
- package/src/tts/provider-catalog.ts +201 -0
- package/src/tts/provider-registry.ts +73 -0
- package/src/tts/providers/deepgram-provider.ts +219 -0
- package/src/tts/providers/elevenlabs-provider.ts +211 -0
- package/src/tts/providers/fish-audio-provider.ts +183 -0
- package/src/tts/providers/index.ts +42 -0
- package/src/tts/providers/register-builtins.ts +130 -0
- package/src/tts/synthesize-text.ts +110 -0
- package/src/tts/tts-config-resolver.ts +78 -0
- package/src/tts/types.ts +153 -0
- package/src/types/onboarding-context.ts +7 -0
- package/src/util/abort-reasons.ts +58 -0
- package/src/util/device-id.ts +32 -16
- package/src/util/errors.ts +9 -1
- package/src/util/platform.ts +63 -24
- package/src/util/pricing.ts +66 -3
- package/src/util/spawn.ts +1 -1
- package/src/util/truncate.ts +4 -2
- package/src/util/unicode.ts +201 -0
- package/src/version.ts +19 -24
- package/src/watcher/engine.ts +23 -0
- package/src/watcher/watcher-store.ts +31 -0
- package/src/workspace/migrations/003-seed-device-id.ts +9 -3
- package/src/workspace/migrations/017-seed-persona-dirs.ts +68 -4
- package/src/workspace/migrations/029-seed-pkb.ts +1 -1
- package/src/workspace/migrations/031-drop-user-md.ts +317 -0
- package/src/workspace/migrations/031-llm-log-retention-zero-to-null.ts +73 -0
- package/src/workspace/migrations/032-tts-provider-unification.ts +227 -0
- package/src/workspace/migrations/033-stt-service-explicit-config.ts +122 -0
- package/src/workspace/migrations/034-remove-calls-voice-transcription-provider.ts +215 -0
- package/src/workspace/migrations/035-seed-slack-channel-persona.ts +50 -0
- package/src/workspace/migrations/036-update-pkb-index-bar.ts +37 -0
- package/src/workspace/migrations/037-create-meets-dir.ts +61 -0
- package/src/workspace/migrations/registry.ts +16 -0
- package/src/workspace/top-level-renderer.ts +31 -1
- package/src/workspace/turn-commit.ts +31 -0
- package/src/__tests__/chrome-cdp.test.ts +0 -419
- package/src/__tests__/email-cli.test.ts +0 -297
- package/src/__tests__/email-service-config-fallback.test.ts +0 -102
- package/src/__tests__/permission-mode-sse.test.ts +0 -418
- package/src/__tests__/permission-mode-store.test.ts +0 -277
- package/src/browser-extension-relay/protocol.ts +0 -63
- package/src/browser-extension-relay/server.ts +0 -203
- package/src/cli/commands/browser-relay.ts +0 -536
- package/src/config/schemas/sandbox.ts +0 -14
- package/src/email/guardrails.ts +0 -221
- package/src/email/provider.ts +0 -117
- package/src/email/providers/agentmail.ts +0 -361
- package/src/email/providers/index.ts +0 -65
- package/src/email/service.ts +0 -384
- package/src/email/types.ts +0 -126
- package/src/permissions/permission-mode-store.ts +0 -180
- package/src/prompts/templates/USER.md +0 -13
- package/src/providers/speech-to-text/types.ts +0 -17
- package/src/tools/browser/chrome-cdp.ts +0 -239
- package/src/tools/system/set-permission-mode.ts +0 -103
|
@@ -1,13 +1,32 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* HTTP route handlers for Twilio voice webhooks.
|
|
3
3
|
*
|
|
4
|
-
* - handleVoiceWebhook: initial voice webhook; returns TwiML to connect
|
|
4
|
+
* - handleVoiceWebhook: initial voice webhook; returns TwiML to connect
|
|
5
|
+
* ConversationRelay (Twilio-native STT) or Stream (custom media-stream STT)
|
|
5
6
|
* - handleStatusCallback: call status updates (ringing, in-progress, completed, etc.)
|
|
6
7
|
* - handleConnectAction: called when the ConversationRelay connection ends
|
|
8
|
+
*
|
|
9
|
+
* ## STT routing
|
|
10
|
+
*
|
|
11
|
+
* TwiML generation is driven by `services.stt.provider` via
|
|
12
|
+
* {@link resolveTelephonySttRouting}. The resolver returns a discriminated
|
|
13
|
+
* strategy that determines which TwiML path to use:
|
|
14
|
+
*
|
|
15
|
+
* - **`conversation-relay-native`** (deepgram, google-gemini) — emits
|
|
16
|
+
* `<Connect><ConversationRelay>` with Twilio-native `transcriptionProvider`
|
|
17
|
+
* and `speechModel` attributes.
|
|
18
|
+
*
|
|
19
|
+
* - **`media-stream-custom`** (openai-whisper) — emits
|
|
20
|
+
* `<Connect><Stream>` so the daemon receives raw audio and transcribes
|
|
21
|
+
* server-side.
|
|
7
22
|
*/
|
|
8
23
|
|
|
9
24
|
import { loadConfig } from "../config/loader.js";
|
|
10
|
-
import {
|
|
25
|
+
import {
|
|
26
|
+
getTwilioMediaStreamUrl,
|
|
27
|
+
getTwilioRelayUrl,
|
|
28
|
+
} from "../inbound/public-ingress-urls.js";
|
|
29
|
+
import { getProviderEntry } from "../providers/speech-to-text/provider-catalog.js";
|
|
11
30
|
import { mintEdgeRelayToken } from "../runtime/auth/token-service.js";
|
|
12
31
|
import { getLogger } from "../util/logger.js";
|
|
13
32
|
import { persistCallCompletionMessage } from "./call-conversation-messages.js";
|
|
@@ -26,12 +45,33 @@ import {
|
|
|
26
45
|
releaseCallbackClaim,
|
|
27
46
|
updateCallSession,
|
|
28
47
|
} from "./call-store.js";
|
|
48
|
+
import { routeSetup } from "./relay-setup-router.js";
|
|
29
49
|
import { resolveCallHints } from "./stt-hints.js";
|
|
50
|
+
import { resolveTelephonySttRouting } from "./telephony-stt-routing.js";
|
|
30
51
|
import type { CallStatus } from "./types.js";
|
|
31
52
|
import { resolveVoiceQualityProfile } from "./voice-quality.js";
|
|
32
53
|
|
|
33
54
|
const log = getLogger("twilio-routes");
|
|
34
55
|
|
|
56
|
+
// ── Speech config type ───────────────────────────────────────────────
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Twilio ConversationRelay speech-to-text attributes.
|
|
60
|
+
*
|
|
61
|
+
* All values are pre-formatted strings ready for direct insertion into
|
|
62
|
+
* TwiML XML attribute values (XML escaping is the caller's responsibility).
|
|
63
|
+
*/
|
|
64
|
+
export interface TwilioRelaySpeechConfig {
|
|
65
|
+
/** STT provider name (e.g. "Deepgram", "Google"). */
|
|
66
|
+
transcriptionProvider: string;
|
|
67
|
+
/** ASR model identifier, or undefined when the provider default should be used. */
|
|
68
|
+
speechModel: string | undefined;
|
|
69
|
+
/** Comma-separated vocabulary hints for the STT provider, or undefined when no hints are available. */
|
|
70
|
+
hints: string | undefined;
|
|
71
|
+
/** How aggressively the provider detects the start of caller speech. */
|
|
72
|
+
interruptSensitivity: string;
|
|
73
|
+
}
|
|
74
|
+
|
|
35
75
|
// ── Helpers ──────────────────────────────────────────────────────────
|
|
36
76
|
|
|
37
77
|
function escapeXml(str: string): string {
|
|
@@ -49,15 +89,12 @@ export function generateTwiML(
|
|
|
49
89
|
welcomeGreeting: string | null,
|
|
50
90
|
profile: {
|
|
51
91
|
language: string;
|
|
52
|
-
transcriptionProvider: string;
|
|
53
|
-
speechModel?: string;
|
|
54
92
|
ttsProvider: string;
|
|
55
93
|
voice: string;
|
|
56
|
-
interruptSensitivity: string;
|
|
57
94
|
},
|
|
95
|
+
speechConfig: TwilioRelaySpeechConfig,
|
|
58
96
|
relayToken?: string,
|
|
59
97
|
customParameters?: Record<string, string>,
|
|
60
|
-
hints?: string,
|
|
61
98
|
): string {
|
|
62
99
|
const greetingAttr =
|
|
63
100
|
welcomeGreeting && welcomeGreeting.trim().length > 0
|
|
@@ -95,16 +132,76 @@ export function generateTwiML(
|
|
|
95
132
|
${greetingAttr}
|
|
96
133
|
voice="${escapeXml(profile.voice)}"
|
|
97
134
|
language="${escapeXml(profile.language)}"
|
|
98
|
-
transcriptionProvider="${escapeXml(
|
|
135
|
+
transcriptionProvider="${escapeXml(speechConfig.transcriptionProvider)}"${speechConfig.speechModel ? `\n speechModel="${escapeXml(speechConfig.speechModel)}"` : ""}
|
|
99
136
|
ttsProvider="${escapeXml(profile.ttsProvider)}"
|
|
100
137
|
interruptible="true"
|
|
101
138
|
dtmfDetection="true"
|
|
102
|
-
interruptSensitivity="${escapeXml(
|
|
139
|
+
interruptSensitivity="${escapeXml(speechConfig.interruptSensitivity)}"${speechConfig.hints ? `\n hints="${escapeXml(speechConfig.hints)}"` : ""}
|
|
103
140
|
${relayClose}
|
|
104
141
|
</Connect>
|
|
105
142
|
</Response>`;
|
|
106
143
|
}
|
|
107
144
|
|
|
145
|
+
/**
|
|
146
|
+
* Generate `<Connect><Stream>` TwiML for the media-stream STT path.
|
|
147
|
+
*
|
|
148
|
+
* Used when the telephony STT routing resolver selects `media-stream-custom`
|
|
149
|
+
* (e.g. OpenAI Whisper). Twilio opens a WebSocket to `streamUrl` and sends
|
|
150
|
+
* raw audio frames; the daemon transcribes server-side.
|
|
151
|
+
*
|
|
152
|
+
* `callSessionId` and `token` are encoded as **path segments** on the
|
|
153
|
+
* WebSocket URL so the gateway can validate and route the upgrade request
|
|
154
|
+
* before any Twilio `start` frame arrives. Twilio Media Streams does not
|
|
155
|
+
* reliably preserve URL query parameters across the WebSocket upgrade, so
|
|
156
|
+
* path-based encoding is the primary transport for handshake metadata.
|
|
157
|
+
*
|
|
158
|
+
* Both values are also propagated as `<Parameter>` children so Twilio
|
|
159
|
+
* includes them in the `start` event's `customParameters` object for
|
|
160
|
+
* downstream observability.
|
|
161
|
+
*/
|
|
162
|
+
export function generateStreamTwiML(
|
|
163
|
+
callSessionId: string,
|
|
164
|
+
streamUrl: string,
|
|
165
|
+
relayToken?: string,
|
|
166
|
+
customParameters?: Record<string, string>,
|
|
167
|
+
): string {
|
|
168
|
+
// Build the WebSocket URL with callSessionId and token as path segments.
|
|
169
|
+
// Twilio Media Streams does not reliably preserve query parameters
|
|
170
|
+
// across the WebSocket upgrade, so path-based encoding is the primary
|
|
171
|
+
// transport. The gateway extracts metadata from path segments first,
|
|
172
|
+
// falling back to query parameters for legacy compatibility.
|
|
173
|
+
let fullStreamUrl = streamUrl.replace(/\/+$/, "");
|
|
174
|
+
fullStreamUrl += `/${encodeURIComponent(callSessionId)}`;
|
|
175
|
+
if (relayToken) {
|
|
176
|
+
fullStreamUrl += `/${encodeURIComponent(relayToken)}`;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
// Build <Parameter> elements for the Twilio start event payload.
|
|
180
|
+
// Spread customParameters first so callSessionId and token cannot be
|
|
181
|
+
// overridden by caller-supplied values.
|
|
182
|
+
const allParams: Record<string, string> = {
|
|
183
|
+
...customParameters,
|
|
184
|
+
callSessionId,
|
|
185
|
+
};
|
|
186
|
+
|
|
187
|
+
if (relayToken) {
|
|
188
|
+
allParams.token = relayToken;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
let parameterElements = "";
|
|
192
|
+
for (const [key, value] of Object.entries(allParams)) {
|
|
193
|
+
parameterElements += `\n <Parameter name="${escapeXml(key)}" value="${escapeXml(value)}" />`;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
return `<?xml version="1.0" encoding="UTF-8"?>
|
|
197
|
+
<Response>
|
|
198
|
+
<Connect>
|
|
199
|
+
<Stream url="${escapeXml(fullStreamUrl)}">${parameterElements}
|
|
200
|
+
</Stream>
|
|
201
|
+
</Connect>
|
|
202
|
+
</Response>`;
|
|
203
|
+
}
|
|
204
|
+
|
|
108
205
|
export function buildWelcomeGreeting(
|
|
109
206
|
task: string | null,
|
|
110
207
|
configuredGreeting?: string,
|
|
@@ -233,8 +330,17 @@ export async function handleVoiceWebhook(req: Request): Promise<Response> {
|
|
|
233
330
|
|
|
234
331
|
/**
|
|
235
332
|
* Shared TwiML generation for both inbound and outbound voice webhooks.
|
|
236
|
-
*
|
|
237
|
-
*
|
|
333
|
+
*
|
|
334
|
+
* Resolves the telephony STT routing strategy from `services.stt.provider`
|
|
335
|
+
* and branches:
|
|
336
|
+
*
|
|
337
|
+
* - **`conversation-relay-native`** — emits `<Connect><ConversationRelay>`
|
|
338
|
+
* TwiML with Twilio-native STT attributes (`transcriptionProvider`,
|
|
339
|
+
* `speechModel`). Used for deepgram and google-gemini.
|
|
340
|
+
*
|
|
341
|
+
* - **`media-stream-custom`** — emits `<Connect><Stream>` TwiML so the
|
|
342
|
+
* daemon receives raw audio for server-side transcription. Used for
|
|
343
|
+
* openai-whisper.
|
|
238
344
|
*
|
|
239
345
|
* When `verificationSessionId` is provided, it is included as a
|
|
240
346
|
* `<Parameter>` in the TwiML for observability and compatibility with
|
|
@@ -253,18 +359,137 @@ function buildVoiceWebhookTwiml(
|
|
|
253
359
|
} | null,
|
|
254
360
|
verificationSessionId?: string | null,
|
|
255
361
|
): Response {
|
|
256
|
-
const
|
|
257
|
-
|
|
258
|
-
const hints = resolveCallHints(sessionContext, profile.hints);
|
|
362
|
+
const cfg = loadConfig();
|
|
363
|
+
const profile = resolveVoiceQualityProfile(cfg);
|
|
259
364
|
|
|
260
365
|
log.info(
|
|
261
366
|
{ callSessionId, ttsProvider: profile.ttsProvider, voice: profile.voice },
|
|
262
367
|
"Voice quality profile resolved",
|
|
263
368
|
);
|
|
264
369
|
|
|
265
|
-
|
|
266
|
-
|
|
370
|
+
// Resolve telephony STT strategy from services.stt.provider.
|
|
371
|
+
// The routing resolver handles speech-model defaults internally per provider.
|
|
372
|
+
const routingResult = resolveTelephonySttRouting();
|
|
373
|
+
|
|
374
|
+
// Derive Deepgram fallback values from the provider catalog so they stay
|
|
375
|
+
// in sync with the single source of truth. Hardcoded strings are kept only
|
|
376
|
+
// as a final safety net in case the catalog entry is missing.
|
|
377
|
+
const dgEntry = getProviderEntry("deepgram");
|
|
378
|
+
const fallbackProvider =
|
|
379
|
+
dgEntry?.telephonyRouting.twilioNativeMapping?.provider ?? "Deepgram";
|
|
380
|
+
const fallbackModel =
|
|
381
|
+
dgEntry?.telephonyRouting.twilioNativeMapping?.defaultSpeechModel ??
|
|
382
|
+
"nova-3";
|
|
383
|
+
|
|
384
|
+
if (routingResult.status === "unknown-provider") {
|
|
385
|
+
log.error(
|
|
386
|
+
{
|
|
387
|
+
callSessionId,
|
|
388
|
+
providerId: routingResult.providerId,
|
|
389
|
+
reason: routingResult.reason,
|
|
390
|
+
},
|
|
391
|
+
"Telephony STT routing failed — unknown provider; falling back to ConversationRelay with Deepgram",
|
|
392
|
+
);
|
|
393
|
+
// Graceful degradation: fall back to Deepgram ConversationRelay so
|
|
394
|
+
// calls don't fail entirely on a misconfigured provider.
|
|
395
|
+
return buildConversationRelayResponse(
|
|
396
|
+
callSessionId,
|
|
397
|
+
cfg,
|
|
398
|
+
profile,
|
|
399
|
+
sessionContext,
|
|
400
|
+
verificationSessionId,
|
|
401
|
+
{ transcriptionProvider: fallbackProvider, speechModel: fallbackModel },
|
|
402
|
+
);
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
const { strategy } = routingResult;
|
|
406
|
+
|
|
407
|
+
if (strategy.strategy === "conversation-relay-native") {
|
|
408
|
+
return buildConversationRelayResponse(
|
|
409
|
+
callSessionId,
|
|
410
|
+
cfg,
|
|
411
|
+
profile,
|
|
412
|
+
sessionContext,
|
|
413
|
+
verificationSessionId,
|
|
414
|
+
{
|
|
415
|
+
transcriptionProvider: strategy.transcriptionProvider,
|
|
416
|
+
speechModel: strategy.speechModel,
|
|
417
|
+
},
|
|
418
|
+
);
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
// media-stream-custom path — preflight check to reject interactive setup
|
|
422
|
+
// flows that the media-stream transport cannot support. The media-stream
|
|
423
|
+
// server has a defensive fallback for these cases, but catching them here
|
|
424
|
+
// avoids bootstrapping a WebSocket session that will immediately fail.
|
|
425
|
+
const session = getCallSession(callSessionId);
|
|
426
|
+
const from = sessionContext?.fromNumber ?? "";
|
|
427
|
+
const to = sessionContext?.toNumber ?? "";
|
|
267
428
|
|
|
429
|
+
const { outcome } = routeSetup({
|
|
430
|
+
callSessionId,
|
|
431
|
+
session: session ?? null,
|
|
432
|
+
from,
|
|
433
|
+
to,
|
|
434
|
+
});
|
|
435
|
+
|
|
436
|
+
// The media-stream transport supports normal_call and deny (which speaks
|
|
437
|
+
// a message and tears down). All other outcomes require interactive
|
|
438
|
+
// sub-flows (DTMF entry, name capture, guardian wait) that media-stream
|
|
439
|
+
// cannot perform. Reject these deterministically before stream bootstrap.
|
|
440
|
+
if (outcome.action !== "normal_call" && outcome.action !== "deny") {
|
|
441
|
+
log.warn(
|
|
442
|
+
{
|
|
443
|
+
callSessionId,
|
|
444
|
+
setupAction: outcome.action,
|
|
445
|
+
strategy: "media-stream-custom",
|
|
446
|
+
},
|
|
447
|
+
"Media-stream preflight rejected unsupported interactive setup flow — falling back to ConversationRelay with Deepgram",
|
|
448
|
+
);
|
|
449
|
+
// Fall back to ConversationRelay so the interactive flow can proceed
|
|
450
|
+
// through the relay server which supports it natively.
|
|
451
|
+
return buildConversationRelayResponse(
|
|
452
|
+
callSessionId,
|
|
453
|
+
cfg,
|
|
454
|
+
profile,
|
|
455
|
+
sessionContext,
|
|
456
|
+
verificationSessionId,
|
|
457
|
+
{ transcriptionProvider: fallbackProvider, speechModel: fallbackModel },
|
|
458
|
+
);
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
return buildMediaStreamResponse(callSessionId, cfg, verificationSessionId);
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
/**
|
|
465
|
+
* Build a ConversationRelay TwiML response for Twilio-native STT providers.
|
|
466
|
+
*/
|
|
467
|
+
function buildConversationRelayResponse(
|
|
468
|
+
callSessionId: string,
|
|
469
|
+
cfg: ReturnType<typeof loadConfig>,
|
|
470
|
+
profile: ReturnType<typeof resolveVoiceQualityProfile>,
|
|
471
|
+
sessionContext: {
|
|
472
|
+
task: string | null;
|
|
473
|
+
toNumber: string;
|
|
474
|
+
fromNumber: string;
|
|
475
|
+
direction: "inbound" | "outbound";
|
|
476
|
+
inviteFriendName: string | null;
|
|
477
|
+
inviteGuardianName: string | null;
|
|
478
|
+
} | null,
|
|
479
|
+
verificationSessionId: string | null | undefined,
|
|
480
|
+
sttAttrs: { transcriptionProvider: string; speechModel: string | undefined },
|
|
481
|
+
): Response {
|
|
482
|
+
const rawHints = resolveCallHints(sessionContext, profile.hints);
|
|
483
|
+
|
|
484
|
+
const speechConfig: TwilioRelaySpeechConfig = {
|
|
485
|
+
transcriptionProvider: sttAttrs.transcriptionProvider,
|
|
486
|
+
speechModel: sttAttrs.speechModel,
|
|
487
|
+
hints: rawHints && rawHints.length > 0 ? rawHints : undefined,
|
|
488
|
+
interruptSensitivity: profile.interruptSensitivity,
|
|
489
|
+
};
|
|
490
|
+
|
|
491
|
+
const relayUrl = getTwilioRelayUrl(cfg);
|
|
492
|
+
const welcomeGreeting = buildWelcomeGreeting(sessionContext?.task ?? null);
|
|
268
493
|
const relayToken = mintEdgeRelayToken();
|
|
269
494
|
|
|
270
495
|
// Propagate verificationSessionId as a TwiML <Parameter> for
|
|
@@ -278,12 +503,51 @@ function buildVoiceWebhookTwiml(
|
|
|
278
503
|
relayUrl,
|
|
279
504
|
welcomeGreeting,
|
|
280
505
|
profile,
|
|
506
|
+
speechConfig,
|
|
281
507
|
relayToken,
|
|
282
508
|
customParameters,
|
|
283
|
-
hints || undefined,
|
|
284
509
|
);
|
|
285
510
|
|
|
286
|
-
log.info(
|
|
511
|
+
log.info(
|
|
512
|
+
{
|
|
513
|
+
callSessionId,
|
|
514
|
+
strategy: "conversation-relay-native",
|
|
515
|
+
transcriptionProvider: sttAttrs.transcriptionProvider,
|
|
516
|
+
},
|
|
517
|
+
"Returning ConversationRelay TwiML",
|
|
518
|
+
);
|
|
519
|
+
|
|
520
|
+
return new Response(twiml, {
|
|
521
|
+
status: 200,
|
|
522
|
+
headers: { "Content-Type": "text/xml" },
|
|
523
|
+
});
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
/**
|
|
527
|
+
* Build a Stream TwiML response for custom media-stream STT providers.
|
|
528
|
+
*/
|
|
529
|
+
function buildMediaStreamResponse(
|
|
530
|
+
callSessionId: string,
|
|
531
|
+
cfg: ReturnType<typeof loadConfig>,
|
|
532
|
+
verificationSessionId: string | null | undefined,
|
|
533
|
+
): Response {
|
|
534
|
+
const streamUrl = getTwilioMediaStreamUrl(cfg);
|
|
535
|
+
const relayToken = mintEdgeRelayToken();
|
|
536
|
+
|
|
537
|
+
const customParameters: Record<string, string> | undefined =
|
|
538
|
+
verificationSessionId ? { verificationSessionId } : undefined;
|
|
539
|
+
|
|
540
|
+
const twiml = generateStreamTwiML(
|
|
541
|
+
callSessionId,
|
|
542
|
+
streamUrl,
|
|
543
|
+
relayToken,
|
|
544
|
+
customParameters,
|
|
545
|
+
);
|
|
546
|
+
|
|
547
|
+
log.info(
|
|
548
|
+
{ callSessionId, strategy: "media-stream-custom" },
|
|
549
|
+
"Returning Stream TwiML",
|
|
550
|
+
);
|
|
287
551
|
|
|
288
552
|
return new Response(twiml, {
|
|
289
553
|
status: 200,
|
|
@@ -1,9 +1,14 @@
|
|
|
1
1
|
import { loadConfig } from "../config/loader.js";
|
|
2
|
+
import { DEFAULT_ELEVENLABS_VOICE_ID } from "../config/schemas/elevenlabs.js";
|
|
3
|
+
import { getTtsProvider } from "../tts/provider-registry.js";
|
|
4
|
+
import { resolveTtsConfig } from "../tts/tts-config-resolver.js";
|
|
5
|
+
import {
|
|
6
|
+
getNativeTwilioVoiceSpec,
|
|
7
|
+
resolveCallStrategy,
|
|
8
|
+
} from "./tts-call-strategy.js";
|
|
2
9
|
|
|
3
10
|
export interface VoiceQualityProfile {
|
|
4
11
|
language: string;
|
|
5
|
-
transcriptionProvider: string;
|
|
6
|
-
speechModel?: string;
|
|
7
12
|
ttsProvider: string;
|
|
8
13
|
voice: string;
|
|
9
14
|
interruptSensitivity: string;
|
|
@@ -45,49 +50,87 @@ export function buildElevenLabsVoiceSpec(config: {
|
|
|
45
50
|
/**
|
|
46
51
|
* Resolve the effective voice quality profile from config.
|
|
47
52
|
*
|
|
48
|
-
*
|
|
49
|
-
*
|
|
50
|
-
*
|
|
51
|
-
*
|
|
52
|
-
*
|
|
53
|
+
* Uses the explicit call strategy from the TTS provider catalog to
|
|
54
|
+
* determine the call path. The catalog's `callMode` field
|
|
55
|
+
* (`"native-twilio"` vs `"synthesized-play"`) drives the decision
|
|
56
|
+
* rather than inferring behavior from runtime `supportsStreaming`
|
|
57
|
+
* capability.
|
|
53
58
|
*
|
|
54
|
-
* For
|
|
55
|
-
*
|
|
59
|
+
* For **synthesized-play** providers (e.g. Fish Audio),
|
|
60
|
+
* ConversationRelay needs a valid TTS provider in TwiML, so we set
|
|
61
|
+
* `ttsProvider` to `"Google"` as a placeholder and leave `voice` empty
|
|
62
|
+
* since actual audio is delivered via `play` messages.
|
|
63
|
+
*
|
|
64
|
+
* For **native-twilio** providers (e.g. ElevenLabs), `ttsProvider` and
|
|
65
|
+
* `voice` are populated via the provider's registered
|
|
66
|
+
* {@link NativeTwilioVoiceSpec} builder so Twilio handles TTS natively.
|
|
67
|
+
* New native providers plug in by registering a voice-spec builder --
|
|
68
|
+
* no edits to this module required.
|
|
69
|
+
*
|
|
70
|
+
* NOTE: STT provider and speech model are intentionally NOT part of this
|
|
71
|
+
* profile. STT resolution is handled once in the voice webhook route
|
|
72
|
+
* (`twilio-routes.ts`) via `resolveTelephonySttRouting()` to maintain a
|
|
73
|
+
* single point of ownership.
|
|
56
74
|
*/
|
|
57
75
|
export function resolveVoiceQualityProfile(
|
|
58
76
|
config?: ReturnType<typeof loadConfig>,
|
|
59
77
|
): VoiceQualityProfile {
|
|
60
78
|
const cfg = config ?? loadConfig();
|
|
61
79
|
const voice = cfg.calls.voice;
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
80
|
+
|
|
81
|
+
// Resolve the call strategy from catalog metadata.
|
|
82
|
+
// Falls back to native ElevenLabs when config/catalog is unavailable.
|
|
83
|
+
const strategy = resolveCallStrategy(cfg);
|
|
84
|
+
|
|
85
|
+
// Before committing to the catalog-derived strategy, verify the
|
|
86
|
+
// runtime provider is actually registered. If the provider registry
|
|
87
|
+
// hasn't been initialised (early startup, test mocks), fall back to
|
|
88
|
+
// native mode so this function and resolveCallTtsProvider agree on
|
|
89
|
+
// the same degraded-mode path.
|
|
90
|
+
let runtimeAvailable = false;
|
|
91
|
+
try {
|
|
92
|
+
const resolved = resolveTtsConfig(cfg);
|
|
93
|
+
getTtsProvider(resolved.provider);
|
|
94
|
+
runtimeAvailable = true;
|
|
95
|
+
} catch {
|
|
96
|
+
// Provider not registered — will fall through to native path below.
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
let ttsProvider: string;
|
|
100
|
+
let voiceSpec: string;
|
|
101
|
+
|
|
102
|
+
if (runtimeAvailable && strategy.callMode === "synthesized-play") {
|
|
103
|
+
// Synthesized providers stream audio via `play` messages.
|
|
104
|
+
// Twilio still needs a valid ttsProvider in TwiML, so use a
|
|
105
|
+
// placeholder and leave voice empty.
|
|
106
|
+
ttsProvider = "Google";
|
|
107
|
+
voiceSpec = "";
|
|
108
|
+
} else {
|
|
109
|
+
// Native providers: delegate voice-spec building to the
|
|
110
|
+
// provider's registered builder.
|
|
111
|
+
try {
|
|
112
|
+
const spec = getNativeTwilioVoiceSpec(strategy.providerId);
|
|
113
|
+
ttsProvider = spec.twilioProviderName;
|
|
114
|
+
|
|
115
|
+
const resolved = resolveTtsConfig(cfg);
|
|
116
|
+
voiceSpec = spec.buildVoiceSpec(resolved.providerConfig);
|
|
117
|
+
} catch {
|
|
118
|
+
// Voice-spec builder not registered or config unavailable --
|
|
119
|
+
// fall back to ElevenLabs using the config's elevenlabs block.
|
|
120
|
+
ttsProvider = "ElevenLabs";
|
|
121
|
+
voiceSpec = buildElevenLabsVoiceSpec(
|
|
122
|
+
cfg.services?.tts?.providers?.elevenlabs ?? {
|
|
123
|
+
voiceId: DEFAULT_ELEVENLABS_VOICE_ID,
|
|
124
|
+
},
|
|
125
|
+
);
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
|
|
74
129
|
return {
|
|
75
130
|
language: voice.language,
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
ttsProvider: fishAudio ? "Google" : "ElevenLabs",
|
|
79
|
-
voice: fishAudio ? "" : buildElevenLabsVoiceSpec(cfg.elevenlabs),
|
|
131
|
+
ttsProvider,
|
|
132
|
+
voice: voiceSpec,
|
|
80
133
|
interruptSensitivity: voice.interruptSensitivity ?? "low",
|
|
81
134
|
hints: voice.hints ?? [],
|
|
82
135
|
};
|
|
83
136
|
}
|
|
84
|
-
|
|
85
|
-
/**
|
|
86
|
-
* Check whether Fish Audio TTS is configured for phone calls.
|
|
87
|
-
*/
|
|
88
|
-
export function isFishAudioTts(
|
|
89
|
-
config?: ReturnType<typeof loadConfig>,
|
|
90
|
-
): boolean {
|
|
91
|
-
const cfg = config ?? loadConfig();
|
|
92
|
-
return cfg.calls.voice?.ttsProvider === "fish-audio";
|
|
93
|
-
}
|
|
@@ -21,6 +21,7 @@ import { buildAssistantEvent } from "../runtime/assistant-event.js";
|
|
|
21
21
|
import { assistantEventHub } from "../runtime/assistant-event-hub.js";
|
|
22
22
|
import { DAEMON_INTERNAL_ASSISTANT_ID } from "../runtime/assistant-scope.js";
|
|
23
23
|
import { computeToolApprovalDigest } from "../security/tool-approval-digest.js";
|
|
24
|
+
import { createAbortReason } from "../util/abort-reasons.js";
|
|
24
25
|
import { getLogger } from "../util/logger.js";
|
|
25
26
|
import {
|
|
26
27
|
CALL_OPENING_MARKER,
|
|
@@ -539,7 +540,13 @@ export async function startVoiceTurn(
|
|
|
539
540
|
|
|
540
541
|
const abortFn = () => {
|
|
541
542
|
if (conversation.currentRequestId === requestId) {
|
|
542
|
-
conversation.abort(
|
|
543
|
+
conversation.abort(
|
|
544
|
+
createAbortReason(
|
|
545
|
+
"voice_session_aborted",
|
|
546
|
+
"voice-session-bridge.abortFn",
|
|
547
|
+
conversation.conversationId,
|
|
548
|
+
),
|
|
549
|
+
);
|
|
543
550
|
}
|
|
544
551
|
};
|
|
545
552
|
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
import { describe, expect, test } from "bun:test";
|
|
2
|
+
|
|
3
|
+
import {
|
|
4
|
+
INTERACTIVE_INTERFACES,
|
|
5
|
+
INTERFACE_IDS,
|
|
6
|
+
isInterfaceId,
|
|
7
|
+
supportsHostProxy,
|
|
8
|
+
} from "../types.js";
|
|
9
|
+
|
|
10
|
+
describe("INTERFACE_IDS", () => {
|
|
11
|
+
test("includes chrome-extension", () => {
|
|
12
|
+
expect(
|
|
13
|
+
(INTERFACE_IDS as readonly string[]).includes("chrome-extension"),
|
|
14
|
+
).toBe(true);
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
test("still includes macos and other existing interfaces", () => {
|
|
18
|
+
for (const id of [
|
|
19
|
+
"macos",
|
|
20
|
+
"ios",
|
|
21
|
+
"cli",
|
|
22
|
+
"telegram",
|
|
23
|
+
"phone",
|
|
24
|
+
"vellum",
|
|
25
|
+
"whatsapp",
|
|
26
|
+
"slack",
|
|
27
|
+
"email",
|
|
28
|
+
]) {
|
|
29
|
+
expect((INTERFACE_IDS as readonly string[]).includes(id)).toBe(true);
|
|
30
|
+
}
|
|
31
|
+
});
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
describe("INTERACTIVE_INTERFACES", () => {
|
|
35
|
+
test("does NOT include chrome-extension", () => {
|
|
36
|
+
// Chrome extensions don't render SSE-backed prompter UI, so they must
|
|
37
|
+
// stay out of the interactive set even though they have an InterfaceId.
|
|
38
|
+
expect(INTERACTIVE_INTERFACES.has("chrome-extension" as never)).toBe(false);
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
test("still includes macos", () => {
|
|
42
|
+
expect(INTERACTIVE_INTERFACES.has("macos")).toBe(true);
|
|
43
|
+
});
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
describe("isInterfaceId", () => {
|
|
47
|
+
test("returns true for chrome-extension", () => {
|
|
48
|
+
expect(isInterfaceId("chrome-extension")).toBe(true);
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
test("returns true for macos", () => {
|
|
52
|
+
expect(isInterfaceId("macos")).toBe(true);
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
test("returns false for unknown interface", () => {
|
|
56
|
+
expect(isInterfaceId("safari-extension")).toBe(false);
|
|
57
|
+
});
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
describe("supportsHostProxy", () => {
|
|
61
|
+
// ── macOS: supports host_bash / host_file / host_cu, but NOT host_browser. ──
|
|
62
|
+
test("macos returns true (no capability)", () => {
|
|
63
|
+
expect(supportsHostProxy("macos")).toBe(true);
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
test("macos returns true for host_bash", () => {
|
|
67
|
+
expect(supportsHostProxy("macos", "host_bash")).toBe(true);
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
test("macos returns true for host_file", () => {
|
|
71
|
+
expect(supportsHostProxy("macos", "host_file")).toBe(true);
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
test("macos returns true for host_cu", () => {
|
|
75
|
+
expect(supportsHostProxy("macos", "host_cu")).toBe(true);
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
test("macos returns false for host_browser", () => {
|
|
79
|
+
expect(supportsHostProxy("macos", "host_browser")).toBe(false);
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
// ── chrome-extension: only host_browser. ──
|
|
83
|
+
test("chrome-extension returns false (no capability)", () => {
|
|
84
|
+
// Chrome extension does not support "any host proxy at all" — it only
|
|
85
|
+
// supports host_browser, so the no-arg form must return false to keep
|
|
86
|
+
// existing call sites that guard desktop-only behavior unchanged.
|
|
87
|
+
expect(supportsHostProxy("chrome-extension")).toBe(false);
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
test("chrome-extension returns true for host_browser", () => {
|
|
91
|
+
expect(supportsHostProxy("chrome-extension", "host_browser")).toBe(true);
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
test("chrome-extension returns false for host_bash", () => {
|
|
95
|
+
expect(supportsHostProxy("chrome-extension", "host_bash")).toBe(false);
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
test("chrome-extension returns false for host_file", () => {
|
|
99
|
+
expect(supportsHostProxy("chrome-extension", "host_file")).toBe(false);
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
test("chrome-extension returns false for host_cu", () => {
|
|
103
|
+
expect(supportsHostProxy("chrome-extension", "host_cu")).toBe(false);
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
// ── Non-supporting interfaces: false in all forms. ──
|
|
107
|
+
test("cli returns false (no capability)", () => {
|
|
108
|
+
expect(supportsHostProxy("cli")).toBe(false);
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
test("cli returns false for host_bash", () => {
|
|
112
|
+
expect(supportsHostProxy("cli", "host_bash")).toBe(false);
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
test("cli returns false for host_browser", () => {
|
|
116
|
+
expect(supportsHostProxy("cli", "host_browser")).toBe(false);
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
test("telegram returns false (no capability)", () => {
|
|
120
|
+
expect(supportsHostProxy("telegram")).toBe(false);
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
test("telegram returns false for host_browser", () => {
|
|
124
|
+
expect(supportsHostProxy("telegram", "host_browser")).toBe(false);
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
test("vellum returns false (no capability)", () => {
|
|
128
|
+
expect(supportsHostProxy("vellum")).toBe(false);
|
|
129
|
+
});
|
|
130
|
+
|
|
131
|
+
test("email returns false for host_browser", () => {
|
|
132
|
+
expect(supportsHostProxy("email", "host_browser")).toBe(false);
|
|
133
|
+
});
|
|
134
|
+
});
|