@vellumai/assistant 0.6.2 → 0.6.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +273 -10
- package/Dockerfile +2 -3
- package/bun.lock +41 -49
- package/bunfig.toml +3 -0
- package/docs/architecture/memory.md +1 -1
- package/docs/backup-troubleshooting.md +52 -0
- package/docs/browser-use-architecture-phase2.md +174 -0
- package/docs/stt-provider-onboarding.md +120 -0
- package/knip.json +12 -2
- package/node_modules/@vellumai/ces-contracts/bun.lock +8 -6
- package/node_modules/@vellumai/ces-contracts/package.json +3 -3
- package/node_modules/@vellumai/ces-contracts/src/rpc.ts +42 -0
- package/openapi.yaml +1111 -86
- package/package.json +40 -42
- package/scripts/generate-openapi.ts +0 -2
- package/scripts/test.sh +73 -18
- package/src/__tests__/acp-session.test.ts +43 -0
- package/src/__tests__/agent-image-optimize.test.ts +28 -0
- package/src/__tests__/agent-loop.test.ts +123 -0
- package/src/__tests__/anthropic-provider.test.ts +263 -10
- package/src/__tests__/app-builder-tool-scripts.test.ts +1 -0
- package/src/__tests__/app-executors.test.ts +1 -0
- package/src/__tests__/app-source-watcher.test.ts +37 -11
- package/src/__tests__/approval-routes-http.test.ts +178 -1
- package/src/__tests__/auto-analysis-end-to-end.test.ts +550 -0
- package/src/__tests__/auto-analysis-prompt.test.ts +50 -0
- package/src/__tests__/browser-fill-credential.test.ts +240 -94
- package/src/__tests__/browser-manager.test.ts +40 -27
- package/src/__tests__/browser-skill-baseline-tool-payload.test.ts +2 -2
- package/src/__tests__/browser-skill-endstate.test.ts +31 -7
- package/src/__tests__/btw-routes.test.ts +7 -0
- package/src/__tests__/call-controller.test.ts +581 -20
- package/src/__tests__/catalog-files.test.ts +1000 -0
- package/src/__tests__/channel-approvals.test.ts +53 -0
- package/src/__tests__/channel-invite-transport.test.ts +2 -2
- package/src/__tests__/channel-readiness-routes.test.ts +16 -20
- package/src/__tests__/channel-readiness-service.test.ts +12 -7
- package/src/__tests__/checker.test.ts +157 -10
- package/src/__tests__/clawhub-files.test.ts +347 -0
- package/src/__tests__/commit-message-enrichment-service.test.ts +36 -19
- package/src/__tests__/config-analysis.test.ts +100 -0
- package/src/__tests__/config-managed-gemini-defaults.test.ts +326 -0
- package/src/__tests__/config-schema-cmd.test.ts +2 -2
- package/src/__tests__/config-schema.test.ts +1248 -224
- package/src/__tests__/config-watcher-cleanup-throttle.test.ts +339 -0
- package/src/__tests__/config-watcher.test.ts +43 -8
- package/src/__tests__/confirmation-request-guardian-bridge.test.ts +23 -0
- package/src/__tests__/contact-store-user-file.test.ts +512 -0
- package/src/__tests__/contacts-write.test.ts +197 -0
- package/src/__tests__/context-overflow-approval.test.ts +16 -1
- package/src/__tests__/context-window-manager.test.ts +88 -0
- package/src/__tests__/conversation-abort-tool-results.test.ts +2 -0
- package/src/__tests__/conversation-agent-loop-overflow.test.ts +2 -1
- package/src/__tests__/conversation-agent-loop.test.ts +99 -3
- package/src/__tests__/conversation-analysis-routes.test.ts +2 -2
- package/src/__tests__/conversation-attachments.test.ts +80 -4
- package/src/__tests__/conversation-confirmation-signals.test.ts +290 -0
- package/src/__tests__/conversation-error.test.ts +70 -0
- package/src/__tests__/conversation-fork-crud.test.ts +17 -0
- package/src/__tests__/conversation-history-web-search.test.ts +12 -4
- package/src/__tests__/conversation-host-access-routes.test.ts +229 -0
- package/src/__tests__/conversation-init.benchmark.test.ts +6 -1
- package/src/__tests__/conversation-inject-context.test.ts +103 -0
- package/src/__tests__/conversation-launcher-skill-regression.test.ts +51 -0
- package/src/__tests__/conversation-list-source.test.ts +145 -0
- package/src/__tests__/conversation-pre-run-repair.test.ts +2 -0
- package/src/__tests__/conversation-provider-retry-repair.test.ts +2 -0
- package/src/__tests__/conversation-queue.test.ts +946 -62
- package/src/__tests__/conversation-routes-disk-view.test.ts +275 -0
- package/src/__tests__/conversation-routes-guardian-reply.test.ts +16 -0
- package/src/__tests__/conversation-routes-slash-commands.test.ts +1 -0
- package/src/__tests__/conversation-runtime-assembly.test.ts +324 -46
- package/src/__tests__/conversation-skill-tools.test.ts +7 -4
- package/src/__tests__/conversation-slash-commands.test.ts +33 -0
- package/src/__tests__/conversation-slash-queue.test.ts +89 -18
- package/src/__tests__/conversation-slash-unknown.test.ts +2 -0
- package/src/__tests__/conversation-starter-routes.test.ts +126 -0
- package/src/__tests__/conversation-starters-cadence.test.ts +161 -0
- package/src/__tests__/conversation-store.test.ts +195 -0
- package/src/__tests__/conversation-tool-setup-batch-authorized.test.ts +226 -0
- package/src/__tests__/conversation-workspace-cache-state.test.ts +193 -0
- package/src/__tests__/conversation-workspace-injection.test.ts +2 -0
- package/src/__tests__/conversation-workspace-tool-tracking.test.ts +2 -0
- package/src/__tests__/credential-execution-approval-bridge.test.ts +32 -1
- package/src/__tests__/credential-health-service.test.ts +352 -0
- package/src/__tests__/credential-security-invariants.test.ts +6 -3
- package/src/__tests__/credential-vault-unit.test.ts +383 -7
- package/src/__tests__/credential-vault.test.ts +152 -13
- package/src/__tests__/credentials-cli.test.ts +42 -18
- package/src/__tests__/cross-provider-web-search.test.ts +146 -35
- package/src/__tests__/date-context.test.ts +4 -4
- package/src/__tests__/deterministic-verification-control-plane.test.ts +10 -1
- package/src/__tests__/device-id.test.ts +112 -0
- package/src/__tests__/docker-signing-key-bootstrap.test.ts +167 -4
- package/src/__tests__/dynamic-skill-workflow-prompt.test.ts +1 -3
- package/src/__tests__/email-html-renderer.test.ts +71 -0
- package/src/__tests__/email-invite-adapter.test.ts +36 -32
- package/src/__tests__/embedding-managed-proxy-selection.test.ts +256 -0
- package/src/__tests__/emit-event-signal.test.ts +71 -0
- package/src/__tests__/extension-id-sync-guard.test.ts +222 -0
- package/src/__tests__/fixtures/mock-chrome-extension.ts +386 -0
- package/src/__tests__/gateway-only-enforcement.test.ts +206 -1
- package/src/__tests__/gateway-only-guard.test.ts +2 -0
- package/src/__tests__/gemini-provider.test.ts +66 -2
- package/src/__tests__/get-skill-detail-audit.test.ts +325 -0
- package/src/__tests__/gmail-archive-fallback.test.ts +193 -0
- package/src/__tests__/gmail-archive-gate.test.ts +246 -0
- package/src/__tests__/gmail-preferences.test.ts +117 -0
- package/src/__tests__/guardian-routing-invariants.test.ts +70 -2
- package/src/__tests__/headless-browser-interactions.test.ts +738 -359
- package/src/__tests__/headless-browser-mode.test.ts +614 -0
- package/src/__tests__/headless-browser-navigate.test.ts +528 -49
- package/src/__tests__/headless-browser-read-tools.test.ts +274 -100
- package/src/__tests__/headless-browser-snapshot.test.ts +250 -77
- package/src/__tests__/heartbeat-service.test.ts +70 -17
- package/src/__tests__/home-state-routes.test.ts +162 -0
- package/src/__tests__/host-bash-proxy.test.ts +145 -1
- package/src/__tests__/host-browser-e2e-cloud.test.ts +596 -0
- package/src/__tests__/host-browser-e2e-self-hosted-capability.test.ts +286 -0
- package/src/__tests__/host-browser-e2e-self-hosted.test.ts +374 -0
- package/src/__tests__/host-browser-event-routes.test.ts +350 -0
- package/src/__tests__/host-browser-proxy.test.ts +444 -0
- package/src/__tests__/host-browser-routes.test.ts +198 -0
- package/src/__tests__/host-browser-ws-events-e2e.test.ts +423 -0
- package/src/__tests__/host-cu-proxy.test.ts +166 -1
- package/src/__tests__/host-file-proxy.test.ts +185 -1
- package/src/__tests__/host-file-read-tool.test.ts +52 -0
- package/src/__tests__/host-proxy-interface.test.ts +165 -0
- package/src/__tests__/host-shell-tool.test.ts +1 -11
- package/src/__tests__/http-user-message-parity.test.ts +1 -0
- package/src/__tests__/identity-intro-cache.test.ts +40 -10
- package/src/__tests__/init-feature-flag-overrides.test.ts +38 -112
- package/src/__tests__/integration-status.test.ts +6 -7
- package/src/__tests__/jobs-store-upsert-debounced.test.ts +141 -0
- package/src/__tests__/list-messages-tool-merge.test.ts +37 -12
- package/src/__tests__/llm-context-normalization.test.ts +488 -0
- package/src/__tests__/llm-context-route-provider.test.ts +86 -5
- package/src/__tests__/llm-usage-store.test.ts +363 -0
- package/src/__tests__/mcp-client-auth.test.ts +40 -4
- package/src/__tests__/mcp-health-check.test.ts +10 -3
- package/src/__tests__/media-stream-output.test.ts +555 -0
- package/src/__tests__/media-stream-parser.test.ts +374 -0
- package/src/__tests__/media-stream-server-integration.test.ts +1234 -0
- package/src/__tests__/media-stream-stt-session.test.ts +588 -0
- package/src/__tests__/media-turn-detector.test.ts +440 -0
- package/src/__tests__/message-queue.test.ts +125 -0
- package/src/__tests__/migration-cross-version-compatibility.test.ts +3 -1
- package/src/__tests__/migration-export-http.test.ts +67 -8
- package/src/__tests__/migration-export-streaming.test.ts +66 -0
- package/src/__tests__/migration-import-commit-http.test.ts +109 -7
- package/src/__tests__/migration-import-preflight-http.test.ts +6 -5
- package/src/__tests__/migration-validate-http.test.ts +3 -3
- package/src/__tests__/mock-gateway-ipc.ts +151 -0
- package/src/__tests__/model-intents.test.ts +2 -2
- package/src/__tests__/native-host-marker-sync-guard.test.ts +157 -0
- package/src/__tests__/oauth-apps-routes.test.ts +18 -12
- package/src/__tests__/oauth-cli.test.ts +709 -60
- package/src/__tests__/oauth-connect-orchestrator.test.ts +118 -24
- package/src/__tests__/oauth-provider-seed-logos.test.ts +23 -0
- package/src/__tests__/oauth-provider-serializer.test.ts +147 -10
- package/src/__tests__/oauth-provider-visibility.test.ts +19 -21
- package/src/__tests__/oauth-providers-routes.test.ts +52 -14
- package/src/__tests__/oauth-store.test.ts +1465 -176
- package/src/__tests__/oauth2-gateway-transport.test.ts +460 -26
- package/src/__tests__/onboarding-template-contract.test.ts +81 -70
- package/src/__tests__/openai-provider.test.ts +178 -2
- package/src/__tests__/openai-responses-cutover-guard.test.ts +184 -0
- package/src/__tests__/openai-responses-provider.test.ts +1105 -0
- package/src/__tests__/openrouter-token-estimation.test.ts +100 -0
- package/src/__tests__/outlook-categories.test.ts +1 -1
- package/src/__tests__/outlook-client-automation.test.ts +1 -1
- package/src/__tests__/outlook-compose-tools.test.ts +1 -1
- package/src/__tests__/outlook-email-watcher.test.ts +1 -1
- package/src/__tests__/outlook-follow-up.test.ts +1 -1
- package/src/__tests__/outlook-messaging-provider.test.ts +2 -2
- package/src/__tests__/outlook-trash.test.ts +1 -1
- package/src/__tests__/outlook-unsubscribe.test.ts +32 -3
- package/src/__tests__/permission-checker-host-gate.test.ts +74 -14
- package/src/__tests__/permission-mode.test.ts +28 -56
- package/src/__tests__/persona-resolver.test.ts +251 -0
- package/src/__tests__/platform-bash-auto-approve.test.ts +4 -0
- package/src/__tests__/platform-callback-registration.test.ts +19 -0
- package/src/__tests__/platform.test.ts +92 -1
- package/src/__tests__/post-turn-tool-result-truncation.test.ts +343 -0
- package/src/__tests__/prechat-onboarding-contract.test.ts +267 -0
- package/src/__tests__/pricing.test.ts +174 -0
- package/src/__tests__/proxy-approval-callback.test.ts +18 -0
- package/src/__tests__/qdrant-manager.test.ts +29 -8
- package/src/__tests__/regenerate-fire-and-forget-trace.test.ts +194 -0
- package/src/__tests__/relationship-state-contract.test.ts +175 -0
- package/src/__tests__/relay-server.test.ts +423 -5
- package/src/__tests__/require-fresh-approval.test.ts +40 -1
- package/src/__tests__/sanitize-config-for-transfer.test.ts +132 -0
- package/src/__tests__/schedule-routes.test.ts +162 -0
- package/src/__tests__/search-skills-unified.test.ts +118 -0
- package/src/__tests__/secret-detection-handler.test.ts +84 -0
- package/src/__tests__/secret-ingress-http.test.ts +1 -0
- package/src/__tests__/secret-scanner-executor.test.ts +4 -0
- package/src/__tests__/secure-keys.test.ts +107 -0
- package/src/__tests__/send-endpoint-busy.test.ts +8 -1
- package/src/__tests__/sequence-store.test.ts +1 -1
- package/src/__tests__/server-history-render.test.ts +49 -0
- package/src/__tests__/set-permission-mode.test.ts +13 -250
- package/src/__tests__/settings-routes.test.ts +201 -0
- package/src/__tests__/skill-load-feature-flag.test.ts +1 -0
- package/src/__tests__/skills-file-content-endpoint.test.ts +801 -0
- package/src/__tests__/skills-files-catalog-fallback.test.ts +738 -0
- package/src/__tests__/skills.test.ts +5 -2
- package/src/__tests__/skillssh-files.test.ts +446 -0
- package/src/__tests__/slack-block-formatting.test.ts +110 -0
- package/src/__tests__/slack-channel-config.test.ts +576 -16
- package/src/__tests__/stt-catalog-parity.test.ts +282 -0
- package/src/__tests__/stt-stream-session.test.ts +535 -0
- package/src/__tests__/subagent-detail.test.ts +44 -2
- package/src/__tests__/subagent-disposal.test.ts +1 -0
- package/src/__tests__/subagent-fork-notifications.test.ts +291 -0
- package/src/__tests__/subagent-fork-spawn.test.ts +384 -0
- package/src/__tests__/subagent-manager-notify.test.ts +1 -0
- package/src/__tests__/subagent-notify-parent.test.ts +1 -0
- package/src/__tests__/subagent-spawn-tool-fork.test.ts +411 -0
- package/src/__tests__/subagent-tools.test.ts +1 -0
- package/src/__tests__/subagent-types.test.ts +1 -0
- package/src/__tests__/system-prompt-ask-mode.test.ts +27 -71
- package/src/__tests__/system-prompt.test.ts +184 -27
- package/src/__tests__/task-scheduler.test.ts +32 -6
- package/src/__tests__/telegram-config.test.ts +10 -13
- package/src/__tests__/telephony-stt-routing.test.ts +329 -0
- package/src/__tests__/terminal-tools.test.ts +25 -5
- package/src/__tests__/test-preload.ts +18 -0
- package/src/__tests__/test-support/browser-skill-harness.ts +4 -1
- package/src/__tests__/tool-approval-handler.test.ts +73 -0
- package/src/__tests__/tool-executor-lifecycle-events.test.ts +9 -5
- package/src/__tests__/tool-executor-shell-integration.test.ts +4 -0
- package/src/__tests__/tool-executor.test.ts +33 -24
- package/src/__tests__/tool-result-truncation.test.ts +36 -0
- package/src/__tests__/tool-side-effects-slack-dm.test.ts +22 -0
- package/src/__tests__/top-level-renderer.test.ts +73 -1
- package/src/__tests__/transport-hints-queue.test.ts +14 -29
- package/src/__tests__/trust-store.test.ts +7 -1
- package/src/__tests__/trusted-contact-approval-notifier.test.ts +1 -1
- package/src/__tests__/trusted-contact-inline-approval-integration.test.ts +109 -0
- package/src/__tests__/tts-catalog-parity.test.ts +345 -0
- package/src/__tests__/twilio-routes-twiml.test.ts +512 -114
- package/src/__tests__/twilio-routes.test.ts +376 -0
- package/src/__tests__/unicode.test.ts +293 -0
- package/src/__tests__/update-bulletin-format.test.ts +59 -0
- package/src/__tests__/update-bulletin.test.ts +206 -5
- package/src/__tests__/usage-routes.test.ts +25 -4
- package/src/__tests__/user-reference.test.ts +46 -61
- package/src/__tests__/v2-consent-policy.test.ts +103 -0
- package/src/__tests__/verification-control-plane-policy.test.ts +4 -0
- package/src/__tests__/voice-config-update.test.ts +403 -0
- package/src/__tests__/voice-quality.test.ts +434 -19
- package/src/__tests__/workspace-heartbeat-service.test.ts +7 -0
- package/src/__tests__/workspace-migration-033-stt-service-explicit-config.test.ts +547 -0
- package/src/__tests__/workspace-migration-034-remove-calls-voice-transcription-provider.test.ts +596 -0
- package/src/__tests__/workspace-migration-drop-user-md.test.ts +368 -0
- package/src/__tests__/workspace-migration-meets.test.ts +244 -0
- package/src/__tests__/workspace-migration-seed-device-id.test.ts +14 -20
- package/src/__tests__/workspace-policy.test.ts +2 -0
- package/src/acp/client-handler.ts +30 -4
- package/src/agent/image-optimize.ts +24 -12
- package/src/agent/loop.ts +55 -9
- package/src/approvals/guardian-request-resolvers.ts +21 -15
- package/src/backup/__tests__/backup-key.test.ts +152 -0
- package/src/backup/__tests__/backup-worker.test.ts +767 -0
- package/src/backup/__tests__/list-snapshots.test.ts +87 -0
- package/src/backup/__tests__/local-writer.test.ts +218 -0
- package/src/backup/__tests__/offsite-writer.test.ts +641 -0
- package/src/backup/__tests__/paths.test.ts +300 -0
- package/src/backup/__tests__/restore.test.ts +498 -0
- package/src/backup/__tests__/snapshot-lock.test.ts +352 -0
- package/src/backup/__tests__/stream-crypt.test.ts +228 -0
- package/src/backup/backup-key.ts +137 -0
- package/src/backup/backup-worker.ts +459 -0
- package/src/backup/list-snapshots.ts +147 -0
- package/src/backup/local-writer.ts +133 -0
- package/src/backup/offsite-writer.ts +222 -0
- package/src/backup/paths.ts +226 -0
- package/src/backup/restore.ts +322 -0
- package/src/backup/snapshot-lock.ts +431 -0
- package/src/backup/stream-crypt.ts +263 -0
- package/src/browser-session/__tests__/manager.test.ts +297 -0
- package/src/browser-session/backends/cdp-inspect.ts +30 -0
- package/src/browser-session/backends/extension.ts +26 -0
- package/src/browser-session/backends/local.ts +24 -0
- package/src/browser-session/events.ts +164 -0
- package/src/browser-session/index.ts +27 -0
- package/src/browser-session/manager.ts +159 -0
- package/src/browser-session/types.ts +28 -0
- package/src/bundler/package-resolver.ts +4 -0
- package/src/calls/audio-store.ts +11 -5
- package/src/calls/call-controller.ts +226 -71
- package/src/calls/call-domain.ts +9 -0
- package/src/calls/call-speech-output.ts +190 -0
- package/src/calls/call-transport.ts +77 -0
- package/src/calls/media-stream-audio-transcode.ts +173 -0
- package/src/calls/media-stream-output.ts +660 -0
- package/src/calls/media-stream-parser.ts +300 -0
- package/src/calls/media-stream-protocol.ts +166 -0
- package/src/calls/media-stream-server.ts +592 -0
- package/src/calls/media-stream-stt-session.ts +460 -0
- package/src/calls/media-turn-detector.ts +230 -0
- package/src/calls/relay-server.ts +90 -75
- package/src/calls/resolve-call-tts-provider.ts +136 -0
- package/src/calls/telephony-stt-routing.ts +145 -0
- package/src/calls/tts-call-strategy.ts +161 -0
- package/src/calls/tts-text-sanitizer.ts +32 -16
- package/src/calls/twilio-routes.ts +281 -17
- package/src/calls/voice-quality.ts +78 -35
- package/src/calls/voice-session-bridge.ts +8 -1
- package/src/channels/__tests__/types.test.ts +134 -0
- package/src/channels/types.ts +69 -3
- package/src/cli/__tests__/run-assistant-command.ts +11 -1
- package/src/cli/commands/__tests__/backup.test.ts +1165 -0
- package/src/cli/commands/__tests__/domain-register.test.ts +234 -0
- package/src/cli/commands/__tests__/domain-status.test.ts +132 -0
- package/src/cli/commands/__tests__/email-attachment.test.ts +422 -0
- package/src/cli/commands/__tests__/email-download.test.ts +16 -1
- package/src/cli/commands/__tests__/email-list.test.ts +22 -4
- package/src/cli/commands/__tests__/email-register.test.ts +4 -4
- package/src/cli/commands/__tests__/email-send.test.ts +37 -4
- package/src/cli/commands/__tests__/email-status.test.ts +5 -1
- package/src/cli/commands/__tests__/email-unregister.test.ts +34 -5
- package/src/cli/commands/backup.ts +993 -0
- package/src/cli/commands/conversations.ts +77 -0
- package/src/cli/commands/credentials.ts +3 -4
- package/src/cli/commands/domain.ts +210 -0
- package/src/cli/commands/email.ts +273 -16
- package/src/cli/commands/mcp.ts +16 -4
- package/src/cli/commands/oauth/__tests__/connect.test.ts +56 -44
- package/src/cli/commands/oauth/__tests__/disconnect.test.ts +21 -21
- package/src/cli/commands/oauth/__tests__/mode.test.ts +17 -17
- package/src/cli/commands/oauth/__tests__/ping.test.ts +16 -16
- package/src/cli/commands/oauth/__tests__/providers-delete.test.ts +32 -33
- package/src/cli/commands/oauth/__tests__/providers-register.test.ts +330 -0
- package/src/cli/commands/oauth/__tests__/providers-update.test.ts +117 -12
- package/src/cli/commands/oauth/__tests__/status.test.ts +10 -10
- package/src/cli/commands/oauth/__tests__/token.test.ts +7 -7
- package/src/cli/commands/oauth/apps.ts +7 -4
- package/src/cli/commands/oauth/connect.ts +6 -3
- package/src/cli/commands/oauth/disconnect.ts +1 -1
- package/src/cli/commands/oauth/mode.ts +12 -3
- package/src/cli/commands/oauth/providers.ts +215 -36
- package/src/cli/commands/oauth/shared.ts +7 -6
- package/src/cli/commands/platform/__tests__/callback-routes-list.test.ts +254 -0
- package/src/cli/commands/platform/__tests__/connect.test.ts +6 -0
- package/src/cli/commands/platform/__tests__/disconnect.test.ts +7 -1
- package/src/cli/commands/platform/__tests__/status.test.ts +6 -0
- package/src/cli/commands/platform/index.ts +107 -10
- package/src/cli/commands/usage.ts +10 -9
- package/src/cli/lib/daemon-credential-client.ts +4 -0
- package/src/cli/program.ts +30 -4
- package/src/config/__tests__/backup-schema.test.ts +134 -0
- package/src/config/assistant-feature-flags.ts +61 -62
- package/src/config/bundled-skills/app-builder/SKILL.md +26 -249
- package/src/config/bundled-skills/app-builder/references/CUSTOM_ROUTES.md +141 -0
- package/src/config/bundled-skills/app-builder/references/INTERACTION_HOOKS.md +56 -0
- package/src/config/bundled-skills/app-builder/references/WIDGETS.md +125 -0
- package/src/config/bundled-skills/browser/SKILL.md +30 -5
- package/src/config/bundled-skills/browser/TOOLS.json +123 -0
- package/src/config/bundled-skills/browser/tools/browser-attach.ts +12 -0
- package/src/config/bundled-skills/browser/tools/browser-detach.ts +12 -0
- package/src/config/bundled-skills/browser/tools/browser-status.ts +12 -0
- package/src/config/bundled-skills/browser/tools/browser-wait-for-download.ts +17 -0
- package/src/config/bundled-skills/contacts/SKILL.md +5 -2
- package/src/config/bundled-skills/document/SKILL.md +4 -0
- package/src/config/bundled-skills/gmail/SKILL.md +54 -8
- package/src/config/bundled-skills/gmail/TOOLS.json +33 -3
- package/src/config/bundled-skills/gmail/tools/gmail-archive.ts +116 -9
- package/src/config/bundled-skills/gmail/tools/gmail-outreach-scan.ts +138 -11
- package/src/config/bundled-skills/gmail/tools/gmail-preferences-tool.ts +59 -0
- package/src/config/bundled-skills/gmail/tools/gmail-preferences.ts +82 -0
- package/src/config/bundled-skills/gmail/tools/gmail-sender-digest.ts +113 -17
- package/src/config/bundled-skills/gmail/tools/gmail-unsubscribe.ts +2 -2
- package/src/config/bundled-skills/media-processing/SKILL.md +3 -9
- package/src/config/bundled-skills/media-processing/TOOLS.json +1 -6
- package/src/config/bundled-skills/media-processing/__tests__/audio-transcribe.test.ts +125 -0
- package/src/config/bundled-skills/media-processing/__tests__/extract-keyframes.test.ts +181 -0
- package/src/config/bundled-skills/media-processing/__tests__/preprocess-audio.test.ts +141 -0
- package/src/config/bundled-skills/media-processing/services/audio-transcribe.ts +32 -87
- package/src/config/bundled-skills/media-processing/services/preprocess.ts +8 -4
- package/src/config/bundled-skills/media-processing/tools/extract-keyframes.ts +0 -10
- package/src/config/bundled-skills/messaging/SKILL.md +3 -3
- package/src/config/bundled-skills/messaging/tools/messaging-archive-by-sender.ts +2 -2
- package/src/config/bundled-skills/outlook/SKILL.md +9 -2
- package/src/config/bundled-skills/outlook/tools/outlook-unsubscribe.ts +2 -2
- package/src/config/bundled-skills/phone-calls/SKILL.md +2 -2
- package/src/config/bundled-skills/phone-calls/references/CONFIG.md +27 -18
- package/src/config/bundled-skills/phone-calls/references/TROUBLESHOOTING.md +3 -3
- package/src/config/bundled-skills/settings/TOOLS.json +3 -3
- package/src/config/bundled-skills/settings/tools/voice-config-update.ts +26 -22
- package/src/config/bundled-skills/slack/SKILL.md +1 -0
- package/src/config/bundled-skills/subagent/SKILL.md +21 -0
- package/src/config/bundled-skills/subagent/TOOLS.json +8 -4
- package/src/config/bundled-skills/tasks/SKILL.md +5 -0
- package/src/config/bundled-skills/transcribe/SKILL.md +9 -14
- package/src/config/bundled-skills/transcribe/TOOLS.json +2 -7
- package/src/config/bundled-skills/transcribe/tools/transcribe-media.test.ts +256 -0
- package/src/config/bundled-skills/transcribe/tools/transcribe-media.ts +38 -188
- package/src/config/bundled-tool-registry.ts +8 -0
- package/src/config/env-registry.ts +38 -0
- package/src/config/env.ts +49 -4
- package/src/config/feature-flag-registry.json +85 -14
- package/src/config/loader.ts +82 -13
- package/src/config/sanitize-for-transfer.ts +47 -0
- package/src/config/schema.ts +81 -15
- package/src/config/schemas/__tests__/stt.test.ts +43 -0
- package/src/config/schemas/analysis.ts +51 -0
- package/src/config/schemas/backup.ts +72 -0
- package/src/config/schemas/calls.ts +1 -26
- package/src/config/schemas/elevenlabs.ts +0 -59
- package/src/config/schemas/filing.ts +47 -7
- package/src/config/schemas/heartbeat.ts +27 -5
- package/src/config/schemas/host-browser.ts +112 -0
- package/src/config/schemas/inference.ts +1 -1
- package/src/config/schemas/memory-lifecycle.ts +14 -2
- package/src/config/schemas/memory-retrieval.ts +103 -0
- package/src/config/schemas/security.ts +0 -6
- package/src/config/schemas/services.ts +52 -0
- package/src/config/schemas/stt.ts +59 -0
- package/src/config/schemas/tts.ts +230 -0
- package/src/config/schemas/updates.ts +14 -0
- package/src/config/skills.ts +4 -0
- package/src/config/types.ts +4 -1
- package/src/contacts/contact-store.ts +56 -11
- package/src/contacts/contacts-write.ts +38 -1
- package/src/context/post-turn-tool-result-truncation.ts +177 -0
- package/src/context/tool-result-truncation.ts +2 -1
- package/src/context/window-manager.ts +61 -10
- package/src/credential-execution/approval-bridge.ts +49 -15
- package/src/credential-execution/executable-discovery.ts +12 -2
- package/src/credential-execution/process-manager.ts +33 -2
- package/src/credential-health/credential-health-service.ts +366 -0
- package/src/daemon/__tests__/conversation-lifecycle-auto-analyze.test.ts +324 -0
- package/src/daemon/__tests__/conversation-surfaces-launch.test.ts +497 -0
- package/src/daemon/__tests__/conversation-tool-setup.test.ts +195 -0
- package/src/daemon/__tests__/lifecycle-startup-ordering.test.ts +127 -0
- package/src/daemon/app-source-watcher.ts +35 -0
- package/src/daemon/config-watcher.ts +99 -5
- package/src/daemon/context-overflow-approval.ts +5 -0
- package/src/daemon/conversation-agent-loop-handlers.ts +23 -2
- package/src/daemon/conversation-agent-loop.ts +153 -42
- package/src/daemon/conversation-attachments.ts +40 -0
- package/src/daemon/conversation-error.ts +11 -0
- package/src/daemon/conversation-history.ts +40 -6
- package/src/daemon/conversation-launch.ts +220 -0
- package/src/daemon/conversation-lifecycle.ts +59 -9
- package/src/daemon/conversation-messaging.ts +37 -3
- package/src/daemon/conversation-notifiers.ts +5 -0
- package/src/daemon/conversation-process.ts +622 -13
- package/src/daemon/conversation-queue-manager.ts +24 -0
- package/src/daemon/conversation-runtime-assembly.ts +128 -36
- package/src/daemon/conversation-slash.ts +36 -0
- package/src/daemon/conversation-surfaces.ts +131 -40
- package/src/daemon/conversation-tool-setup.ts +99 -8
- package/src/daemon/conversation-usage.ts +7 -4
- package/src/daemon/conversation-workspace.ts +12 -0
- package/src/daemon/conversation.ts +292 -16
- package/src/daemon/date-context.ts +10 -10
- package/src/daemon/first-greeting.ts +3 -2
- package/src/daemon/handlers/config-slack-channel.ts +269 -94
- package/src/daemon/handlers/conversations.ts +13 -141
- package/src/daemon/handlers/shared.ts +80 -0
- package/src/daemon/handlers/skills.ts +483 -44
- package/src/daemon/host-bash-proxy.ts +48 -13
- package/src/daemon/host-browser-proxy.ts +192 -0
- package/src/daemon/host-cu-proxy.ts +36 -11
- package/src/daemon/host-file-proxy.ts +57 -9
- package/src/daemon/lifecycle.ts +179 -28
- package/src/daemon/message-protocol.ts +13 -0
- package/src/daemon/message-types/conversations.ts +89 -14
- package/src/daemon/message-types/home.ts +40 -0
- package/src/daemon/message-types/host-browser.ts +100 -0
- package/src/daemon/message-types/meet.ts +143 -0
- package/src/daemon/message-types/messages.ts +19 -5
- package/src/daemon/message-types/schedules.ts +34 -2
- package/src/daemon/message-types/skills.ts +26 -0
- package/src/daemon/message-types/subagents.ts +2 -0
- package/src/daemon/message-types/surfaces.ts +2 -0
- package/src/daemon/server.ts +439 -14
- package/src/daemon/shutdown-handlers.ts +32 -4
- package/src/daemon/shutdown-registry.ts +40 -0
- package/src/daemon/tool-side-effects.ts +15 -0
- package/src/daemon/transport-hints.ts +5 -24
- package/src/email/html-renderer.ts +76 -0
- package/src/heartbeat/heartbeat-service.ts +93 -7
- package/src/home/__tests__/assistant-feed-authoring.test.ts +156 -0
- package/src/home/__tests__/emit-feed-event.test.ts +169 -0
- package/src/home/__tests__/feed-scheduler.test.ts +194 -0
- package/src/home/__tests__/feed-types.test.ts +275 -0
- package/src/home/__tests__/feed-writer.test.ts +688 -0
- package/src/home/__tests__/phase5-exit-criteria.test.ts +212 -0
- package/src/home/__tests__/platform-gmail-digest.test.ts +222 -0
- package/src/home/__tests__/progress-formula.test.ts +213 -0
- package/src/home/__tests__/relationship-state-writer.test.ts +740 -0
- package/src/home/__tests__/rollup-producer.test.ts +398 -0
- package/src/home/assistant-feed-authoring.ts +124 -0
- package/src/home/emit-feed-event.ts +158 -0
- package/src/home/feed-scheduler.ts +247 -0
- package/src/home/feed-types.ts +181 -0
- package/src/home/feed-writer.ts +469 -0
- package/src/home/platform-gmail-digest.ts +163 -0
- package/src/home/progress-formula.ts +86 -0
- package/src/home/relationship-state-writer.ts +824 -0
- package/src/home/relationship-state.ts +143 -0
- package/src/home/rollup-producer.ts +384 -0
- package/src/hooks/runner.ts +7 -0
- package/src/inbound/platform-callback-registration.ts +30 -20
- package/src/inbound/public-ingress-urls.ts +12 -0
- package/src/instrument.ts +1 -1
- package/src/ipc/__tests__/cli-ipc.test.ts +200 -0
- package/src/ipc/cli-client.ts +151 -0
- package/src/ipc/cli-server.ts +234 -0
- package/src/ipc/gateway-client.ts +180 -0
- package/src/ipc/routes/index.ts +5 -0
- package/src/ipc/routes/wake-conversation.ts +19 -0
- package/src/mcp/client.ts +59 -24
- package/src/memory/__tests__/auto-analysis-enqueue.test.ts +356 -0
- package/src/memory/__tests__/auto-analysis-guard.test.ts +57 -0
- package/src/memory/__tests__/conversation-analyze-job.test.ts +232 -0
- package/src/memory/__tests__/find-analysis-conversation.test.ts +196 -0
- package/src/memory/app-store.ts +31 -1
- package/src/memory/attachments-store.ts +70 -0
- package/src/memory/auto-analysis-enqueue.ts +127 -0
- package/src/memory/auto-analysis-guard.ts +27 -0
- package/src/memory/cleanup-schedule-state.ts +37 -0
- package/src/memory/conversation-analyze-job.ts +73 -0
- package/src/memory/conversation-crud.ts +122 -0
- package/src/memory/conversation-disk-view.ts +7 -0
- package/src/memory/conversation-group-migration.ts +34 -2
- package/src/memory/conversation-queries.ts +6 -5
- package/src/memory/conversation-starters-cadence.ts +76 -0
- package/src/memory/conversation-title-service.ts +5 -2
- package/src/memory/db-init.ts +18 -0
- package/src/memory/db-maintenance.ts +108 -0
- package/src/memory/db.ts +1 -0
- package/src/memory/embedding-backend.test.ts +75 -0
- package/src/memory/embedding-backend.ts +131 -5
- package/src/memory/embedding-gemini.test.ts +54 -0
- package/src/memory/embedding-gemini.ts +20 -9
- package/src/memory/embedding-local.ts +176 -17
- package/src/memory/graph/consolidation.ts +10 -23
- package/src/memory/graph/conversation-graph-memory.ts +15 -0
- package/src/memory/graph/extraction-job.ts +15 -0
- package/src/memory/graph/extraction.test.ts +23 -0
- package/src/memory/graph/extraction.ts +8 -0
- package/src/memory/graph/retriever.ts +67 -40
- package/src/memory/graph/scoring.test.ts +186 -0
- package/src/memory/graph/scoring.ts +31 -1
- package/src/memory/graph/store.test.ts +7 -3
- package/src/memory/graph/store.ts +47 -12
- package/src/memory/graph/tools.ts +1 -1
- package/src/memory/group-crud.ts +6 -1
- package/src/memory/indexer.ts +95 -16
- package/src/memory/job-handlers/cleanup.ts +11 -8
- package/src/memory/job-handlers/conversation-starters.ts +16 -10
- package/src/memory/jobs-store.ts +64 -4
- package/src/memory/jobs-worker.ts +22 -9
- package/src/memory/llm-usage-store.ts +137 -60
- package/src/memory/migrations/213-oauth-providers-scope-separator.ts +13 -0
- package/src/memory/migrations/214-oauth-providers-refresh-url.ts +11 -0
- package/src/memory/migrations/215-oauth-providers-revoke.ts +14 -0
- package/src/memory/migrations/216-oauth-providers-token-auth-method.ts +30 -0
- package/src/memory/migrations/217-conversation-host-access.ts +40 -0
- package/src/memory/migrations/218-oauth-providers-logo-url.ts +11 -0
- package/src/memory/migrations/219-oauth-providers-token-exchange-body-format.ts +15 -0
- package/src/memory/migrations/220-normalize-user-file-by-principal.ts +190 -0
- package/src/memory/migrations/221-conversations-archived-at.ts +16 -0
- package/src/memory/migrations/index.ts +12 -0
- package/src/memory/migrations/registry.ts +16 -0
- package/src/memory/qdrant-manager.ts +43 -16
- package/src/memory/schema/conversations.ts +3 -0
- package/src/memory/schema/oauth.ts +21 -13
- package/src/memory/usage-buckets.ts +396 -0
- package/src/messaging/providers/gmail/client.ts +57 -6
- package/src/messaging/providers/slack/__tests__/adapter-token-routing.test.ts +282 -0
- package/src/messaging/providers/slack/adapter.ts +143 -38
- package/src/messaging/providers/slack/client.ts +16 -0
- package/src/messaging/providers/slack/types.ts +4 -0
- package/src/notifications/decision-engine.ts +3 -3
- package/src/notifications/signal.ts +5 -0
- package/src/oauth/AGENTS.md +76 -0
- package/src/oauth/__tests__/identity-verifier.test.ts +25 -19
- package/src/oauth/__tests__/seed-providers-managed.test.ts +32 -0
- package/src/oauth/byo-connection.test.ts +26 -9
- package/src/oauth/byo-connection.ts +10 -8
- package/src/oauth/connect-orchestrator.ts +25 -21
- package/src/oauth/connect-types.ts +3 -3
- package/src/oauth/connection-resolver.test.ts +17 -4
- package/src/oauth/connection-resolver.ts +22 -18
- package/src/oauth/connection.ts +3 -1
- package/src/oauth/manual-token-connection.ts +13 -13
- package/src/oauth/oauth-store.ts +223 -100
- package/src/oauth/platform-connection.test.ts +101 -3
- package/src/oauth/platform-connection.ts +56 -35
- package/src/oauth/provider-serializer.ts +31 -5
- package/src/oauth/revoke.ts +76 -0
- package/src/oauth/seed-providers.ts +133 -87
- package/src/oauth/token-persistence.ts +1 -1
- package/src/permissions/checker.ts +16 -6
- package/src/permissions/defaults.ts +49 -1
- package/src/permissions/permission-mode.ts +4 -11
- package/src/permissions/prompter.ts +13 -1
- package/src/permissions/trust-store.ts +3 -3
- package/src/permissions/v2-consent-policy.ts +87 -0
- package/src/permissions/workspace-policy.ts +3 -0
- package/src/platform/client.test.ts +10 -0
- package/src/platform/sync-identity.ts +129 -0
- package/src/prompts/persona-resolver.ts +126 -2
- package/src/prompts/system-prompt.ts +76 -38
- package/src/prompts/templates/BOOTSTRAP-REFERENCE.md +3 -65
- package/src/prompts/templates/BOOTSTRAP.md +59 -105
- package/src/prompts/templates/SOUL.md +3 -1
- package/src/prompts/templates/UPDATES.md +12 -0
- package/src/prompts/templates/channels/slack.md +20 -0
- package/src/prompts/update-bulletin-format.ts +26 -9
- package/src/prompts/update-bulletin.ts +34 -23
- package/src/prompts/user-reference.ts +20 -17
- package/src/providers/__tests__/provider-secret-catalog.test.ts +42 -0
- package/src/providers/anthropic/client.ts +157 -60
- package/src/providers/fireworks/client.ts +2 -2
- package/src/providers/gemini/client.ts +9 -1
- package/src/providers/model-catalog.ts +6 -0
- package/src/providers/model-intents.ts +4 -4
- package/src/providers/ollama/client.ts +2 -2
- package/src/providers/openai/chat-completions-provider.ts +474 -0
- package/src/providers/openai/client.ts +25 -440
- package/src/providers/openai/responses-provider.ts +502 -0
- package/src/providers/openrouter/client.ts +101 -4
- package/src/providers/provider-secret-catalog.ts +139 -0
- package/src/providers/registry.ts +2 -2
- package/src/providers/retry.ts +14 -3
- package/src/providers/speech-to-text/__tests__/provider-catalog.test.ts +251 -0
- package/src/providers/speech-to-text/__tests__/resolve.test.ts +828 -0
- package/src/providers/speech-to-text/deepgram-realtime.test.ts +980 -0
- package/src/providers/speech-to-text/deepgram-realtime.ts +767 -0
- package/src/providers/speech-to-text/deepgram.test.ts +332 -0
- package/src/providers/speech-to-text/deepgram.ts +115 -0
- package/src/providers/speech-to-text/google-gemini-live-stream.test.ts +743 -0
- package/src/providers/speech-to-text/google-gemini-live-stream.ts +625 -0
- package/src/providers/speech-to-text/google-gemini.test.ts +226 -0
- package/src/providers/speech-to-text/google-gemini.ts +101 -0
- package/src/providers/speech-to-text/openai-whisper-stream.test.ts +564 -0
- package/src/providers/speech-to-text/openai-whisper-stream.ts +381 -0
- package/src/providers/speech-to-text/openai-whisper.test.ts +1 -37
- package/src/providers/speech-to-text/openai-whisper.ts +63 -33
- package/src/providers/speech-to-text/provider-catalog.ts +306 -0
- package/src/providers/speech-to-text/resolve.ts +386 -6
- package/src/providers/types.ts +10 -1
- package/src/runtime/AGENTS.md +65 -0
- package/src/runtime/__tests__/agent-wake.test.ts +831 -0
- package/src/runtime/__tests__/browser-extension-pair-routes.test.ts +715 -0
- package/src/runtime/__tests__/capability-tokens.test.ts +258 -0
- package/src/runtime/__tests__/chrome-extension-registry.test.ts +518 -0
- package/src/runtime/__tests__/runtime-mode.test.ts +62 -0
- package/src/runtime/__tests__/slack-block-formatting.test.ts +481 -0
- package/src/runtime/agent-wake.ts +512 -0
- package/src/runtime/assistant-event-hub.ts +2 -2
- package/src/runtime/auth/__tests__/guard-tests.test.ts +1 -0
- package/src/runtime/auth/__tests__/middleware.test.ts +116 -1
- package/src/runtime/auth/__tests__/route-policy.test.ts +48 -0
- package/src/runtime/auth/middleware.ts +98 -0
- package/src/runtime/auth/route-policy.ts +33 -9
- package/src/runtime/auth/token-service.ts +56 -1
- package/src/runtime/btw-sidechain.ts +2 -0
- package/src/runtime/capability-tokens.ts +414 -0
- package/src/runtime/channel-approvals.ts +18 -5
- package/src/runtime/channel-invite-transport.ts +1 -1
- package/src/runtime/channel-invite-transports/email.ts +14 -6
- package/src/runtime/channel-readiness-service.ts +12 -22
- package/src/runtime/chrome-extension-registry.ts +368 -0
- package/src/runtime/confirmation-request-guardian-bridge.ts +6 -0
- package/src/runtime/guardian-decision-types.ts +7 -0
- package/src/runtime/http-server.ts +815 -75
- package/src/runtime/http-types.ts +6 -2
- package/src/runtime/migrations/__tests__/rebind-secrets-credentials.test.ts +172 -0
- package/src/runtime/migrations/__tests__/vbundle-builder-credentials.test.ts +276 -0
- package/src/runtime/migrations/__tests__/vbundle-import-credentials.test.ts +198 -0
- package/src/runtime/migrations/__tests__/vbundle-legacy-user-md.test.ts +360 -0
- package/src/runtime/migrations/migration-transport.ts +7 -0
- package/src/runtime/migrations/migration-wizard.ts +23 -2
- package/src/runtime/migrations/rebind-secrets-screen.ts +76 -15
- package/src/runtime/migrations/vbundle-builder.ts +145 -38
- package/src/runtime/migrations/vbundle-import-analyzer.ts +96 -1
- package/src/runtime/migrations/vbundle-importer.ts +89 -5
- package/src/runtime/pending-interactions.ts +18 -13
- package/src/runtime/routes/__tests__/backup-routes.test.ts +967 -0
- package/src/runtime/routes/__tests__/home-feed-routes.test.ts +507 -0
- package/src/runtime/routes/__tests__/migration-import-credential-filter.test.ts +208 -0
- package/src/runtime/routes/__tests__/stt-routes.test.ts +406 -0
- package/src/runtime/routes/__tests__/tts-routes.test.ts +474 -0
- package/src/runtime/routes/__tests__/user-route-dispatcher.test.ts +148 -17
- package/src/runtime/routes/app-management-routes.ts +12 -18
- package/src/runtime/routes/approval-routes.ts +90 -16
- package/src/runtime/routes/attachment-routes.test.ts +9 -3
- package/src/runtime/routes/attachment-routes.ts +216 -17
- package/src/runtime/routes/backup-routes.ts +519 -0
- package/src/runtime/routes/browser-extension-pair-routes.ts +556 -0
- package/src/runtime/routes/btw-routes.ts +8 -6
- package/src/runtime/routes/contact-routes.test.ts +298 -0
- package/src/runtime/routes/contact-routes.ts +132 -5
- package/src/runtime/routes/conversation-analysis-routes.ts +22 -141
- package/src/runtime/routes/conversation-management-routes.ts +223 -0
- package/src/runtime/routes/conversation-routes.ts +598 -103
- package/src/runtime/routes/conversation-starter-routes.ts +78 -16
- package/src/runtime/routes/filing-routes.ts +93 -0
- package/src/runtime/routes/guardian-action-routes.ts +24 -13
- package/src/runtime/routes/home-feed-routes.ts +334 -0
- package/src/runtime/routes/home-state-routes.ts +138 -0
- package/src/runtime/routes/host-browser-routes.ts +268 -0
- package/src/runtime/routes/host-file-routes.ts +9 -1
- package/src/runtime/routes/identity-intro-cache.ts +7 -3
- package/src/runtime/routes/identity-routes.ts +262 -33
- package/src/runtime/routes/inbound-stages/transcribe-audio.test.ts +46 -39
- package/src/runtime/routes/inbound-stages/transcribe-audio.ts +15 -15
- package/src/runtime/routes/integrations/slack/__tests__/channel.test.ts +137 -0
- package/src/runtime/routes/integrations/slack/__tests__/share.test.ts +179 -0
- package/src/runtime/routes/integrations/slack/channel.ts +11 -3
- package/src/runtime/routes/integrations/slack/share.ts +45 -7
- package/src/runtime/routes/llm-context-normalization.ts +303 -0
- package/src/runtime/routes/log-export-routes.ts +42 -22
- package/src/runtime/routes/memory-item-routes.test.ts +3 -2
- package/src/runtime/routes/memory-item-routes.ts +1 -7
- package/src/runtime/routes/migration-routes.ts +122 -2
- package/src/runtime/routes/oauth-apps.ts +15 -17
- package/src/runtime/routes/oauth-providers.ts +4 -0
- package/src/runtime/routes/schedule-routes.ts +24 -11
- package/src/runtime/routes/settings-routes.ts +31 -102
- package/src/runtime/routes/skills-routes.ts +128 -9
- package/src/runtime/routes/stt-routes.ts +233 -0
- package/src/runtime/routes/subagents-routes.ts +14 -10
- package/src/runtime/routes/surface-action-routes.ts +41 -2
- package/src/runtime/routes/tts-routes.ts +108 -24
- package/src/runtime/routes/usage-routes.ts +38 -9
- package/src/runtime/routes/user-route-dispatcher.ts +50 -5
- package/src/runtime/routes/user-routes.ts +13 -1
- package/src/runtime/routes/work-items-routes.ts +8 -1
- package/src/runtime/routes/workspace-routes.test.ts +22 -0
- package/src/runtime/routes/workspace-routes.ts +8 -1
- package/src/runtime/routes/workspace-utils.ts +2 -0
- package/src/runtime/runtime-mode.ts +33 -0
- package/src/runtime/services/__tests__/analyze-conversation.test.ts +444 -0
- package/src/runtime/services/__tests__/analyze-deps-singleton.test.ts +67 -0
- package/src/runtime/services/__tests__/auto-analysis-prompt.test.ts +53 -0
- package/src/runtime/services/__tests__/manual-analysis-prompt.test.ts +41 -0
- package/src/runtime/services/analyze-conversation.ts +344 -0
- package/src/runtime/services/analyze-deps-singleton.ts +32 -0
- package/src/runtime/services/auto-analysis-prompt.ts +55 -0
- package/src/runtime/skill-route-registry.ts +49 -0
- package/src/runtime/slack-block-formatting.ts +437 -10
- package/src/schedule/scheduler.ts +57 -5
- package/src/security/ces-credential-client.ts +20 -0
- package/src/security/ces-rpc-credential-backend.ts +17 -0
- package/src/security/credential-backend.ts +5 -0
- package/src/security/oauth2.ts +68 -29
- package/src/security/secure-keys.ts +143 -27
- package/src/security/token-manager.ts +31 -10
- package/src/sequence/engine.ts +23 -0
- package/src/sequence/types.ts +1 -1
- package/src/skills/catalog-files.ts +554 -0
- package/src/skills/category-inference.ts +122 -0
- package/src/skills/clawhub-files.ts +213 -0
- package/src/skills/clawhub.ts +84 -23
- package/src/skills/skill-file-provider.ts +40 -0
- package/src/skills/skillssh-files.ts +395 -0
- package/src/skills/skillssh-registry.ts +4 -4
- package/src/stt/__tests__/daemon-batch-transcriber.test.ts +392 -0
- package/src/stt/__tests__/types.test.ts +89 -0
- package/src/stt/daemon-batch-transcriber.ts +195 -0
- package/src/stt/stt-stream-session.ts +499 -0
- package/src/stt/types.ts +330 -0
- package/src/stt/wav-encoder.test.ts +373 -0
- package/src/stt/wav-encoder.ts +175 -0
- package/src/subagent/manager.ts +169 -40
- package/src/subagent/types.ts +19 -0
- package/src/tools/apps/executors.ts +11 -2
- package/src/tools/browser/__tests__/auth-detector.test.ts +202 -108
- package/src/tools/browser/__tests__/browser-mode.test.ts +119 -0
- package/src/tools/browser/__tests__/browser-status.test.ts +123 -0
- package/src/tools/browser/auth-detector.ts +43 -12
- package/src/tools/browser/browser-execution.ts +1787 -342
- package/src/tools/browser/browser-manager.ts +81 -12
- package/src/tools/browser/browser-mode-constants.ts +12 -0
- package/src/tools/browser/browser-mode.ts +92 -0
- package/src/tools/browser/browser-status-constants.ts +33 -0
- package/src/tools/browser/cdp-client/__tests__/accessibility-snapshot.test.ts +318 -0
- package/src/tools/browser/cdp-client/__tests__/cdp-dom-helpers.test.ts +1175 -0
- package/src/tools/browser/cdp-client/__tests__/cdp-inspect-client.test.ts +1263 -0
- package/src/tools/browser/cdp-client/__tests__/extension-cdp-client.test.ts +359 -0
- package/src/tools/browser/cdp-client/__tests__/factory.test.ts +1993 -0
- package/src/tools/browser/cdp-client/__tests__/fixtures/ax-tree-nested-frames.json +64 -0
- package/src/tools/browser/cdp-client/__tests__/fixtures/ax-tree-simple.json +69 -0
- package/src/tools/browser/cdp-client/__tests__/local-cdp-client.test.ts +310 -0
- package/src/tools/browser/cdp-client/__tests__/types.test.ts +96 -0
- package/src/tools/browser/cdp-client/accessibility-snapshot.ts +387 -0
- package/src/tools/browser/cdp-client/cdp-dom-helpers.ts +695 -0
- package/src/tools/browser/cdp-client/cdp-inspect/__tests__/discovery.test.ts +1007 -0
- package/src/tools/browser/cdp-client/cdp-inspect/__tests__/ws-transport.test.ts +580 -0
- package/src/tools/browser/cdp-client/cdp-inspect/discovery.ts +744 -0
- package/src/tools/browser/cdp-client/cdp-inspect/ws-transport.ts +579 -0
- package/src/tools/browser/cdp-client/cdp-inspect-client.ts +868 -0
- package/src/tools/browser/cdp-client/errors.ts +49 -0
- package/src/tools/browser/cdp-client/extension-cdp-client.ts +148 -0
- package/src/tools/browser/cdp-client/factory.ts +914 -0
- package/src/tools/browser/cdp-client/index.ts +28 -0
- package/src/tools/browser/cdp-client/local-cdp-client.ts +187 -0
- package/src/tools/browser/cdp-client/types.ts +120 -0
- package/src/tools/credentials/vault.ts +35 -6
- package/src/tools/filesystem/edit.ts +1 -1
- package/src/tools/filesystem/list.ts +1 -1
- package/src/tools/filesystem/read.ts +1 -1
- package/src/tools/filesystem/write.ts +2 -1
- package/src/tools/host-filesystem/edit.ts +1 -1
- package/src/tools/host-filesystem/read.ts +12 -15
- package/src/tools/host-filesystem/write.ts +1 -1
- package/src/tools/host-terminal/host-shell.ts +21 -16
- package/src/tools/network/web-fetch.ts +5 -2
- package/src/tools/network/web-search.ts +5 -2
- package/src/tools/permission-checker.ts +77 -82
- package/src/tools/registry.ts +0 -2
- package/src/tools/secret-detection-handler.ts +34 -0
- package/src/tools/shared/filesystem/image-read.ts +61 -40
- package/src/tools/shared/shell-output.ts +3 -1
- package/src/tools/side-effects.ts +2 -0
- package/src/tools/skills/sandbox-runner.ts +3 -2
- package/src/tools/subagent/spawn.ts +47 -3
- package/src/tools/subagent/status.ts +2 -0
- package/src/tools/system/register.ts +2 -16
- package/src/tools/terminal/safe-env.ts +15 -0
- package/src/tools/terminal/shell.ts +36 -20
- package/src/tools/tool-approval-handler.ts +48 -2
- package/src/tools/tool-manifest.ts +21 -0
- package/src/tools/types.ts +19 -0
- package/src/tools/ui-surface/definitions.ts +6 -1
- package/src/tts/__tests__/provider-adapters.test.ts +834 -0
- package/src/tts/__tests__/provider-catalog-consistency.test.ts +196 -0
- package/src/tts/__tests__/provider-catalog.test.ts +183 -0
- package/src/tts/__tests__/provider-registry.test.ts +90 -0
- package/src/tts/provider-catalog.ts +201 -0
- package/src/tts/provider-registry.ts +73 -0
- package/src/tts/providers/deepgram-provider.ts +219 -0
- package/src/tts/providers/elevenlabs-provider.ts +211 -0
- package/src/tts/providers/fish-audio-provider.ts +183 -0
- package/src/tts/providers/index.ts +42 -0
- package/src/tts/providers/register-builtins.ts +130 -0
- package/src/tts/synthesize-text.ts +110 -0
- package/src/tts/tts-config-resolver.ts +78 -0
- package/src/tts/types.ts +153 -0
- package/src/types/onboarding-context.ts +7 -0
- package/src/util/abort-reasons.ts +58 -0
- package/src/util/device-id.ts +32 -16
- package/src/util/errors.ts +9 -1
- package/src/util/platform.ts +63 -24
- package/src/util/pricing.ts +66 -3
- package/src/util/spawn.ts +1 -1
- package/src/util/truncate.ts +4 -2
- package/src/util/unicode.ts +201 -0
- package/src/version.ts +19 -24
- package/src/watcher/engine.ts +23 -0
- package/src/watcher/watcher-store.ts +31 -0
- package/src/workspace/migrations/003-seed-device-id.ts +9 -3
- package/src/workspace/migrations/017-seed-persona-dirs.ts +68 -4
- package/src/workspace/migrations/029-seed-pkb.ts +1 -1
- package/src/workspace/migrations/031-drop-user-md.ts +317 -0
- package/src/workspace/migrations/031-llm-log-retention-zero-to-null.ts +73 -0
- package/src/workspace/migrations/032-tts-provider-unification.ts +227 -0
- package/src/workspace/migrations/033-stt-service-explicit-config.ts +122 -0
- package/src/workspace/migrations/034-remove-calls-voice-transcription-provider.ts +215 -0
- package/src/workspace/migrations/035-seed-slack-channel-persona.ts +50 -0
- package/src/workspace/migrations/036-update-pkb-index-bar.ts +37 -0
- package/src/workspace/migrations/037-create-meets-dir.ts +61 -0
- package/src/workspace/migrations/registry.ts +16 -0
- package/src/workspace/top-level-renderer.ts +31 -1
- package/src/workspace/turn-commit.ts +31 -0
- package/src/__tests__/chrome-cdp.test.ts +0 -419
- package/src/__tests__/email-cli.test.ts +0 -297
- package/src/__tests__/email-service-config-fallback.test.ts +0 -102
- package/src/__tests__/permission-mode-sse.test.ts +0 -418
- package/src/__tests__/permission-mode-store.test.ts +0 -277
- package/src/browser-extension-relay/protocol.ts +0 -63
- package/src/browser-extension-relay/server.ts +0 -203
- package/src/cli/commands/browser-relay.ts +0 -536
- package/src/config/schemas/sandbox.ts +0 -14
- package/src/email/guardrails.ts +0 -221
- package/src/email/provider.ts +0 -117
- package/src/email/providers/agentmail.ts +0 -361
- package/src/email/providers/index.ts +0 -65
- package/src/email/service.ts +0 -384
- package/src/email/types.ts +0 -126
- package/src/permissions/permission-mode-store.ts +0 -180
- package/src/prompts/templates/USER.md +0 -13
- package/src/providers/speech-to-text/types.ts +0 -17
- package/src/tools/browser/chrome-cdp.ts +0 -239
- package/src/tools/system/set-permission-mode.ts +0 -103
package/ARCHITECTURE.md
CHANGED
|
@@ -7,7 +7,7 @@ This document owns assistant-runtime architecture details. The repo-level archit
|
|
|
7
7
|
- Transport metadata arrives via `conversation_create.transport` (HTTP) or `/channels/inbound` (`channelId`, optional `hints`, optional `uxBrief`).
|
|
8
8
|
- Telegram webhook ingress injects deterministic channel-safe transport metadata (`hints` + `uxBrief`) so non-dashboard channels defer dashboard-only UI tasks cleanly.
|
|
9
9
|
- `OnboardingPlaybookManager` resolves `<channel>_onboarding.md`, checks `onboarding/playbooks/registry.json`, and applies per-channel first-time fast-path onboarding.
|
|
10
|
-
- `OnboardingOrchestrator` derives onboarding-mode guidance (post-hatch sequence,
|
|
10
|
+
- `OnboardingOrchestrator` derives onboarding-mode guidance (post-hatch sequence, `users/<slug>.md` persona capture) from playbook + transport context.
|
|
11
11
|
- Conversation runtime assembly injects both `<channel_onboarding_playbook>` and `<onboarding_mode>` context before provider calls, then strips both from persisted conversation history.
|
|
12
12
|
- Permission setup remains user-initiated and hatch + first-conversation flows avoid proactive permission asks.
|
|
13
13
|
|
|
@@ -333,11 +333,11 @@ The Slack channel provides text-based messaging via Slack's Socket Mode API. Unl
|
|
|
333
333
|
|
|
334
334
|
**Control-plane endpoints** (`/v1/integrations/slack/channel/config`):
|
|
335
335
|
|
|
336
|
-
| Endpoint | Method | Description
|
|
337
|
-
| --------------------------------------- | ------ |
|
|
338
|
-
| `/v1/integrations/slack/channel/config` | GET | Returns current config status: `hasBotToken`, `hasAppToken`, `connected`, plus workspace metadata (`teamId`, `teamName`, `botUserId`, `botUsername`)
|
|
339
|
-
| `/v1/integrations/slack/channel/config` | POST | Validates and stores credentials. Body: `{ botToken?: string, appToken?: string }`
|
|
340
|
-
| `/v1/integrations/slack/channel/config` | DELETE | Clears all Slack channel credentials from secure storage and credential metadata
|
|
336
|
+
| Endpoint | Method | Description |
|
|
337
|
+
| --------------------------------------- | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
338
|
+
| `/v1/integrations/slack/channel/config` | GET | Returns current config status: `hasBotToken`, `hasAppToken`, `hasUserToken`, `connected`, plus workspace metadata (`teamId`, `teamName`, `botUserId`, `botUsername`) |
|
|
339
|
+
| `/v1/integrations/slack/channel/config` | POST | Validates and stores credentials. Body: `{ botToken?: string, appToken?: string, userToken?: string }` |
|
|
340
|
+
| `/v1/integrations/slack/channel/config` | DELETE | Clears all Slack channel credentials (bot, app, and user tokens) from secure storage and credential metadata. Surgical user-token-only deletion is exposed internally via `clearSlackUserToken` (used by the credential vault) but is not reachable through this HTTP endpoint. |
|
|
341
341
|
|
|
342
342
|
All endpoints are JWT-authenticated via `Authorization: Bearer <jwt>`.
|
|
343
343
|
|
|
@@ -345,13 +345,18 @@ All endpoints are JWT-authenticated via `Authorization: Bearer <jwt>`.
|
|
|
345
345
|
|
|
346
346
|
Both tokens are stored in the secure key store (CES credential store with encrypted file fallback):
|
|
347
347
|
|
|
348
|
-
| Secure key
|
|
349
|
-
|
|
|
350
|
-
| `credential/slack_channel/bot_token`
|
|
351
|
-
| `credential/slack_channel/app_token`
|
|
348
|
+
| Secure key | Content |
|
|
349
|
+
| ------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
350
|
+
| `credential/slack_channel/bot_token` | Slack bot token (used for `chat.postMessage` and `auth.test`) |
|
|
351
|
+
| `credential/slack_channel/app_token` | Slack app token (`xapp-...`, used for Socket Mode `apps.connections.open`) |
|
|
352
|
+
| `credential/slack_channel/user_token` | Optional. Slack user OAuth token (`xoxp-...`). Enables reading channels and DMs the bot isn't a member of (for triage). Never used for writes. |
|
|
352
353
|
|
|
353
354
|
Workspace metadata (team ID, team name, bot user ID, bot username) is stored as JSON in the credential metadata store under `('slack_channel', 'bot_token')`.
|
|
354
355
|
|
|
356
|
+
**Read/write auth split:** The Slack adapter (`src/messaging/providers/slack/adapter.ts`) caches read and write auth separately. Reads (`listConversations`, `getHistory`, replies, search, `users.info`) prefer the user token when present via `getReadAuth()` — giving visibility into channels and DMs the bot hasn't been invited to. Writes (`postMessage`, `markRead`, and any future state-changing calls) always use the bot token via `getWriteAuth()` so posts come from the bot identity, never the user's. When no user token is stored, reads fall back to the bot token (unchanged legacy behavior).
|
|
357
|
+
|
|
358
|
+
**Workspace-consistency invariant:** The user token and bot token must be for the same Slack workspace. `setSlackChannelConfig` enforces this by calling `auth.test` on each token and comparing the returned `team_id`; if the user token's workspace does not match an already-stored bot token's workspace, the user token is rejected and not stored.
|
|
359
|
+
|
|
355
360
|
**Token validation via `auth.test`:**
|
|
356
361
|
|
|
357
362
|
When a bot token is provided via `POST /v1/integrations/slack/channel/config`, the handler calls `POST https://slack.com/api/auth.test` with the token before storing it. A successful response yields workspace metadata (`team_id`, `team`, `user_id`, `user`) that is persisted alongside the token. If `auth.test` fails, the token is rejected and not stored.
|
|
@@ -582,6 +587,167 @@ All guardian decisions for voice access requests flow through:
|
|
|
582
587
|
| `src/runtime/actor-trust-resolver.ts` | `resolveActorTrust` — caller trust classification |
|
|
583
588
|
| `src/memory/canonical-guardian-store.ts` | Canonical request persistence and CAS resolution |
|
|
584
589
|
|
|
590
|
+
### Speech-to-Text (STT) Boundaries
|
|
591
|
+
|
|
592
|
+
Audio-to-text conversion occurs in five distinct runtime boundaries, each with its own provider model and adapter layer. The `services.stt` config block is the single source of truth for STT provider selection across both daemon and telephony boundaries.
|
|
593
|
+
|
|
594
|
+
**Provider catalog model:** The daemon's canonical provider catalog (`src/providers/speech-to-text/provider-catalog.ts`) is the single source of truth for STT provider metadata — credential mappings, supported boundaries, telephony mode, and conversation streaming mode. The client-facing catalog (`meta/stt-provider-catalog.json`) carries display metadata (names, hints, setup mode, `conversationStreamingMode`) and is bundled into native apps at build time. A CI parity test (`src/__tests__/stt-catalog-parity.test.ts`) enforces that provider IDs, ordering, and credential-provider name mappings stay aligned between the two catalogs. To add a new provider, follow the checklist in `docs/stt-provider-onboarding.md`.
|
|
595
|
+
|
|
596
|
+
**Boundary overview:**
|
|
597
|
+
|
|
598
|
+
| Boundary | Runtime | Provider (current) | Adapter module | Caller |
|
|
599
|
+
| ---------------------------- | ----------------------------------------------------------------------------- | ---------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------- |
|
|
600
|
+
| **Telephony (hybrid)** | Twilio-native ConversationRelay or daemon media-stream (provider-conditional) | Configured STT provider (via `services.stt`) | `src/calls/telephony-stt-routing.ts` | `src/calls/twilio-routes.ts` |
|
|
601
|
+
| **Daemon batch** | Daemon process (REST API to provider) | Configured STT provider (via `services.stt`) | `src/stt/daemon-batch-transcriber.ts` | `src/runtime/routes/inbound-stages/transcribe-audio.ts` |
|
|
602
|
+
| **Conversation streaming** | Daemon process (WebSocket-based) | Deepgram or Google Gemini (via `services.stt`) | `src/stt/stt-stream-session.ts`, `src/providers/speech-to-text/deepgram-realtime.ts`, `src/providers/speech-to-text/google-gemini-live-stream.ts` | `VoiceInputManager` (macOS conversation), `InputBarView` (iOS conversation) via gateway WS proxy |
|
|
603
|
+
| **Client service-first** | macOS / iOS via gateway → daemon | Configured STT provider (via `services.stt`) | `src/runtime/routes/stt-routes.ts`, `clients/shared/Network/STTClient.swift` | `VoiceInputManager` (macOS dictation), `InputBarView` (iOS), `OpenAIVoiceService` (macOS voice mode) |
|
|
604
|
+
| **Client-native (fallback)** | macOS / iOS on-device | Apple Speech (`SFSpeechRecognizer`) | `clients/macos/.../SpeechRecognizerAdapter.swift`, `clients/ios/.../SpeechRecognizerAdapter.swift` | Fallback when STT service is unconfigured or fails |
|
|
605
|
+
|
|
606
|
+
**Telephony boundary (hybrid routing):**
|
|
607
|
+
|
|
608
|
+
Telephony STT uses a provider-conditional hybrid model driven by `services.stt.provider`. The routing resolver (`src/calls/telephony-stt-routing.ts`) maps the configured provider to a discriminated strategy at call setup time:
|
|
609
|
+
|
|
610
|
+
- **`conversation-relay-native`** (Deepgram, Google) — TwiML emits `<Connect><ConversationRelay>` with `transcriptionProvider` and `speechModel` attributes. Twilio handles audio ingestion and transcription natively; the daemon receives transcribed text via the relay WebSocket. The Twilio-native provider name and default speech model are read from the provider catalog entry's `telephonyRouting.twilioNativeMapping` (e.g. Deepgram maps to `provider: "Deepgram"` with `defaultSpeechModel: "nova-3"`; Google maps to `provider: "Google"` with `defaultSpeechModel: undefined`).
|
|
611
|
+
|
|
612
|
+
- **`media-stream-custom`** (OpenAI Whisper) — TwiML emits `<Connect><Stream>` pointing to the gateway's media-stream proxy. The `<Stream url="...">` encodes `callSessionId` and auth `token` as **URL path segments** (e.g. `.../media-stream/<callSessionId>/<token>`) because Twilio Media Streams does not reliably preserve query parameters across the WebSocket upgrade. The gateway extracts metadata from path segments (with query-parameter fallback for backward compatibility) and proxies raw audio frames to the daemon, which transcribes server-side via the provider's batch API.
|
|
613
|
+
|
|
614
|
+
Key modules:
|
|
615
|
+
|
|
616
|
+
| Module | Purpose |
|
|
617
|
+
| --------------------------------------------------- | ---------------------------------------------------------------------- |
|
|
618
|
+
| `src/calls/telephony-stt-routing.ts` | Maps `services.stt.provider` to a discriminated `TelephonySttStrategy` |
|
|
619
|
+
| `src/calls/twilio-routes.ts` | Voice webhook handler; generates provider-conditional TwiML |
|
|
620
|
+
| `src/calls/media-stream-parser.ts` | Twilio Media Streams protocol parser |
|
|
621
|
+
| `src/calls/media-turn-detector.ts` | Energy-based VAD turn detector for raw audio |
|
|
622
|
+
| `src/calls/media-stream-stt-session.ts` | STT session that transcribes audio turns via `services.stt` |
|
|
623
|
+
| `src/calls/call-transport.ts` | Transport interface decoupling CallController from wire protocol |
|
|
624
|
+
| `src/calls/media-stream-output.ts` | Output adapter for sending TTS audio back via Media Streams |
|
|
625
|
+
| `src/calls/media-stream-server.ts` | WebSocket server binding media-stream lifecycle to call sessions |
|
|
626
|
+
| `gateway/src/http/routes/twilio-media-websocket.ts` | Gateway WebSocket proxy for Media Streams frames |
|
|
627
|
+
|
|
628
|
+
Guard tests in `__tests__/twilio-routes-twiml.test.ts` and `__tests__/twilio-routes.test.ts` assert that TwiML generation matches the provider-conditional strategy for each supported provider.
|
|
629
|
+
|
|
630
|
+
To add a new telephony STT provider: add a `telephonyRouting` entry to the provider's catalog entry in `provider-catalog.ts`. Set `strategyKind` to `"conversation-relay-native"` for Twilio-native providers (and include a `twilioNativeMapping` with the Twilio `provider` name and `defaultSpeechModel`), or `"media-stream-custom"` for providers that require daemon-side transcription. The routing resolver reads these fields from the catalog — no hardcoded maps to update.
|
|
631
|
+
|
|
632
|
+
**Daemon batch boundary:**
|
|
633
|
+
|
|
634
|
+
The daemon transcribes audio attachments (e.g. voice messages from channel inbound) by calling a provider's REST API directly.
|
|
635
|
+
|
|
636
|
+
- `src/stt/types.ts` defines provider-agnostic domain types: `BatchTranscriber` interface, `SttTranscribeRequest`, `SttTranscribeResult`, `SttError` with normalized categories (`auth`, `rate-limit`, `timeout`, `invalid-audio`, `provider-error`), and `SttProviderId` / `SttBoundaryId` discriminants.
|
|
637
|
+
- `createDaemonBatchTranscriber()` in `src/stt/daemon-batch-transcriber.ts` is the factory that returns a `BatchTranscriber` backed by the configured STT provider (OpenAI Whisper or Deepgram, selected via `services.stt.provider`). Returns `null` when no API key is available for the selected provider. `normalizeSttError()` maps raw provider errors to `SttError` categories.
|
|
638
|
+
- `resolveBatchTranscriber()` in `src/providers/speech-to-text/resolve.ts` is the credential-aware entry point — it reads the configured provider from `services.stt`, resolves the corresponding API key from the secure key store, and delegates to the factory.
|
|
639
|
+
- `tryTranscribeAudioAttachments()` in `src/runtime/routes/inbound-stages/transcribe-audio.ts` is the callsite that uses the facade for channel audio attachment transcription.
|
|
640
|
+
|
|
641
|
+
To add a new daemon batch STT provider, follow the full checklist in `docs/stt-provider-onboarding.md` — it covers the daemon catalog, type registration, config schema, adapter wiring, credential plumbing, client catalog, and parity tests.
|
|
642
|
+
|
|
643
|
+
**Conversation streaming boundary:**
|
|
644
|
+
|
|
645
|
+
Real-time conversation chat message capture on macOS and iOS uses a WebSocket-based streaming STT path. When the configured `services.stt` provider supports conversation streaming (determined by the `conversationStreamingMode` field in the provider catalog), native clients open a WebSocket session through the gateway to the daemon's `/v1/stt/stream` endpoint. The daemon resolves a `StreamingTranscriber` for the configured provider and streams partial/final transcript events back to the client in real time.
|
|
646
|
+
|
|
647
|
+
Two provider adapters are supported, each implementing the `StreamingTranscriber` interface from `src/stt/types.ts`:
|
|
648
|
+
|
|
649
|
+
| Provider | Adapter | Mode | Mechanism |
|
|
650
|
+
| ----------------- | ----------------------------------------------------------- | ------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
651
|
+
| **Deepgram** | `src/providers/speech-to-text/deepgram-realtime.ts` | `realtime-ws` | Opens a WebSocket to Deepgram's `/v1/listen` endpoint, forwards raw PCM audio, normalizes Deepgram's `is_final`/`speech_final` semantics into `partial`/`final` events. Uses model `nova-2`. |
|
|
652
|
+
| **Google Gemini** | `src/providers/speech-to-text/google-gemini-live-stream.ts` | `realtime-ws` | Opens a bidirectional streaming session against Gemini's Live API (`ai.live.connect`), forwards PCM audio frames, and normalizes `serverContent.inputTranscription` events into `partial`/`final` events. Uses model `gemini-live-2.5-flash-preview`. |
|
|
653
|
+
|
|
654
|
+
**Provider-specific behavior differences:**
|
|
655
|
+
|
|
656
|
+
- **Deepgram (`realtime-ws`)**: True WebSocket streaming with sub-second partial latency. Emits `partial` events for `is_final: false` frames and `final` events for `is_final: true` frames. Supports backpressure (drops audio frames when `bufferedAmount > 1 MiB`). Sends `CloseStream` message on stop with a 5-second grace period for the provider to flush remaining finals. Inactivity timeout: 30 seconds (provider-side hang detection). Connect timeout: 10 seconds. Auth errors map to close codes 1008/4001; rate limits to 1013.
|
|
657
|
+
- **Google Gemini (`realtime-ws`)**: WebSocket-backed Live API session. Partials are emitted as Gemini streams `inputTranscription.text` fragments; a `final` is emitted when the server signals `generationComplete` or `turnComplete`. On `stop()`, the adapter sends `audioStreamEnd: true` and waits up to a 5-second grace window for the server to flush remaining transcription before force-closing. Inactivity timeout: 30 seconds. Connect timeout: 10 seconds. Close codes 1008/4001 map to `auth`; 1013 maps to `rate-limit`; other codes map to `provider-error`. The model's own text turn is suppressed via a silent system instruction so we only pay for transcription.
|
|
658
|
+
|
|
659
|
+
**Session lifecycle (daemon side):**
|
|
660
|
+
|
|
661
|
+
1. Client opens a WebSocket to `/v1/stt/stream` with required query parameter `mimeType` and optional `provider` and `sampleRate`. The `provider` parameter is optional compatibility metadata — the runtime is config-authoritative and always resolves the streaming transcriber from `services.stt.provider`. When a requested provider disagrees with the configured provider, the runtime logs a mismatch warning.
|
|
662
|
+
2. `SttStreamSession` (in `src/stt/stt-stream-session.ts`) resolves a `StreamingTranscriber` via `resolveStreamingTranscriber()` from `src/providers/speech-to-text/resolve.ts`, using the configured provider (not the requested one).
|
|
663
|
+
3. The transcriber's `start()` method opens the provider session.
|
|
664
|
+
4. A `ready` event (with `provider` field) is sent to the client, signaling that audio frames are accepted.
|
|
665
|
+
5. Client sends `audio` frames (binary WebSocket frames or base64-encoded JSON) and a `stop` event when recording ends.
|
|
666
|
+
6. The transcriber emits `partial` and `final` events, forwarded to the client as JSON frames with monotonic `seq` numbers.
|
|
667
|
+
7. The session closes deterministically on: client disconnect, `stop` event followed by provider `closed`, idle timeout (60 seconds), or runtime shutdown.
|
|
668
|
+
|
|
669
|
+
**Session lifecycle (client side):**
|
|
670
|
+
|
|
671
|
+
- `STTStreamingClient` (`clients/shared/Network/STTStreamingClient.swift`) manages the WebSocket session using `URLSessionWebSocketTask`. It builds the gateway WebSocket URL via `GatewayHTTPClient.buildWebSocketRequest(path: "stt/stream", params:)`.
|
|
672
|
+
- `STTProviderRegistry` (`clients/shared/Utilities/STTProviderRegistry.swift`) exposes `isStreamingAvailable` (checks the configured provider's `conversationStreamingMode` from the bundled `stt-provider-catalog.json`) and `isServiceConfigured` (checks whether any STT provider is set).
|
|
673
|
+
- macOS: `VoiceInputManager.startStreamingSession()` creates a fresh `STTStreamingClient` per recording session. Streaming partials take priority over `SFSpeechRecognizer` partials while the stream is active and healthy. When recording stops, if the stream delivered at least one `final` event (`streamingReceivedFinal`) and has not failed (`streamingFailed`), the streaming final text is used directly. Otherwise, the batch STT path (`STTClient.transcribe()`) provides the fallback.
|
|
674
|
+
- iOS: `InputBarView.handleStreamingEvent()` applies the same priority scheme. Streaming partials update the text field while `isStreamingActive` is true and the user has not manually typed. A `.final` event commits the result via `onVoiceResult` and tears down the session. On error or close without a final, `resolveTranscriptWithServiceFirst()` triggers batch STT fallback.
|
|
675
|
+
|
|
676
|
+
**Fallback semantics:**
|
|
677
|
+
|
|
678
|
+
The conversation streaming path degrades gracefully to the existing batch STT path:
|
|
679
|
+
|
|
680
|
+
1. **Unsupported provider** (a hypothetical provider with `conversationStreamingMode: "none"`): The client checks `STTProviderRegistry.isStreamingAvailable` before attempting a streaming session. When `false`, recording proceeds with the batch-only flow (no WebSocket is opened). On the daemon side, if a streaming session is somehow opened for an unsupported provider, the session sends an `error` event followed by `closed` and closes the socket with code 1000.
|
|
681
|
+
2. **Connection failure** (network error, gateway down, auth failure): The `STTStreamingClient` reports an `STTStreamFailure` to the client's `onFailure` callback. macOS sets `streamingFailed = true`; iOS sets `isStreamingActive = false`. Both platforms then fall through to batch STT resolution when recording stops.
|
|
682
|
+
3. **Mid-session provider error** (provider WebSocket disconnect, timeout, rate limit): The daemon session emits an `error` event (with a normalized `SttErrorCategory`) followed by `closed`. The client marks the stream as failed and defers to batch STT.
|
|
683
|
+
4. **Missing credentials**: `resolveStreamingTranscriber()` returns `null` when the API key is not configured. The session sends an `error`+`closed` pair and the client falls back to batch.
|
|
684
|
+
|
|
685
|
+
**Error category mapping:**
|
|
686
|
+
|
|
687
|
+
| Category | Deepgram close codes | Google Gemini close codes | Client action |
|
|
688
|
+
| ---------------- | -------------------- | ------------------------------------ | ---------------------------------- |
|
|
689
|
+
| `auth` | 1008, 4001 | 1008, 4001 | Mark stream failed; batch fallback |
|
|
690
|
+
| `rate-limit` | 1013 | 1013 | Mark stream failed; batch fallback |
|
|
691
|
+
| `timeout` | N/A (inactivity) | N/A (inactivity) | Mark stream failed; batch fallback |
|
|
692
|
+
| `provider-error` | All other codes | All other codes / Live session error | Mark stream failed; batch fallback |
|
|
693
|
+
|
|
694
|
+
**Key source files:**
|
|
695
|
+
|
|
696
|
+
| File | Purpose |
|
|
697
|
+
| ----------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
698
|
+
| `src/stt/types.ts` | `StreamingTranscriber` interface, `SttStreamClientEvent`/`SttStreamServerEvent` discriminated unions, `ConversationStreamingMode` type |
|
|
699
|
+
| `src/stt/stt-stream-session.ts` | Runtime session orchestrator: lifecycle management, idle timeout, event forwarding with `seq` ordering |
|
|
700
|
+
| `src/providers/speech-to-text/deepgram-realtime.ts` | Deepgram realtime-ws adapter: WebSocket to Deepgram `/v1/listen`, `is_final`/`speech_final` normalization |
|
|
701
|
+
| `src/providers/speech-to-text/google-gemini-live-stream.ts` | Google Gemini realtime-ws adapter: bidirectional Live API session, `serverContent.inputTranscription` normalization |
|
|
702
|
+
| `src/providers/speech-to-text/provider-catalog.ts` | Provider catalog with `conversationStreamingMode` per entry (`realtime-ws`, `incremental-batch`, `none`) |
|
|
703
|
+
| `src/providers/speech-to-text/resolve.ts` | `resolveStreamingTranscriber()`: credential-aware factory for streaming adapters; `resolveConversationStreamingSttCapability()`: capability validator |
|
|
704
|
+
| `src/runtime/http-server.ts` | Runtime WebSocket upgrade handler for `/v1/stt/stream`, session registry (`activeSttStreamSessions`), graceful shutdown |
|
|
705
|
+
| `gateway/src/http/routes/stt-stream-websocket.ts` | Gateway WebSocket proxy: authenticates client, opens upstream WS to daemon with service token |
|
|
706
|
+
| `clients/shared/Network/STTStreamingClient.swift` | Shared Swift WebSocket client: `URLSessionWebSocketTask`-based, event parsing, failure reporting |
|
|
707
|
+
| `clients/shared/Utilities/STTProviderRegistry.swift` | Client-side provider catalog: `isStreamingAvailable`, `conversationStreamingMode` per provider |
|
|
708
|
+
| `clients/macos/.../VoiceInputManager.swift` | macOS integration: `startStreamingSession()`, streaming/batch priority, fallback on failure |
|
|
709
|
+
| `clients/ios/Views/InputBarView.swift` | iOS integration: `handleStreamingEvent()`, auto-stop coordination, batch fallback |
|
|
710
|
+
|
|
711
|
+
**Client service-first boundary:**
|
|
712
|
+
|
|
713
|
+
All product-facing dictation and voice-streaming paths on macOS and iOS use a service-first STT strategy. Clients record audio, encode it to WAV via `AudioWavEncoder` (shared utility in `clients/shared/Utilities/AudioWavEncoder.swift`), and POST it through the gateway to the daemon's `POST /v1/stt/transcribe` endpoint via `STTClient` (`clients/shared/Network/STTClient.swift`). The daemon resolves the configured STT provider through `resolveBatchTranscriber()` and returns the transcribed text.
|
|
714
|
+
|
|
715
|
+
- `STTClient` conforms to `STTClientProtocol` and returns a typed `STTResult` enum (`success`, `notConfigured`, `serviceUnavailable`, `error`). Callers pattern-match on the result to deterministically trigger native fallback.
|
|
716
|
+
- The gateway proxies the request via assistant-scoped path rewriting: `/v1/assistants/:id/stt/transcribe` is rewritten to `/v1/stt/transcribe` on the daemon.
|
|
717
|
+
- `stt-routes.ts` (`src/runtime/routes/stt-routes.ts`) defines the HTTP endpoint, validates the audio payload, and delegates to `resolveBatchTranscriber()`.
|
|
718
|
+
|
|
719
|
+
Product-facing flows using service-first STT:
|
|
720
|
+
|
|
721
|
+
| Flow | Client | Entry point |
|
|
722
|
+
| ----------------------------- | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
|
|
723
|
+
| **Push-to-talk dictation** | macOS | `VoiceInputManager.resolveTranscription()` — encodes accumulated PCM buffers to WAV, calls `sttClient.transcribe()`, falls back to native text on failure |
|
|
724
|
+
| **Conversation chat capture** | macOS | `VoiceInputManager.handleFinalTranscription()` — prefers streaming final when available; falls back to batch `sttClient.transcribe()` when streaming was not used, failed, or produced no finals |
|
|
725
|
+
| **Input bar dictation** | iOS | `InputBarView.resolveTranscriptWithServiceFirst()` — encodes captured audio buffers to WAV, calls `sttClient.transcribe()`, falls back to native transcript on failure. In conversation mode, defers to streaming final when `streamingFinalReceived` is set |
|
|
726
|
+
| **Voice mode (streaming)** | macOS | `OpenAIVoiceService.stopRecordingAndGetTranscription()` — encodes per-turn PCM to WAV, calls `sttClient.transcribe()` for turn-final transcript resolution, falls back to SFSpeechRecognizer result |
|
|
727
|
+
|
|
728
|
+
**Client-native fallback boundary:**
|
|
729
|
+
|
|
730
|
+
Apple-native on-device recognition via `SFSpeechRecognizer` serves two roles in all three product-facing flows above: (1) it provides low-latency partial transcriptions for real-time display during recording, and (2) it provides the fallback final transcription when the STT service is unconfigured (HTTP 503), temporarily unavailable (HTTP 5xx), or returns an empty result. The `SpeechRecognizerAdapter` protocols on each platform abstract Apple Speech for **testability and dependency injection**.
|
|
731
|
+
|
|
732
|
+
- macOS: `SpeechRecognizerAdapter` protocol in `clients/macos/vellum-assistant/Features/Voice/SpeechRecognizerAdapter.swift` abstracts `SFSpeechRecognizer` static APIs and instance creation. `AppleSpeechRecognizerAdapter` is the production implementation. `OpenAIVoiceService` and `VoiceInputManager` consume the adapter via dependency injection. **Note:** The macOS protocol leaks Apple Speech types through its surface — `authorizationStatus()` returns `SFSpeechRecognizerAuthorizationStatus` and `makeRecognizer(locale:)` returns `SFSpeechRecognizer?` directly. This means callers depend on the Speech framework at compile time.
|
|
733
|
+
- iOS: `SpeechRecognizerAdapter` protocol in `clients/ios/Services/SpeechRecognizerAdapter.swift` covers authorization, availability, and task construction. `AppleSpeechRecognizerAdapter` is the production implementation. `InputBarView` consumes the adapter. **Note:** The iOS protocol defines its own framework-agnostic types (`SpeechRecognizerAuthorizationStatus`, `SpeechRecognitionResult`) so callers never see `SFSpeechRecognizer` directly. However, `startRecognitionTask` still returns `SFSpeechAudioBufferRecognitionRequest` in its tuple, so full framework decoupling is not yet achieved.
|
|
734
|
+
|
|
735
|
+
Platform divergence summary:
|
|
736
|
+
|
|
737
|
+
- **Auth API:** macOS uses a callback-based `requestAuthorization(completion:)` returning `SFSpeechRecognizerAuthorizationStatus`; iOS uses `async requestAuthorization()` returning a custom `SpeechRecognizerAuthorizationStatus` enum.
|
|
738
|
+
- **Recognizer exposure:** macOS exposes the raw `SFSpeechRecognizer?` via `makeRecognizer(locale:)`; iOS fully encapsulates recognizer construction inside `startRecognitionTask`.
|
|
739
|
+
- **Concurrency model:** The iOS protocol is `@MainActor`-annotated; the macOS protocol is not.
|
|
740
|
+
|
|
741
|
+
These differences are intentional — the adapters were designed for their respective platform integration needs, not for cross-platform uniformity.
|
|
742
|
+
|
|
743
|
+
**Cross-boundary notes:**
|
|
744
|
+
|
|
745
|
+
- The `services.stt` config block is the single source of truth for STT provider selection across the daemon batch boundary, the conversation streaming boundary, the client service-first boundary, and the telephony boundary. The batch and streaming resolvers (`resolveBatchTranscriber()`, `resolveStreamingTranscriber()`) both read from `services.stt.provider` and resolve credentials through the same catalog; the telephony boundary uses `resolveTelephonySttRouting()` to determine the Twilio integration strategy. The daemon provider catalog (`src/providers/speech-to-text/provider-catalog.ts`) is the authoritative registry of supported providers; the client catalog (`meta/stt-provider-catalog.json`) mirrors it for display purposes (including `conversationStreamingMode`). The parity guard test (`src/__tests__/stt-catalog-parity.test.ts`) fails CI when the two catalogs diverge on provider IDs, ordering, or credential-provider name mappings.
|
|
746
|
+
- Conversation streaming does not replace the client service-first batch path. When streaming is available, it runs concurrently during recording and provides real-time partials and finals. The batch path remains the fallback for providers that do not support streaming, when streaming fails mid-session, or when streaming produces no final transcript.
|
|
747
|
+
- Credential mapping is catalog-driven: `provider-secret-catalog.ts` derives STT API-key provider names from the daemon catalog via `listCredentialProviderNames()`, deduplicating against the LLM/search provider list. Adding a provider to the catalog automatically includes its credential name in `API_KEY_PROVIDERS`.
|
|
748
|
+
- Terminology: "STT" and "transcription" refer to the same operation (converting audio to text). "Speech recognition" is used in client-native contexts where Apple's Speech framework terminology is canonical. All three terms map to the same conceptual operation.
|
|
749
|
+
- **Onboarding**: For a step-by-step guide to adding a new STT provider, see `docs/stt-provider-onboarding.md`.
|
|
750
|
+
|
|
585
751
|
### Update Bulletin System
|
|
586
752
|
|
|
587
753
|
Release-driven update notification system that surfaces release notes to the assistant via the system prompt.
|
|
@@ -2026,6 +2192,76 @@ The guardian trust system uses a three-valued `TrustClass` — `'guardian'`, `'t
|
|
|
2026
2192
|
| `src/memory/channel-verification-sessions.ts` | `GuardianBinding` with required `guardianPrincipalId` |
|
|
2027
2193
|
| `src/__tests__/trust-context-guards.test.ts` | Guard tests enforcing trust-context type invariants |
|
|
2028
2194
|
|
|
2195
|
+
### TTS Provider Abstraction (`services.tts`)
|
|
2196
|
+
|
|
2197
|
+
All text-to-speech functionality (in-app message playback and phone call voice) routes through a catalog-driven, provider-agnostic TTS abstraction. The architecture consists of six layers: a canonical provider catalog, a config schema, a config resolver, a provider registry, an explicit call-strategy abstraction, and a top-level synthesis orchestrator.
|
|
2198
|
+
|
|
2199
|
+
**Canonical provider catalog (`provider-catalog.ts`):** The provider catalog is the **single source of truth** for TTS provider identity and metadata on the assistant side. Every provider the system supports is declared as a `TtsProviderCatalogEntry` in the `CATALOG` array. Each entry captures the provider's unique ID (`TtsProviderId`), display name, telephony call mode (`TtsCallMode`: `"native-twilio"` or `"synthesized-play"`), static capabilities (`supportsStreaming`, `supportedFormats`), and secret requirements (credential store keys, display names, setup commands). Downstream modules query the catalog via `getCatalogProvider()`, `listCatalogProviders()`, or `listCatalogProviderIds()` instead of hardcoding provider IDs.
|
|
2200
|
+
|
|
2201
|
+
A parallel **client artifact** (`meta/tts-provider-catalog.json`) captures the subset of provider metadata needed by native clients (macOS, iOS) for display and setup UX. The client artifact must list exactly the same provider IDs as the assistant catalog. A CI consistency guard test (`src/tts/__tests__/provider-catalog-consistency.test.ts`) compares the two sets and fails if they drift.
|
|
2202
|
+
|
|
2203
|
+
**Config schema (`services.tts`):** The canonical config block lives at `services.tts` in the assistant config. The set of valid provider IDs and provider-specific config objects is catalog-driven — the Zod schema reads from the catalog rather than maintaining a separate hardcoded enum. It contains:
|
|
2204
|
+
|
|
2205
|
+
| Field | Type | Default | Description |
|
|
2206
|
+
| ----------------------------- | ------ | -------------- | --------------------------------------------------------- |
|
|
2207
|
+
| `services.tts.mode` | enum | `"your-own"` | Service mode (only `"your-own"` is supported) |
|
|
2208
|
+
| `services.tts.provider` | enum | `"elevenlabs"` | Active TTS provider (must be a catalog-known provider ID) |
|
|
2209
|
+
| `services.tts.providers.<id>` | object | _(defaults)_ | Provider-specific settings, one block per catalog entry |
|
|
2210
|
+
|
|
2211
|
+
Provider-specific config is nested under `services.tts.providers.<id>`. All legacy top-level keys (`elevenlabs.*`, `fishAudio.*`) were removed by workspace migration 032 — only canonical `services.tts` paths are supported at runtime.
|
|
2212
|
+
|
|
2213
|
+
**Config resolver (`tts-config-resolver.ts`):** `resolveTtsConfig(config)` reads `services.tts.provider` to determine the active provider and returns a `ResolvedTtsConfig` containing the provider ID and its provider-specific config object from `services.tts.providers.<id>`. No legacy fallback logic exists.
|
|
2214
|
+
|
|
2215
|
+
**Provider registry (`provider-registry.ts`):** A runtime registry where provider adapters self-register at startup via `registerTtsProvider()`. Callers resolve a provider by ID with `getTtsProvider()`, which throws for unknown IDs so misconfiguration surfaces immediately. Built-in providers are registered in `providers/register-builtins.ts` during daemon initialization. The registration is catalog-checked — `register-builtins.ts` validates that each adapter's ID exists in the catalog.
|
|
2216
|
+
|
|
2217
|
+
**Provider interface (`types.ts`):** Every provider implements the `TtsProvider` interface:
|
|
2218
|
+
|
|
2219
|
+
- `id` — unique provider identifier (matches `TtsProviderId`)
|
|
2220
|
+
- `capabilities` — static capability advertisement (`supportsStreaming`, `supportedFormats`)
|
|
2221
|
+
- `synthesize(request)` — buffer-oriented synthesis (required for all providers)
|
|
2222
|
+
- `synthesizeStream?(request, onChunk)` — optional chunk-level streaming for real-time use cases
|
|
2223
|
+
|
|
2224
|
+
The `TtsUseCase` discriminator (`"phone-call"` or `"message-playback"`) lets providers tailor format, latency, and quality trade-offs per product surface.
|
|
2225
|
+
|
|
2226
|
+
**Synthesis orchestrator (`synthesize-text.ts`):** `synthesizeText()` is the top-level entry point. It resolves the globally configured provider via the config resolver, looks up the adapter in the registry, and delegates synthesis. Provider selection is always global — per-use-case policy only gates capabilities (e.g. format checks), never overrides the chosen provider.
|
|
2227
|
+
|
|
2228
|
+
**Call strategy abstraction (`tts-call-strategy.ts`):** The call strategy layer determines how a TTS provider integrates with the Twilio ConversationRelay telephony path. Instead of inferring call behavior from runtime capabilities, `resolveCallStrategy(config)` reads the provider's `callMode` from the canonical catalog and returns a `TtsCallStrategy` with the provider ID and call mode. Two modes exist:
|
|
2229
|
+
|
|
2230
|
+
- **`native-twilio`** — Twilio handles TTS natively via ConversationRelay. The profile needs a real `ttsProvider` name (e.g. `"ElevenLabs"`) and a provider-specific voice spec string. New native providers plug in by registering a `NativeTwilioVoiceSpecBuilder` via `registerNativeTwilioVoiceSpec()` — no edits to core call routing logic required.
|
|
2231
|
+
- **`synthesized-play`** — The assistant synthesises audio via the provider's HTTP API and streams chunks to Twilio via `play` messages. Uses a placeholder TTS provider (`"Google"`) and an empty voice string because Twilio never drives TTS itself on this path.
|
|
2232
|
+
|
|
2233
|
+
**Phone call integration:** `resolveVoiceQualityProfile()` in `voice-quality.ts` uses `resolveCallStrategy()` to determine the call mode, then dispatches to the appropriate path. For `native-twilio`, it looks up the registered `NativeTwilioVoiceSpec` to build the voice string. For `synthesized-play`, it uses the placeholder profile. This replaces the previous `supportsStreaming`-based branching with explicit catalog-declared modes.
|
|
2234
|
+
|
|
2235
|
+
**Adding a new TTS provider (catalog-first checklist):**
|
|
2236
|
+
|
|
2237
|
+
1. **Catalog entry** — Add a new `TtsProviderCatalogEntry` to the `CATALOG` array in `src/tts/provider-catalog.ts`. Declare the provider's ID, display name, call mode, capabilities, and secret requirements.
|
|
2238
|
+
2. **Client artifact** — Add a corresponding entry to `meta/tts-provider-catalog.json` with the same provider ID, display name, and client-facing metadata (subtitle, setup mode, setup hint). The CI consistency guard will fail if this is skipped.
|
|
2239
|
+
3. **Config schema** — Add a new Zod object under `TtsProvidersSchema` in `src/config/schemas/tts.ts` for the provider's settings. The valid provider ID enum is catalog-driven.
|
|
2240
|
+
4. **Provider adapter** — Create `src/tts/providers/<id>-provider.ts` implementing `TtsProvider` with the appropriate `capabilities` and `synthesize`/`synthesizeStream` methods.
|
|
2241
|
+
5. **Register the adapter** — Add a factory entry for the provider to the `providerFactories` map in `src/tts/providers/index.ts`. The `register-builtins.ts` module iterates the catalog at startup and looks up each ID in this map — a missing entry is a fatal error.
|
|
2242
|
+
6. **Optional: native Twilio voice builder** — If the provider uses `native-twilio` call mode, add a `NativeTwilioVoiceSpec` entry to the `nativeVoiceSpecs` map in `src/tts/providers/register-builtins.ts`. Synthesized-play providers skip this step entirely.
|
|
2243
|
+
|
|
2244
|
+
No hardcoded enum edits are required — the `TtsProviderId` union in `types.ts` uses an open string union (`(string & {})`), the config schema reads valid IDs from the catalog, and the call strategy dispatches based on the catalog's `callMode` field. The registry, resolver, orchestrator, and call strategy all automatically pick up the new provider when selected via `services.tts.provider`.
|
|
2245
|
+
|
|
2246
|
+
**Key source files:**
|
|
2247
|
+
|
|
2248
|
+
| File | Purpose |
|
|
2249
|
+
| ---------------------------------------------------------- | -------------------------------------------------------------------------------------------- |
|
|
2250
|
+
| `src/tts/provider-catalog.ts` | Canonical provider catalog: single source of truth for provider IDs and metadata |
|
|
2251
|
+
| `src/tts/types.ts` | Core domain types: `TtsProvider`, `TtsProviderId`, `TtsCallMode`, `TtsUseCase`, capabilities |
|
|
2252
|
+
| `src/tts/provider-registry.ts` | Runtime provider registry: register, lookup, list |
|
|
2253
|
+
| `src/tts/tts-config-resolver.ts` | Config resolver: `resolveTtsConfig()` reads `services.tts` and returns resolved |
|
|
2254
|
+
| `src/tts/synthesize-text.ts` | Top-level orchestrator: `synthesizeText()` entry point |
|
|
2255
|
+
| `src/tts/providers/register-builtins.ts` | Startup registration of built-in providers (catalog-checked) |
|
|
2256
|
+
| `src/tts/providers/elevenlabs-provider.ts` | ElevenLabs adapter implementation |
|
|
2257
|
+
| `src/tts/providers/fish-audio-provider.ts` | Fish Audio adapter implementation |
|
|
2258
|
+
| `src/config/schemas/tts.ts` | Zod schema for `services.tts` config block (catalog-driven valid provider IDs) |
|
|
2259
|
+
| `src/calls/tts-call-strategy.ts` | Explicit call strategy: resolves call mode from catalog, native voice spec registry |
|
|
2260
|
+
| `src/calls/voice-quality.ts` | Phone call integration: `resolveVoiceQualityProfile()` uses call strategy |
|
|
2261
|
+
| `meta/tts-provider-catalog.json` | Client artifact: provider metadata for macOS/iOS settings UI |
|
|
2262
|
+
| `src/tts/__tests__/provider-catalog-consistency.test.ts` | CI guard: catalog vs client artifact provider ID consistency |
|
|
2263
|
+
| `src/workspace/migrations/032-tts-provider-unification.ts` | Migration that materialised canonical `services.tts` fields |
|
|
2264
|
+
|
|
2029
2265
|
### Managed Profiler Runtime
|
|
2030
2266
|
|
|
2031
2267
|
Managed cloud assistants use Bun's built-in CPU and heap profiling to capture runtime performance data. The profiler subsystem consists of a persistent on-disk run store, a retention/pruning sweep, and HTTP routes for remote management.
|
|
@@ -2078,3 +2314,30 @@ These endpoints allow the platform (via vembda proxy) to enumerate, inspect, exp
|
|
|
2078
2314
|
| `src/util/platform.ts` | Profiler directory path helpers |
|
|
2079
2315
|
| `src/__tests__/profiler-run-store.test.ts` | Profiler store unit tests |
|
|
2080
2316
|
| `src/__tests__/profiler-routes.test.ts` | Profiler HTTP route tests |
|
|
2317
|
+
|
|
2318
|
+
### LLM Provider Transport — OpenAI Responses API
|
|
2319
|
+
|
|
2320
|
+
OpenAI inference uses the **Responses API** (`client.responses.stream()`), not the Chat Completions API. OpenAI-compatible providers (OpenRouter, Fireworks, Ollama) continue to use the chat-completions transport.
|
|
2321
|
+
|
|
2322
|
+
**Transport split:**
|
|
2323
|
+
|
|
2324
|
+
| Provider key | Transport class | API surface |
|
|
2325
|
+
| ------------ | ------------------------- | --------------------------- |
|
|
2326
|
+
| `openai` | `OpenAIResponsesProvider` | `client.responses.stream()` |
|
|
2327
|
+
| `openrouter` | `OpenRouterProvider` | `chat.completions.create()` |
|
|
2328
|
+
| `fireworks` | `FireworksProvider` | `chat.completions.create()` |
|
|
2329
|
+
| `ollama` | `OllamaProvider` | `chat.completions.create()` |
|
|
2330
|
+
|
|
2331
|
+
The registry (`src/providers/registry.ts`) imports `OpenAIResponsesProvider` from `openai/client.ts` and wires it to the `openai` key. The chat-completions transport (`OpenAIChatCompletionsProvider`) remains available for OpenAI-compatible providers that implement the Chat Completions API.
|
|
2332
|
+
|
|
2333
|
+
Both transports produce the same `ProviderResponse` contract so downstream code (agent loop, context management, conversation history) is transport-agnostic.
|
|
2334
|
+
|
|
2335
|
+
**Key files:**
|
|
2336
|
+
|
|
2337
|
+
| File | Purpose |
|
|
2338
|
+
| ------------------------------------------------------ | -------------------------------------------------------------------- |
|
|
2339
|
+
| `src/providers/openai/responses-provider.ts` | Responses API transport (streaming, tool calls, usage mapping) |
|
|
2340
|
+
| `src/providers/openai/chat-completions-provider.ts` | Chat Completions transport (OpenAI-compatible providers) |
|
|
2341
|
+
| `src/providers/openai/client.ts` | Re-exports both transports + `validateOpenAIApiKey()` |
|
|
2342
|
+
| `src/providers/registry.ts` | Provider initialization (wires `openai` → `OpenAIResponsesProvider`) |
|
|
2343
|
+
| `src/__tests__/openai-responses-cutover-guard.test.ts` | CI guard preventing chat-completions regression in OpenAI path |
|
package/Dockerfile
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
# Use debian as base image
|
|
2
|
-
FROM debian:trixie@sha256:
|
|
2
|
+
FROM debian:trixie-slim@sha256:4ffb3a1511099754cddc70eb1b12e50ffdb67619aa0ab6c13fcd800a78ef7c7a AS builder
|
|
3
3
|
|
|
4
4
|
WORKDIR /app
|
|
5
5
|
|
|
@@ -20,7 +20,6 @@ ENV PATH="/root/.bun/bin:${PATH}"
|
|
|
20
20
|
COPY packages/ces-contracts ./packages/ces-contracts
|
|
21
21
|
COPY packages/credential-storage ./packages/credential-storage
|
|
22
22
|
COPY packages/egress-proxy ./packages/egress-proxy
|
|
23
|
-
|
|
24
23
|
# Install assistant dependencies first for cache reuse
|
|
25
24
|
COPY assistant/package.json assistant/bun.lock ./assistant/
|
|
26
25
|
RUN cd /app/assistant && bun install --frozen-lockfile
|
|
@@ -29,7 +28,7 @@ RUN cd /app/assistant && bun install --frozen-lockfile
|
|
|
29
28
|
COPY assistant ./assistant
|
|
30
29
|
|
|
31
30
|
# Final stage
|
|
32
|
-
FROM debian:trixie@sha256:
|
|
31
|
+
FROM debian:trixie-slim@sha256:4ffb3a1511099754cddc70eb1b12e50ffdb67619aa0ab6c13fcd800a78ef7c7a AS runner
|
|
33
32
|
|
|
34
33
|
WORKDIR /app/assistant
|
|
35
34
|
|
package/bun.lock
CHANGED
|
@@ -5,53 +5,51 @@
|
|
|
5
5
|
"": {
|
|
6
6
|
"name": "@vellumai/assistant",
|
|
7
7
|
"dependencies": {
|
|
8
|
-
"@agentclientprotocol/sdk": "
|
|
9
|
-
"@anthropic-ai/sdk": "
|
|
10
|
-
"@google/genai": "
|
|
11
|
-
"@modelcontextprotocol/sdk": "
|
|
12
|
-
"@qdrant/js-client-rest": "
|
|
13
|
-
"@resvg/resvg-js": "
|
|
14
|
-
"@sentry/node": "
|
|
8
|
+
"@agentclientprotocol/sdk": "0.16.1",
|
|
9
|
+
"@anthropic-ai/sdk": "0.78.0",
|
|
10
|
+
"@google/genai": "1.45.0",
|
|
11
|
+
"@modelcontextprotocol/sdk": "1.27.1",
|
|
12
|
+
"@qdrant/js-client-rest": "1.17.0",
|
|
13
|
+
"@resvg/resvg-js": "2.6.2",
|
|
14
|
+
"@sentry/node": "10.43.0",
|
|
15
15
|
"@vellumai/ces-contracts": "file:../packages/ces-contracts",
|
|
16
16
|
"@vellumai/credential-storage": "file:../packages/credential-storage",
|
|
17
17
|
"@vellumai/egress-proxy": "file:../packages/egress-proxy",
|
|
18
|
-
"
|
|
19
|
-
"
|
|
20
|
-
"
|
|
21
|
-
"
|
|
22
|
-
"
|
|
23
|
-
"
|
|
24
|
-
"
|
|
25
|
-
"minimatch": "
|
|
26
|
-
"openai": "
|
|
27
|
-
"pino": "
|
|
28
|
-
"pino-pretty": "
|
|
29
|
-
"playwright": "
|
|
30
|
-
"postgres": "
|
|
31
|
-
"qrcode": "
|
|
32
|
-
"
|
|
33
|
-
"
|
|
34
|
-
"tldts": "^7.0.23",
|
|
18
|
+
"archiver": "7.0.1",
|
|
19
|
+
"commander": "13.1.0",
|
|
20
|
+
"croner": "10.0.1",
|
|
21
|
+
"dotenv": "17.3.1",
|
|
22
|
+
"drizzle-orm": "0.45.2",
|
|
23
|
+
"jszip": "3.10.1",
|
|
24
|
+
"marked": "18.0.0",
|
|
25
|
+
"minimatch": "10.2.4",
|
|
26
|
+
"openai": "6.29.0",
|
|
27
|
+
"pino": "9.14.0",
|
|
28
|
+
"pino-pretty": "13.1.3",
|
|
29
|
+
"playwright": "1.58.2",
|
|
30
|
+
"postgres": "3.4.8",
|
|
31
|
+
"qrcode": "1.5.4",
|
|
32
|
+
"rrule": "2.8.1",
|
|
33
|
+
"tldts": "7.0.25",
|
|
35
34
|
"tree-sitter-bash": "0.25.1",
|
|
36
|
-
"uuid": "
|
|
35
|
+
"uuid": "11.1.0",
|
|
37
36
|
"web-tree-sitter": "0.26.5",
|
|
38
|
-
"yaml": "
|
|
39
|
-
"zod": "
|
|
37
|
+
"yaml": "2.8.2",
|
|
38
|
+
"zod": "4.3.6",
|
|
40
39
|
},
|
|
41
40
|
"devDependencies": {
|
|
42
|
-
"@types/archiver": "
|
|
43
|
-
"@types/bun": "
|
|
44
|
-
"@types/node": "
|
|
45
|
-
"@types/
|
|
46
|
-
"
|
|
47
|
-
"
|
|
48
|
-
"eslint": "
|
|
49
|
-
"
|
|
50
|
-
"
|
|
51
|
-
"
|
|
52
|
-
"
|
|
53
|
-
"typescript": "
|
|
54
|
-
"typescript-eslint": "^8.54.0",
|
|
41
|
+
"@types/archiver": "7.0.0",
|
|
42
|
+
"@types/bun": "1.3.10",
|
|
43
|
+
"@types/node": "25.5.0",
|
|
44
|
+
"@types/uuid": "10.0.0",
|
|
45
|
+
"drizzle-kit": "0.30.6",
|
|
46
|
+
"eslint": "10.0.3",
|
|
47
|
+
"eslint-plugin-simple-import-sort": "12.1.1",
|
|
48
|
+
"fast-check": "4.6.0",
|
|
49
|
+
"knip": "5.86.0",
|
|
50
|
+
"prettier": "3.8.1",
|
|
51
|
+
"typescript": "5.9.3",
|
|
52
|
+
"typescript-eslint": "8.57.0",
|
|
55
53
|
},
|
|
56
54
|
},
|
|
57
55
|
},
|
|
@@ -356,8 +354,6 @@
|
|
|
356
354
|
|
|
357
355
|
"@types/pg-pool": ["@types/pg-pool@2.0.7", "", { "dependencies": { "@types/pg": "*" } }, "sha512-U4CwmGVQcbEuqpyju8/ptOKg6gEC+Tqsvj2xS9o1g71bUh8twxnC6ZL5rZKCsGN0iyH0CwgUyc9VR5owNQF9Ng=="],
|
|
358
356
|
|
|
359
|
-
"@types/react": ["@types/react@19.2.14", "", { "dependencies": { "csstype": "^3.2.2" } }, "sha512-ilcTH/UniCkMdtexkoCN0bI7pMcJDvmQFPvuPvmEaYA/NSfFTAgdUSLAoVjaRJm7+6PvcM+q1zYOwS4wTYMF9w=="],
|
|
360
|
-
|
|
361
357
|
"@types/readdir-glob": ["@types/readdir-glob@1.1.5", "", { "dependencies": { "@types/node": "*" } }, "sha512-raiuEPUYqXu+nvtY2Pe8s8FEmZ3x5yAH4VkLdihcPdalvsHltomrRC9BzuStrJ9yk06470hS0Crw0f1pXqD+Hg=="],
|
|
362
358
|
|
|
363
359
|
"@types/retry": ["@types/retry@0.12.0", "", {}, "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA=="],
|
|
@@ -404,8 +400,6 @@
|
|
|
404
400
|
|
|
405
401
|
"agent-base": ["agent-base@7.1.4", "", {}, "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ=="],
|
|
406
402
|
|
|
407
|
-
"agentmail": ["agentmail@0.1.19", "", { "dependencies": { "ws": "^8.16.0" } }, "sha512-L0au0GnyH24Ob7l0QrkkdwJWL25Aa26f6YnuQlpvwtQMVOTu6IiMftJ8LnVVwQtKwBSZ6cWLS+9SdCvgw+LWGg=="],
|
|
408
|
-
|
|
409
403
|
"ajv": ["ajv@6.14.0", "", { "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", "json-schema-traverse": "^0.4.1", "uri-js": "^4.2.2" } }, "sha512-IWrosm/yrn43eiKqkfkHis7QioDleaXQHdDVPKg0FSwwd/DuvyX79TZnFOnYpB7dcsFAMmtFztZuXPDvSePkFw=="],
|
|
410
404
|
|
|
411
405
|
"ajv-formats": ["ajv-formats@3.0.1", "", { "dependencies": { "ajv": "^8.0.0" } }, "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ=="],
|
|
@@ -500,8 +494,6 @@
|
|
|
500
494
|
|
|
501
495
|
"cross-spawn": ["cross-spawn@7.0.6", "", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="],
|
|
502
496
|
|
|
503
|
-
"csstype": ["csstype@3.2.3", "", {}, "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ=="],
|
|
504
|
-
|
|
505
497
|
"data-uri-to-buffer": ["data-uri-to-buffer@4.0.1", "", {}, "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A=="],
|
|
506
498
|
|
|
507
499
|
"dateformat": ["dateformat@4.6.3", "", {}, "sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA=="],
|
|
@@ -520,7 +512,7 @@
|
|
|
520
512
|
|
|
521
513
|
"drizzle-kit": ["drizzle-kit@0.30.6", "", { "dependencies": { "@drizzle-team/brocli": "^0.10.2", "@esbuild-kit/esm-loader": "^2.5.5", "esbuild": "^0.19.7", "esbuild-register": "^3.5.0", "gel": "^2.0.0" }, "bin": { "drizzle-kit": "bin.cjs" } }, "sha512-U4wWit0fyZuGuP7iNmRleQyK2V8wCuv57vf5l3MnG4z4fzNTjY/U13M8owyQ5RavqvqxBifWORaR3wIUzlN64g=="],
|
|
522
514
|
|
|
523
|
-
"drizzle-orm": ["drizzle-orm@0.
|
|
515
|
+
"drizzle-orm": ["drizzle-orm@0.45.2", "", { "peerDependencies": { "@aws-sdk/client-rds-data": ">=3", "@cloudflare/workers-types": ">=4", "@electric-sql/pglite": ">=0.2.0", "@libsql/client": ">=0.10.0", "@libsql/client-wasm": ">=0.10.0", "@neondatabase/serverless": ">=0.10.0", "@op-engineering/op-sqlite": ">=2", "@opentelemetry/api": "^1.4.1", "@planetscale/database": ">=1.13", "@prisma/client": "*", "@tidbcloud/serverless": "*", "@types/better-sqlite3": "*", "@types/pg": "*", "@types/sql.js": "*", "@upstash/redis": ">=1.34.7", "@vercel/postgres": ">=0.8.0", "@xata.io/client": "*", "better-sqlite3": ">=7", "bun-types": "*", "expo-sqlite": ">=14.0.0", "gel": ">=2", "knex": "*", "kysely": "*", "mysql2": ">=2", "pg": ">=8", "postgres": ">=3", "sql.js": ">=1", "sqlite3": ">=5" }, "optionalPeers": ["@aws-sdk/client-rds-data", "@cloudflare/workers-types", "@electric-sql/pglite", "@libsql/client", "@libsql/client-wasm", "@neondatabase/serverless", "@op-engineering/op-sqlite", "@opentelemetry/api", "@planetscale/database", "@prisma/client", "@tidbcloud/serverless", "@types/better-sqlite3", "@types/pg", "@types/sql.js", "@upstash/redis", "@vercel/postgres", "@xata.io/client", "better-sqlite3", "bun-types", "expo-sqlite", "gel", "knex", "kysely", "mysql2", "pg", "postgres", "sql.js", "sqlite3"] }, "sha512-kY0BSaTNYWnoDMVoyY8uxmyHjpJW1geOmBMdSSicKo9CIIWkSxMIj2rkeSR51b8KAPB7m+qysjuHme5nKP+E5Q=="],
|
|
524
516
|
|
|
525
517
|
"dunder-proto": ["dunder-proto@1.0.1", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.1", "es-errors": "^1.3.0", "gopd": "^1.2.0" } }, "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A=="],
|
|
526
518
|
|
|
@@ -758,6 +750,8 @@
|
|
|
758
750
|
|
|
759
751
|
"lru-cache": ["lru-cache@10.4.3", "", {}, "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ=="],
|
|
760
752
|
|
|
753
|
+
"marked": ["marked@18.0.0", "", { "bin": { "marked": "bin/marked.js" } }, "sha512-2e7Qiv/HJSXj8rDEpgTvGKsP8yYtI9xXHKDnrftrmnrJPaFNM7VRb2YCzWaX4BP1iCJ/XPduzDJZMFoqTCcIMA=="],
|
|
754
|
+
|
|
761
755
|
"math-intrinsics": ["math-intrinsics@1.1.0", "", {}, "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g=="],
|
|
762
756
|
|
|
763
757
|
"media-typer": ["media-typer@1.1.0", "", {}, "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw=="],
|
|
@@ -902,8 +896,6 @@
|
|
|
902
896
|
|
|
903
897
|
"raw-body": ["raw-body@3.0.2", "", { "dependencies": { "bytes": "~3.1.2", "http-errors": "~2.0.1", "iconv-lite": "~0.7.0", "unpipe": "~1.0.0" } }, "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA=="],
|
|
904
898
|
|
|
905
|
-
"react": ["react@19.2.4", "", {}, "sha512-9nfp2hYpCwOjAN+8TZFGhtWEwgvWHXqESH8qT89AT/lWklpLON22Lc8pEtnpsZz7VmawabSU0gCjnj8aC0euHQ=="],
|
|
906
|
-
|
|
907
899
|
"readable-stream": ["readable-stream@4.7.0", "", { "dependencies": { "abort-controller": "^3.0.0", "buffer": "^6.0.3", "events": "^3.3.0", "process": "^0.11.10", "string_decoder": "^1.3.0" } }, "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg=="],
|
|
908
900
|
|
|
909
901
|
"readdir-glob": ["readdir-glob@1.1.3", "", { "dependencies": { "minimatch": "^5.1.0" } }, "sha512-v05I2k7xN8zXvPD9N+z/uhXPaj0sUFCe2rcWZIpBsqxfP7xXFQ0tipAd/wjj1YxWyWtUS5IDJpOG82JKt2EAVA=="],
|
package/bunfig.toml
CHANGED
|
@@ -566,7 +566,7 @@ The Anthropic provider places `cache_control: { type: 'ephemeral' }` on the **la
|
|
|
566
566
|
|
|
567
567
|
The session injects a unified `<turn_context>` block into every user message, giving the model awareness of the current timestamp (with timezone), interface, channel, and actor identity. This replaces the former separate `<temporal_context>`, `<inbound_actor_context>`, and per-channel turn context blocks. The unified block persists in conversation history so the assistant retains temporal and actor grounding across turns. Legacy blocks from pre-change history are stripped for backward compatibility.
|
|
568
568
|
|
|
569
|
-
The `
|
|
569
|
+
The `current_time:` field format is: `2026-04-02 (Wednesday) 14:30:00 -05:00 (America/Chicago)` — date, weekday name, local time, UTC offset, and IANA timezone name.
|
|
570
570
|
|
|
571
571
|
### Per-turn flow
|
|
572
572
|
|