@vellumai/assistant 0.6.3 → 0.6.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.prettierignore +5 -0
- package/ARCHITECTURE.md +298 -39
- package/Dockerfile +14 -3
- package/README.md +3 -4
- package/bun.lock +13 -16
- package/docs/architecture/integrations.md +1 -20
- package/docs/architecture/security.md +16 -16
- package/docs/backup-troubleshooting.md +52 -0
- package/docs/browser-use-architecture-phase2.md +174 -0
- package/docs/error-handling.md +111 -0
- package/docs/skills.md +10 -10
- package/docs/stt-provider-onboarding.md +121 -0
- package/knip.json +20 -3
- package/node_modules/@vellumai/ces-contracts/bun.lock +8 -6
- package/node_modules/@vellumai/ces-contracts/package.json +5 -4
- package/node_modules/@vellumai/ces-contracts/src/__tests__/trust-rules.test.ts +471 -0
- package/node_modules/@vellumai/ces-contracts/src/trust-rules.ts +398 -4
- package/node_modules/@vellumai/credential-storage/bun.lock +2 -2
- package/node_modules/@vellumai/credential-storage/package.json +2 -2
- package/node_modules/@vellumai/credential-storage/src/oauth-runtime.ts +20 -2
- package/node_modules/@vellumai/egress-proxy/bun.lock +2 -2
- package/node_modules/@vellumai/egress-proxy/package.json +2 -2
- package/openapi.yaml +1094 -72
- package/package.json +9 -8
- package/scripts/generate-openapi.ts +50 -12
- package/scripts/test.sh +73 -18
- package/src/__tests__/agent-image-optimize.test.ts +28 -0
- package/src/__tests__/agent-loop-callsite-precedence.test.ts +318 -0
- package/src/__tests__/agent-loop-sentry-hygiene.test.ts +137 -0
- package/src/__tests__/agent-loop.test.ts +235 -1
- package/src/__tests__/anthropic-error-formatting.test.ts +98 -0
- package/src/__tests__/anthropic-provider.test.ts +434 -12
- package/src/__tests__/approval-cascade.test.ts +31 -10
- package/src/__tests__/approval-routes-http.test.ts +134 -10
- package/src/__tests__/assistant-attachments.test.ts +44 -0
- package/src/__tests__/assistant-feature-flags-integration.test.ts +29 -0
- package/src/__tests__/auto-analysis-end-to-end.test.ts +550 -0
- package/src/__tests__/auto-analysis-prompt.test.ts +50 -0
- package/src/__tests__/browser-fill-credential.test.ts +12 -1
- package/src/__tests__/browser-identifier-parity-guard.test.ts +53 -0
- package/src/__tests__/browser-skill-baseline-tool-payload.test.ts +23 -33
- package/src/__tests__/browser-skill-endstate.test.ts +52 -159
- package/src/__tests__/btw-routes.test.ts +54 -1
- package/src/__tests__/call-controller.test.ts +582 -22
- package/src/__tests__/call-site-routing-provider.test.ts +214 -0
- package/src/__tests__/catalog-cache.test.ts +27 -4
- package/src/__tests__/catalog-files.test.ts +138 -0
- package/src/__tests__/channel-approval-routes.test.ts +4 -4
- package/src/__tests__/channel-invite-transport.test.ts +2 -2
- package/src/__tests__/channel-readiness-routes.test.ts +16 -20
- package/src/__tests__/channel-readiness-service.test.ts +12 -7
- package/src/__tests__/channel-reply-delivery.test.ts +300 -2
- package/src/__tests__/checker.test.ts +576 -502
- package/src/__tests__/clawhub-files.test.ts +347 -0
- package/src/__tests__/cli-command-risk-guard.test.ts +30 -33
- package/src/__tests__/commit-message-enrichment-service.test.ts +36 -19
- package/src/__tests__/compaction-circuit-breaker.test.ts +336 -0
- package/src/__tests__/compaction.benchmark.test.ts +1 -1
- package/src/__tests__/config-analysis.test.ts +83 -0
- package/src/__tests__/config-loader-backfill.test.ts +174 -0
- package/src/__tests__/config-loader-corrupt.test.ts +183 -0
- package/src/__tests__/config-loader-quarantine-bulletin.test.ts +202 -0
- package/src/__tests__/config-schema-cmd.test.ts +11 -5
- package/src/__tests__/config-schema.test.ts +1458 -198
- package/src/__tests__/config-watcher-cleanup-throttle.test.ts +339 -0
- package/src/__tests__/config-watcher.test.ts +45 -10
- package/src/__tests__/contact-store-user-file.test.ts +511 -0
- package/src/__tests__/contacts-write.test.ts +197 -0
- package/src/__tests__/context-token-estimator.test.ts +191 -1
- package/src/__tests__/context-window-manager.test.ts +618 -2
- package/src/__tests__/conversation-abort-tool-results.test.ts +32 -16
- package/src/__tests__/conversation-agent-loop-overflow.test.ts +62 -17
- package/src/__tests__/conversation-agent-loop.test.ts +510 -84
- package/src/__tests__/conversation-attachments.test.ts +1 -1
- package/src/__tests__/conversation-confirmation-signals.test.ts +165 -9
- package/src/__tests__/conversation-error.test.ts +102 -1
- package/src/__tests__/conversation-history-web-search.test.ts +17 -4
- package/src/__tests__/conversation-init.benchmark.test.ts +42 -1
- package/src/__tests__/conversation-launcher-skill-regression.test.ts +51 -0
- package/src/__tests__/conversation-lifecycle.test.ts +336 -0
- package/src/__tests__/conversation-list-source.test.ts +145 -0
- package/src/__tests__/conversation-load-history-repair.test.ts +27 -10
- package/src/__tests__/conversation-pre-run-repair.test.ts +32 -16
- package/src/__tests__/conversation-process-callsite.test.ts +306 -0
- package/src/__tests__/conversation-provider-retry-repair.test.ts +32 -16
- package/src/__tests__/conversation-queue.test.ts +932 -76
- package/src/__tests__/conversation-routes-disk-view.test.ts +299 -1
- package/src/__tests__/conversation-routes-slash-commands.test.ts +31 -3
- package/src/__tests__/conversation-runtime-assembly.test.ts +2790 -55
- package/src/__tests__/conversation-runtime-workspace.test.ts +12 -12
- package/src/__tests__/conversation-skill-tools.test.ts +12 -143
- package/src/__tests__/conversation-slash-commands.test.ts +33 -0
- package/src/__tests__/conversation-slash-queue.test.ts +120 -34
- package/src/__tests__/conversation-slash-unknown.test.ts +32 -16
- package/src/__tests__/conversation-speed-override.test.ts +30 -11
- package/src/__tests__/conversation-surfaces-standalone-payloads.test.ts +1035 -0
- package/src/__tests__/conversation-surfaces-standalone.test.ts +630 -0
- package/src/__tests__/conversation-title-service.test.ts +2 -2
- package/src/__tests__/conversation-tool-setup-batch-authorized.test.ts +226 -0
- package/src/__tests__/conversation-unread-route.test.ts +2 -2
- package/src/__tests__/conversation-usage.test.ts +3 -1
- package/src/__tests__/conversation-workspace-cache-state.test.ts +31 -10
- package/src/__tests__/conversation-workspace-injection.test.ts +45 -15
- package/src/__tests__/conversation-workspace-tool-tracking.test.ts +46 -16
- package/src/__tests__/credential-broker-browser-fill.test.ts +110 -0
- package/src/__tests__/credential-health-service.test.ts +352 -0
- package/src/__tests__/credential-security-invariants.test.ts +8 -3
- package/src/__tests__/credential-storage-oauth-compat.test.ts +18 -0
- package/src/__tests__/credential-storage-static-compat.test.ts +28 -0
- package/src/__tests__/credential-vault-unit.test.ts +495 -3
- package/src/__tests__/credentials-cli.test.ts +32 -16
- package/src/__tests__/cross-provider-web-search.test.ts +230 -35
- package/src/__tests__/daemon-server-persist-and-process-callsite.test.ts +92 -0
- package/src/__tests__/delete-propagation.test.ts +437 -0
- package/src/__tests__/deterministic-verification-control-plane.test.ts +10 -1
- package/src/__tests__/device-id.test.ts +112 -0
- package/src/__tests__/dm-backfill.test.ts +417 -0
- package/src/__tests__/dm-persistence.test.ts +227 -0
- package/src/__tests__/docker-signing-key-bootstrap.test.ts +167 -4
- package/src/__tests__/dynamic-skill-workflow-prompt.test.ts +1 -3
- package/src/__tests__/edit-propagation.test.ts +280 -0
- package/src/__tests__/email-html-renderer.test.ts +71 -0
- package/src/__tests__/email-invite-adapter.test.ts +36 -32
- package/src/__tests__/emit-event-signal.test.ts +71 -0
- package/src/__tests__/ephemeral-permissions.test.ts +93 -3
- package/src/__tests__/estimator-calibration-integration.test.ts +208 -0
- package/src/__tests__/estimator-calibration.test.ts +213 -0
- package/src/__tests__/extension-id-sync-guard.test.ts +101 -15
- package/src/__tests__/file-write-tool.test.ts +151 -1
- package/src/__tests__/filing-service.test.ts +255 -0
- package/src/__tests__/fixtures/mock-chrome-extension.ts +11 -0
- package/src/__tests__/gateway-only-enforcement.test.ts +206 -1
- package/src/__tests__/gateway-only-guard.test.ts +0 -1
- package/src/__tests__/gemini-provider.test.ts +64 -3
- package/src/__tests__/get-skill-detail-audit.test.ts +325 -0
- package/src/__tests__/guardian-grant-minting.test.ts +8 -0
- package/src/__tests__/headless-browser-interactions.test.ts +44 -1
- package/src/__tests__/headless-browser-mode.test.ts +614 -0
- package/src/__tests__/headless-browser-navigate.test.ts +142 -5
- package/src/__tests__/headless-browser-read-tools.test.ts +11 -0
- package/src/__tests__/headless-browser-snapshot.test.ts +10 -0
- package/src/__tests__/heartbeat-service.test.ts +166 -32
- package/src/__tests__/home-state-routes.test.ts +162 -0
- package/src/__tests__/host-bash-proxy.test.ts +0 -5
- package/src/__tests__/host-browser-e2e-cloud.test.ts +138 -4
- package/src/__tests__/host-browser-e2e-self-hosted.test.ts +4 -4
- package/src/__tests__/host-browser-ws-events-e2e.test.ts +103 -0
- package/src/__tests__/host-cu-proxy.test.ts +0 -5
- package/src/__tests__/host-shell-tool.test.ts +124 -18
- package/src/__tests__/http-user-message-parity.test.ts +29 -1
- package/src/__tests__/identity-intro-cache.test.ts +40 -10
- package/src/__tests__/inbound-slack-persistence.test.ts +340 -0
- package/src/__tests__/init-feature-flag-overrides.test.ts +38 -112
- package/src/__tests__/intent-routing.test.ts +1 -40
- package/src/__tests__/jobs-store-upsert-debounced.test.ts +141 -0
- package/src/__tests__/llm-catalog-parity.test.ts +174 -0
- package/src/__tests__/llm-context-normalization.test.ts +609 -0
- package/src/__tests__/llm-context-route-provider.test.ts +86 -5
- package/src/__tests__/llm-resolver.test.ts +214 -0
- package/src/__tests__/llm-schema.test.ts +223 -0
- package/src/__tests__/llm-usage-store.test.ts +363 -0
- package/src/__tests__/managed-proxy-context.test.ts +6 -2
- package/src/__tests__/media-stream-output.test.ts +555 -0
- package/src/__tests__/media-stream-parser.test.ts +374 -0
- package/src/__tests__/media-stream-server-integration.test.ts +1234 -0
- package/src/__tests__/media-stream-stt-session.test.ts +588 -0
- package/src/__tests__/media-turn-detector.test.ts +440 -0
- package/src/__tests__/message-queue.test.ts +125 -0
- package/src/__tests__/messaging-skill-split.test.ts +3 -34
- package/src/__tests__/migration-export-http.test.ts +6 -6
- package/src/__tests__/migration-import-commit-http.test.ts +8 -6
- package/src/__tests__/migration-import-from-url.test.ts +684 -0
- package/src/__tests__/migration-import-preflight-http.test.ts +6 -5
- package/src/__tests__/migration-validate-http.test.ts +3 -3
- package/src/__tests__/mock-gateway-ipc.ts +151 -0
- package/src/__tests__/model-intents.test.ts +10 -84
- package/src/__tests__/notification-decision-fallback.test.ts +0 -10
- package/src/__tests__/notification-decision-identity.test.ts +0 -9
- package/src/__tests__/notification-decision-recipient-context.test.ts +0 -9
- package/src/__tests__/oauth-apps-routes.test.ts +1 -0
- package/src/__tests__/oauth-cli.test.ts +2 -0
- package/src/__tests__/oauth-connect-orchestrator.test.ts +2 -0
- package/src/__tests__/oauth-provider-serializer.test.ts +1 -0
- package/src/__tests__/oauth-providers-routes.test.ts +2 -0
- package/src/__tests__/oauth-store.test.ts +95 -7
- package/src/__tests__/oauth2-gateway-transport.test.ts +257 -9
- package/src/__tests__/oauth2-refresh-retry.test.ts +279 -0
- package/src/__tests__/onboarding-template-contract.test.ts +6 -13
- package/src/__tests__/openai-provider.test.ts +183 -0
- package/src/__tests__/openai-responses-cutover-guard.test.ts +184 -0
- package/src/__tests__/openai-responses-provider.test.ts +1501 -0
- package/src/__tests__/openrouter-provider-only.test.ts +135 -0
- package/src/__tests__/openrouter-token-estimation.test.ts +100 -0
- package/src/__tests__/outbound-slack-persistence.test.ts +293 -0
- package/src/__tests__/permission-checker-host-gate.test.ts +1 -1
- package/src/__tests__/permission-mode.test.ts +16 -0
- package/src/__tests__/permission-types.test.ts +0 -1
- package/src/__tests__/persona-resolver.test.ts +251 -0
- package/src/__tests__/pkb-autoinject.test.ts +37 -1
- package/src/__tests__/platform-bash-auto-approve.test.ts +5 -1
- package/src/__tests__/platform.test.ts +92 -1
- package/src/__tests__/post-turn-tool-result-truncation.test.ts +47 -0
- package/src/__tests__/prechat-onboarding-contract.test.ts +267 -0
- package/src/__tests__/pricing.test.ts +224 -3
- package/src/__tests__/profiler-routes.test.ts +1 -1
- package/src/__tests__/provider-commit-message-generator.test.ts +14 -84
- package/src/__tests__/provider-env-vars-scope.test.ts +52 -0
- package/src/__tests__/provider-error-scenarios.test.ts +135 -6
- package/src/__tests__/provider-managed-proxy-integration.test.ts +42 -11
- package/src/__tests__/provider-registry-ollama.test.ts +1 -2
- package/src/__tests__/proxy-approval-callback.test.ts +0 -1
- package/src/__tests__/qdrant-manager.test.ts +29 -8
- package/src/__tests__/reaction-persistence.test.ts +560 -0
- package/src/__tests__/regenerate-fire-and-forget-trace.test.ts +194 -0
- package/src/__tests__/relationship-state-contract.test.ts +175 -0
- package/src/__tests__/relay-server.test.ts +424 -6
- package/src/__tests__/require-fresh-approval.test.ts +1 -1
- package/src/__tests__/retry-openrouter-only-normalization.test.ts +136 -0
- package/src/__tests__/retry-thinking-tool-choice.test.ts +226 -0
- package/src/__tests__/risk-classifier-parity.test.ts +230 -0
- package/src/__tests__/sanitize-config-for-transfer.test.ts +78 -1
- package/src/__tests__/search-skills-unified.test.ts +118 -0
- package/src/__tests__/secret-ingress-http.test.ts +28 -0
- package/src/__tests__/secret-prompter-channel-fallback.test.ts +125 -0
- package/src/__tests__/secret-routes-managed-proxy.test.ts +2 -3
- package/src/__tests__/secret-scanner-executor.test.ts +5 -1
- package/src/__tests__/secure-keys.test.ts +107 -0
- package/src/__tests__/send-endpoint-busy.test.ts +34 -2
- package/src/__tests__/sequence-store.test.ts +1 -1
- package/src/__tests__/server-history-render.test.ts +80 -0
- package/src/__tests__/settings-routes.test.ts +201 -0
- package/src/__tests__/shell-parser-property.test.ts +13 -13
- package/src/__tests__/skill-cache-store.test.ts +182 -0
- package/src/__tests__/skill-load-feature-flag.test.ts +1 -0
- package/src/__tests__/skills-file-content-endpoint.test.ts +276 -145
- package/src/__tests__/skills-files-catalog-fallback.test.ts +381 -93
- package/src/__tests__/skills.test.ts +19 -30
- package/src/__tests__/skillssh-files.test.ts +446 -0
- package/src/__tests__/slack-app-setup-skill-regression.test.ts +3 -1
- package/src/__tests__/slack-block-formatting.test.ts +110 -0
- package/src/__tests__/slack-channel-config.test.ts +564 -1
- package/src/__tests__/slack-skill.test.ts +3 -8
- package/src/__tests__/starter-bundle.test.ts +35 -0
- package/src/__tests__/stt-catalog-parity.test.ts +282 -0
- package/src/__tests__/stt-stream-session.test.ts +535 -0
- package/src/__tests__/subagent-call-site-routing.test.ts +280 -0
- package/src/__tests__/suggestion-routes.test.ts +160 -3
- package/src/__tests__/system-prompt.test.ts +126 -53
- package/src/__tests__/task-runner.test.ts +3 -1
- package/src/__tests__/tcc-sandbox-deny.test.ts +198 -0
- package/src/__tests__/telephony-stt-routing.test.ts +329 -0
- package/src/__tests__/terminal-tools.test.ts +26 -7
- package/src/__tests__/test-preload.ts +18 -0
- package/src/__tests__/test-support/browser-skill-harness.ts +2 -49
- package/src/__tests__/thread-backfill.test.ts +941 -0
- package/src/__tests__/tool-execution-pipeline.benchmark.test.ts +2 -2
- package/src/__tests__/tool-executor-lifecycle-events.test.ts +10 -6
- package/src/__tests__/tool-executor-shell-integration.test.ts +4 -0
- package/src/__tests__/tool-executor.test.ts +88 -113
- package/src/__tests__/tool-result-truncation.test.ts +36 -0
- package/src/__tests__/trust-store.test.ts +442 -103
- package/src/__tests__/trusted-contact-approval-notifier.test.ts +1 -1
- package/src/__tests__/tts-catalog-parity.test.ts +345 -0
- package/src/__tests__/twilio-routes-twiml.test.ts +512 -114
- package/src/__tests__/twilio-routes.test.ts +376 -0
- package/src/__tests__/unicode.test.ts +293 -0
- package/src/__tests__/update-bulletin-job.test.ts +389 -0
- package/src/__tests__/usage-cache-backfill-migration.test.ts +3 -1
- package/src/__tests__/usage-routes.test.ts +25 -4
- package/src/__tests__/user-reference.test.ts +46 -61
- package/src/__tests__/verification-control-plane-policy.test.ts +5 -22
- package/src/__tests__/voice-config-update.test.ts +403 -0
- package/src/__tests__/voice-quality.test.ts +434 -19
- package/src/__tests__/voice-session-bridge.test.ts +39 -0
- package/src/__tests__/volume-security-guard.test.ts +3 -2
- package/src/__tests__/web-search-history.test.ts +337 -0
- package/src/__tests__/workspace-heartbeat-service.test.ts +7 -0
- package/src/__tests__/workspace-migration-033-stt-service-explicit-config.test.ts +547 -0
- package/src/__tests__/workspace-migration-034-remove-calls-voice-transcription-provider.test.ts +596 -0
- package/src/__tests__/workspace-migration-039-drop-legacy-llm-keys.test.ts +343 -0
- package/src/__tests__/workspace-migration-043-release-notes-latex-rendering.test.ts +202 -0
- package/src/__tests__/workspace-migration-045-release-notes-meet-avatar.test.ts +210 -0
- package/src/__tests__/workspace-migration-drop-user-md.test.ts +368 -0
- package/src/__tests__/workspace-migration-meets.test.ts +244 -0
- package/src/__tests__/workspace-migration-seed-device-id.test.ts +14 -20
- package/src/__tests__/workspace-migration-unify-llm-callsite-configs.test.ts +841 -0
- package/src/__tests__/workspace-policy.test.ts +1 -11
- package/src/acp/client-handler.ts +1 -2
- package/src/agent/image-optimize.ts +24 -12
- package/src/agent/loop.ts +251 -19
- package/src/avatar/resvg-lazy.test.ts +136 -0
- package/src/avatar/resvg-lazy.ts +82 -9
- package/src/avatar/traits-png-sync.ts +21 -1
- package/src/backup/__tests__/backup-key.test.ts +152 -0
- package/src/backup/__tests__/backup-worker.test.ts +767 -0
- package/src/backup/__tests__/list-snapshots.test.ts +87 -0
- package/src/backup/__tests__/local-writer.test.ts +218 -0
- package/src/backup/__tests__/offsite-writer.test.ts +641 -0
- package/src/backup/__tests__/paths.test.ts +300 -0
- package/src/backup/__tests__/restore.test.ts +498 -0
- package/src/backup/__tests__/snapshot-lock.test.ts +352 -0
- package/src/backup/__tests__/stream-crypt.test.ts +228 -0
- package/src/backup/backup-key.ts +137 -0
- package/src/backup/backup-worker.ts +459 -0
- package/src/backup/list-snapshots.ts +147 -0
- package/src/backup/local-writer.ts +133 -0
- package/src/backup/offsite-writer.ts +222 -0
- package/src/backup/paths.ts +226 -0
- package/src/backup/restore.ts +322 -0
- package/src/backup/snapshot-lock.ts +431 -0
- package/src/backup/stream-crypt.ts +263 -0
- package/src/browser/__tests__/operations.test.ts +163 -0
- package/src/browser/identifiers.ts +51 -0
- package/src/browser/operations.ts +660 -0
- package/src/browser/types.ts +81 -0
- package/src/bundler/package-resolver.ts +4 -0
- package/src/calls/audio-store.ts +11 -5
- package/src/calls/call-controller.ts +226 -71
- package/src/calls/call-domain.ts +9 -0
- package/src/calls/call-speech-output.ts +190 -0
- package/src/calls/call-transport.ts +77 -0
- package/src/calls/guardian-question-copy.ts +2 -2
- package/src/calls/media-stream-audio-transcode.ts +173 -0
- package/src/calls/media-stream-output.ts +660 -0
- package/src/calls/media-stream-parser.ts +300 -0
- package/src/calls/media-stream-protocol.ts +166 -0
- package/src/calls/media-stream-server.ts +592 -0
- package/src/calls/media-stream-stt-session.ts +460 -0
- package/src/calls/media-turn-detector.ts +230 -0
- package/src/calls/relay-server.ts +90 -75
- package/src/calls/resolve-call-tts-provider.ts +136 -0
- package/src/calls/telephony-stt-routing.ts +145 -0
- package/src/calls/tts-call-strategy.ts +161 -0
- package/src/calls/tts-text-sanitizer.ts +32 -16
- package/src/calls/twilio-routes.ts +281 -17
- package/src/calls/voice-quality.ts +78 -35
- package/src/calls/voice-session-bridge.ts +9 -1
- package/src/channels/types.ts +16 -0
- package/src/cli/AGENTS.md +1 -1
- package/src/cli/__tests__/run-assistant-command.ts +11 -1
- package/src/cli/commands/__tests__/attachment.test.ts +438 -0
- package/src/cli/commands/__tests__/backup.test.ts +1165 -0
- package/src/cli/commands/__tests__/browser.test.ts +554 -0
- package/src/cli/commands/__tests__/cache.test.ts +623 -0
- package/src/cli/commands/__tests__/domain-register.test.ts +234 -0
- package/src/cli/commands/__tests__/domain-status.test.ts +132 -0
- package/src/cli/commands/__tests__/email-attachment.test.ts +422 -0
- package/src/cli/commands/__tests__/email-download.test.ts +16 -1
- package/src/cli/commands/__tests__/email-list.test.ts +28 -4
- package/src/cli/commands/__tests__/email-register.test.ts +4 -4
- package/src/cli/commands/__tests__/email-send.test.ts +130 -5
- package/src/cli/commands/__tests__/email-status.test.ts +5 -1
- package/src/cli/commands/__tests__/email-unregister.test.ts +34 -5
- package/src/cli/commands/__tests__/image-generation.test.ts +666 -0
- package/src/cli/commands/__tests__/inference-send.test.ts +451 -0
- package/src/cli/commands/__tests__/stt-transcribe.test.ts +454 -0
- package/src/cli/commands/__tests__/task.test.ts +913 -0
- package/src/cli/commands/__tests__/tts-synthesize.test.ts +594 -0
- package/src/cli/commands/__tests__/ui-confirm.test.ts +650 -0
- package/src/cli/commands/__tests__/ui.test.ts +1215 -0
- package/src/cli/commands/__tests__/watchers.test.ts +716 -0
- package/src/cli/commands/attachment.ts +182 -0
- package/src/cli/commands/backup.ts +993 -0
- package/src/cli/commands/browser.ts +350 -0
- package/src/cli/commands/cache.ts +341 -0
- package/src/cli/commands/completions.ts +0 -3
- package/src/cli/commands/config.ts +6 -6
- package/src/cli/commands/conversations-import.ts +347 -0
- package/src/cli/commands/conversations.ts +90 -0
- package/src/cli/commands/credentials.ts +0 -1
- package/src/cli/commands/domain.ts +210 -0
- package/src/cli/commands/email.ts +308 -16
- package/src/cli/commands/image-generation.ts +300 -0
- package/src/cli/commands/inference.ts +200 -0
- package/src/cli/commands/memory.ts +127 -17
- package/src/cli/commands/oauth/__tests__/connect.test.ts +12 -0
- package/src/cli/commands/oauth/__tests__/providers-delete.test.ts +1 -0
- package/src/cli/commands/oauth/__tests__/providers-register.test.ts +1 -0
- package/src/cli/commands/oauth/__tests__/providers-update.test.ts +1 -0
- package/src/cli/commands/oauth/mode.ts +12 -3
- package/src/cli/commands/oauth/providers.ts +15 -0
- package/src/cli/commands/oauth/shared.ts +2 -1
- package/src/cli/commands/platform/__tests__/callback-routes-list.test.ts +4 -10
- package/src/cli/commands/platform/__tests__/connect.test.ts +6 -1
- package/src/cli/commands/platform/__tests__/disconnect.test.ts +7 -2
- package/src/cli/commands/platform/__tests__/status.test.ts +6 -1
- package/src/cli/commands/stt.ts +339 -0
- package/src/cli/commands/task.ts +795 -0
- package/src/cli/commands/trust.ts +50 -19
- package/src/cli/commands/tts.ts +273 -0
- package/src/cli/commands/ui.ts +670 -0
- package/src/cli/commands/watchers.ts +509 -0
- package/src/cli/lib/daemon-credential-client.ts +0 -19
- package/src/cli/program.ts +53 -8
- package/src/cli.ts +0 -37
- package/src/config/__tests__/backup-schema.test.ts +134 -0
- package/src/config/assistant-feature-flags.ts +61 -62
- package/src/config/bundled-skills/app-builder/references/CUSTOM_ROUTES.md +37 -1
- package/src/config/bundled-skills/contacts/SKILL.md +2 -2
- package/src/config/bundled-skills/conversations/tools/rename-conversation.ts +23 -1
- package/src/config/bundled-skills/media-processing/SKILL.md +3 -9
- package/src/config/bundled-skills/media-processing/TOOLS.json +1 -6
- package/src/config/bundled-skills/media-processing/__tests__/audio-transcribe.test.ts +125 -0
- package/src/config/bundled-skills/media-processing/__tests__/extract-keyframes.test.ts +181 -0
- package/src/config/bundled-skills/media-processing/__tests__/preprocess-audio.test.ts +141 -0
- package/src/config/bundled-skills/media-processing/services/audio-transcribe.ts +32 -87
- package/src/config/bundled-skills/media-processing/services/preprocess.ts +8 -4
- package/src/config/bundled-skills/media-processing/services/reduce.ts +1 -1
- package/src/config/bundled-skills/media-processing/tools/extract-keyframes.ts +0 -10
- package/src/config/bundled-skills/messaging/SKILL.md +5 -5
- package/src/config/bundled-skills/messaging/TOOLS.json +4 -0
- package/src/config/bundled-skills/messaging/tools/messaging-archive-by-sender.ts +9 -2
- package/src/config/bundled-skills/messaging/tools/messaging-read.ts +15 -1
- package/src/config/bundled-skills/messaging/tools/messaging-search.ts +21 -1
- package/src/config/bundled-skills/messaging/tools/messaging-send.ts +11 -12
- package/src/config/bundled-skills/phone-calls/SKILL.md +2 -2
- package/src/config/bundled-skills/phone-calls/references/CONFIG.md +28 -18
- package/src/config/bundled-skills/phone-calls/references/TROUBLESHOOTING.md +3 -3
- package/src/config/bundled-skills/settings/TOOLS.json +3 -3
- package/src/config/bundled-skills/settings/tools/voice-config-update.ts +26 -22
- package/src/config/bundled-skills/transcribe/SKILL.md +9 -14
- package/src/config/bundled-skills/transcribe/TOOLS.json +2 -7
- package/src/config/bundled-skills/transcribe/tools/transcribe-media.test.ts +256 -0
- package/src/config/bundled-skills/transcribe/tools/transcribe-media.ts +38 -188
- package/src/config/bundled-tool-registry.ts +0 -167
- package/src/config/env-registry.ts +24 -0
- package/src/config/env.ts +39 -10
- package/src/config/feature-flag-registry.json +63 -15
- package/src/config/llm-resolver.ts +128 -0
- package/src/config/loader.ts +220 -22
- package/src/config/raw-config-utils.ts +30 -2
- package/src/config/sanitize-for-transfer.ts +35 -0
- package/src/config/schema.ts +65 -51
- package/src/config/schemas/__tests__/stt.test.ts +43 -0
- package/src/config/schemas/analysis.ts +32 -0
- package/src/config/schemas/backup.ts +72 -0
- package/src/config/schemas/calls.ts +1 -30
- package/src/config/schemas/elevenlabs.ts +0 -59
- package/src/config/schemas/filing.ts +49 -14
- package/src/config/schemas/heartbeat.ts +27 -10
- package/src/config/schemas/host-browser.ts +47 -1
- package/src/config/schemas/inference.ts +3 -23
- package/src/config/schemas/llm.ts +318 -0
- package/src/config/schemas/memory-lifecycle.ts +14 -2
- package/src/config/schemas/memory-processing.ts +1 -9
- package/src/config/schemas/notifications.ts +4 -11
- package/src/config/schemas/platform.ts +3 -9
- package/src/config/schemas/security.ts +33 -0
- package/src/config/schemas/services.ts +53 -4
- package/src/config/schemas/stt.ts +60 -0
- package/src/config/schemas/tts.ts +283 -0
- package/src/config/schemas/updates.ts +14 -0
- package/src/config/schemas/workspace-git.ts +3 -40
- package/src/config/skills.ts +6 -2
- package/src/config/types.ts +4 -0
- package/src/contacts/contact-store.ts +56 -11
- package/src/contacts/contacts-write.ts +38 -1
- package/src/context/__tests__/compact-prompt.test.ts +45 -0
- package/src/context/__tests__/microcompact.test.ts +805 -0
- package/src/context/estimator-calibration.ts +136 -0
- package/src/context/microcompact.ts +443 -0
- package/src/context/post-turn-tool-result-truncation.ts +3 -2
- package/src/context/prompts/compact.md +12 -0
- package/src/context/token-estimator.ts +61 -3
- package/src/context/tool-result-truncation.ts +2 -1
- package/src/context/window-manager.ts +272 -35
- package/src/credential-execution/approval-bridge.ts +0 -1
- package/src/credential-execution/executable-discovery.ts +23 -2
- package/src/credential-execution/process-manager.test.ts +109 -0
- package/src/credential-execution/process-manager.ts +96 -2
- package/src/credential-health/credential-health-service.ts +366 -0
- package/src/daemon/__tests__/conversation-lifecycle-auto-analyze.test.ts +324 -0
- package/src/daemon/__tests__/conversation-surfaces-launch.test.ts +497 -0
- package/src/daemon/__tests__/conversation-tool-setup.test.ts +17 -8
- package/src/daemon/__tests__/lifecycle-startup-ordering.test.ts +127 -0
- package/src/daemon/approval-generators.ts +29 -4
- package/src/daemon/assistant-attachments.ts +24 -13
- package/src/daemon/classifier.ts +2 -2
- package/src/daemon/config-watcher.ts +99 -6
- package/src/daemon/context-overflow-reducer.ts +4 -1
- package/src/daemon/conversation-agent-loop-handlers.ts +85 -12
- package/src/daemon/conversation-agent-loop.ts +563 -104
- package/src/daemon/conversation-attachments.ts +2 -6
- package/src/daemon/conversation-error.ts +46 -0
- package/src/daemon/conversation-history.ts +40 -6
- package/src/daemon/conversation-launch.ts +220 -0
- package/src/daemon/conversation-lifecycle.ts +85 -11
- package/src/daemon/conversation-messaging.ts +110 -7
- package/src/daemon/conversation-notifiers.ts +5 -0
- package/src/daemon/conversation-process.ts +591 -23
- package/src/daemon/conversation-queue-manager.ts +27 -0
- package/src/daemon/conversation-runtime-assembly.ts +769 -28
- package/src/daemon/conversation-slash.ts +38 -2
- package/src/daemon/conversation-surfaces.ts +483 -5
- package/src/daemon/conversation-tool-setup.ts +35 -5
- package/src/daemon/conversation-usage.ts +8 -5
- package/src/daemon/conversation.ts +193 -47
- package/src/daemon/external-skills-bootstrap.ts +41 -0
- package/src/daemon/guardian-action-generators.ts +34 -14
- package/src/daemon/handlers/config-model.test.ts +86 -0
- package/src/daemon/handlers/config-model.ts +54 -12
- package/src/daemon/handlers/config-slack-channel.ts +269 -94
- package/src/daemon/handlers/conversations.ts +13 -3
- package/src/daemon/handlers/shared.ts +51 -1
- package/src/daemon/handlers/skills.ts +323 -79
- package/src/daemon/handlers/slack-channel-oauth-install.ts +197 -0
- package/src/daemon/host-browser-proxy.ts +2 -1
- package/src/daemon/lifecycle.ts +185 -26
- package/src/daemon/message-protocol.ts +6 -0
- package/src/daemon/message-types/conversations.ts +48 -1
- package/src/daemon/message-types/home.ts +40 -0
- package/src/daemon/message-types/meet.ts +143 -0
- package/src/daemon/message-types/messages.ts +23 -1
- package/src/daemon/message-types/schedules.ts +34 -2
- package/src/daemon/message-types/skills.ts +16 -0
- package/src/daemon/message-types/surfaces.ts +2 -0
- package/src/daemon/message-types/trust.ts +0 -2
- package/src/daemon/parse-actual-tokens-from-error.test.ts +57 -1
- package/src/daemon/parse-actual-tokens-from-error.ts +66 -0
- package/src/daemon/pkb-context-tracker.test.ts +169 -0
- package/src/daemon/pkb-context-tracker.ts +125 -0
- package/src/daemon/pkb-reminder-builder.test.ts +70 -0
- package/src/daemon/pkb-reminder-builder.ts +31 -0
- package/src/daemon/providers-setup.ts +6 -0
- package/src/daemon/server.ts +463 -10
- package/src/daemon/shutdown-handlers.ts +32 -4
- package/src/daemon/shutdown-registry.ts +40 -0
- package/src/daemon/tool-side-effects.ts +9 -9
- package/src/daemon/watch-handler.ts +4 -4
- package/src/daemon/web-search-history.ts +126 -0
- package/src/email/html-renderer.ts +76 -0
- package/src/events/domain-events.ts +0 -1
- package/src/filing/filing-service.ts +9 -10
- package/src/heartbeat/heartbeat-service.ts +156 -22
- package/src/home/__tests__/assistant-feed-authoring.test.ts +156 -0
- package/src/home/__tests__/emit-feed-event.test.ts +169 -0
- package/src/home/__tests__/feed-scheduler.test.ts +222 -0
- package/src/home/__tests__/feed-types.test.ts +275 -0
- package/src/home/__tests__/feed-writer.test.ts +688 -0
- package/src/home/__tests__/phase5-exit-criteria.test.ts +212 -0
- package/src/home/__tests__/platform-gmail-digest.test.ts +222 -0
- package/src/home/__tests__/progress-formula.test.ts +213 -0
- package/src/home/__tests__/relationship-state-writer.test.ts +740 -0
- package/src/home/__tests__/rollup-producer.test.ts +442 -0
- package/src/home/assistant-feed-authoring.ts +128 -0
- package/src/home/emit-feed-event.ts +162 -0
- package/src/home/feed-scheduler.ts +263 -0
- package/src/home/feed-types.ts +235 -0
- package/src/home/feed-writer.ts +469 -0
- package/src/home/platform-gmail-digest.ts +163 -0
- package/src/home/progress-formula.ts +86 -0
- package/src/home/relationship-state-writer.ts +824 -0
- package/src/home/relationship-state.ts +143 -0
- package/src/home/rollup-producer.ts +413 -0
- package/src/home/suggested-prompts.ts +101 -0
- package/src/hooks/runner.ts +7 -0
- package/src/inbound/platform-callback-registration.ts +12 -3
- package/src/inbound/public-ingress-urls.ts +12 -0
- package/src/instrument.ts +1 -1
- package/src/ipc/__tests__/attachment-ipc.test.ts +213 -0
- package/src/ipc/__tests__/browser-ipc.test.ts +339 -0
- package/src/ipc/__tests__/cache-ipc.test.ts +266 -0
- package/src/ipc/__tests__/cli-ipc.test.ts +200 -0
- package/src/ipc/__tests__/socket-path.test.ts +73 -0
- package/src/ipc/__tests__/task-ipc.test.ts +577 -0
- package/src/ipc/__tests__/ui-request-route.test.ts +495 -0
- package/src/ipc/__tests__/watcher-ipc.test.ts +295 -0
- package/src/ipc/cli-client.ts +152 -0
- package/src/ipc/cli-server.ts +252 -0
- package/src/ipc/gateway-client.ts +180 -0
- package/src/ipc/routes/attachment.ts +114 -0
- package/src/ipc/routes/browser-context.ts +61 -0
- package/src/ipc/routes/browser.ts +96 -0
- package/src/ipc/routes/cache.ts +96 -0
- package/src/ipc/routes/index.ts +21 -0
- package/src/ipc/routes/task-queue.ts +226 -0
- package/src/ipc/routes/task.ts +173 -0
- package/src/ipc/routes/ui-request.ts +50 -0
- package/src/ipc/routes/wake-conversation.ts +19 -0
- package/src/ipc/routes/watcher.ts +203 -0
- package/src/ipc/socket-path.ts +100 -0
- package/src/memory/__tests__/auto-analysis-enqueue.test.ts +356 -0
- package/src/memory/__tests__/auto-analysis-guard.test.ts +57 -0
- package/src/memory/__tests__/conversation-analyze-job.test.ts +233 -0
- package/src/memory/__tests__/conversation-group-migration.test.ts +99 -0
- package/src/memory/__tests__/find-analysis-conversation.test.ts +196 -0
- package/src/memory/admin.ts +18 -0
- package/src/memory/app-store.ts +1 -1
- package/src/memory/attachments-store.ts +70 -0
- package/src/memory/auto-analysis-enqueue.ts +127 -0
- package/src/memory/auto-analysis-guard.ts +27 -0
- package/src/memory/cleanup-schedule-state.ts +37 -0
- package/src/memory/conversation-analyze-job.ts +74 -0
- package/src/memory/conversation-attention-store.ts +13 -6
- package/src/memory/conversation-crud.ts +199 -0
- package/src/memory/conversation-disk-view.ts +7 -0
- package/src/memory/conversation-group-migration.ts +65 -1
- package/src/memory/conversation-queries.ts +6 -5
- package/src/memory/conversation-title-service.ts +7 -4
- package/src/memory/db-init.ts +8 -0
- package/src/memory/db-maintenance.ts +108 -0
- package/src/memory/db.ts +1 -0
- package/src/memory/embedding-backend.ts +1 -1
- package/src/memory/graph/compaction.ts +299 -0
- package/src/memory/graph/consolidation.ts +4 -4
- package/src/memory/graph/conversation-graph-memory.ts +104 -29
- package/src/memory/graph/extraction.test.ts +295 -2
- package/src/memory/graph/extraction.ts +181 -51
- package/src/memory/graph/graph-search.test.ts +92 -0
- package/src/memory/graph/graph-search.ts +4 -1
- package/src/memory/graph/narrative.ts +2 -2
- package/src/memory/graph/pattern-scan.ts +2 -2
- package/src/memory/graph/retriever.test.ts +459 -0
- package/src/memory/graph/retriever.ts +257 -66
- package/src/memory/graph/scoring.test.ts +186 -0
- package/src/memory/graph/scoring.ts +31 -1
- package/src/memory/graph/store.ts +41 -0
- package/src/memory/graph/tool-handlers.ts +27 -0
- package/src/memory/graph/tools.ts +6 -1
- package/src/memory/group-crud.ts +6 -1
- package/src/memory/indexer.ts +95 -16
- package/src/memory/job-handlers/cleanup.ts +11 -8
- package/src/memory/job-handlers/conversation-starters.ts +39 -30
- package/src/memory/job-handlers/summarization.ts +2 -2
- package/src/memory/job-utils.ts +7 -1
- package/src/memory/jobs/embed-pkb-file.test.ts +168 -0
- package/src/memory/jobs/embed-pkb-file.ts +54 -0
- package/src/memory/jobs-store.ts +106 -5
- package/src/memory/jobs-worker.ts +26 -9
- package/src/memory/llm-usage-store.ts +92 -56
- package/src/memory/migrations/140-backfill-usage-cache-accounting.ts +1 -1
- package/src/memory/migrations/219-oauth-providers-token-exchange-body-format.ts +15 -0
- package/src/memory/migrations/220-normalize-user-file-by-principal.ts +190 -0
- package/src/memory/migrations/221-conversations-archived-at.ts +16 -0
- package/src/memory/migrations/222-strip-placeholder-sentinels-from-messages.ts +82 -0
- package/src/memory/migrations/index.ts +7 -0
- package/src/memory/migrations/registry.ts +8 -0
- package/src/memory/pkb/pkb-index.test.ts +368 -0
- package/src/memory/pkb/pkb-index.ts +255 -0
- package/src/memory/pkb/pkb-reconcile.test.ts +251 -0
- package/src/memory/pkb/pkb-reconcile.ts +148 -0
- package/src/memory/pkb/pkb-search.test.ts +438 -0
- package/src/memory/pkb/pkb-search.ts +137 -0
- package/src/memory/pkb/types.ts +53 -0
- package/src/memory/qdrant-client.ts +122 -1
- package/src/memory/qdrant-manager.ts +43 -16
- package/src/memory/schema/conversations.ts +2 -0
- package/src/memory/schema/oauth.ts +3 -0
- package/src/memory/slack-thread-store.ts +37 -0
- package/src/memory/usage-buckets.ts +396 -0
- package/src/messaging/providers/gmail/adapter.ts +6 -16
- package/src/messaging/providers/gmail/client.ts +79 -6
- package/src/messaging/providers/gmail/types.ts +7 -0
- package/src/messaging/providers/slack/__tests__/adapter-token-routing.test.ts +282 -0
- package/src/messaging/providers/slack/adapter.ts +155 -38
- package/src/messaging/providers/slack/backfill.test.ts +257 -0
- package/src/messaging/providers/slack/backfill.ts +101 -0
- package/src/messaging/providers/slack/client.ts +16 -0
- package/src/messaging/providers/slack/message-metadata.test.ts +316 -0
- package/src/messaging/providers/slack/message-metadata.ts +123 -0
- package/src/messaging/providers/slack/render-transcript.test.ts +1373 -0
- package/src/messaging/providers/slack/render-transcript.ts +443 -0
- package/src/messaging/providers/slack/types.ts +4 -0
- package/src/messaging/style-analyzer.ts +5 -2
- package/src/notifications/README.md +9 -5
- package/src/notifications/decision-engine.ts +6 -12
- package/src/notifications/preference-extractor.ts +2 -6
- package/src/notifications/signal.ts +5 -0
- package/src/oauth/__tests__/identity-verifier.test.ts +1 -0
- package/src/oauth/byo-connection.test.ts +18 -1
- package/src/oauth/byo-connection.ts +3 -1
- package/src/oauth/connect-orchestrator.ts +2 -0
- package/src/oauth/connection-resolver.ts +6 -2
- package/src/oauth/connection.ts +2 -0
- package/src/oauth/oauth-store.ts +10 -0
- package/src/oauth/platform-connection.test.ts +145 -0
- package/src/oauth/platform-connection.ts +62 -31
- package/src/oauth/seed-providers.ts +10 -1
- package/src/permissions/approval-policy.test.ts +948 -0
- package/src/permissions/approval-policy.ts +257 -0
- package/src/permissions/bash-risk-classifier.test.ts +1208 -0
- package/src/permissions/bash-risk-classifier.ts +707 -0
- package/src/permissions/checker.ts +218 -699
- package/src/permissions/command-registry.test.ts +535 -0
- package/src/permissions/command-registry.ts +825 -0
- package/src/permissions/defaults.ts +71 -75
- package/src/permissions/file-risk-classifier.test.ts +535 -0
- package/src/permissions/file-risk-classifier.ts +274 -0
- package/src/permissions/risk-types.ts +205 -0
- package/src/permissions/secret-prompter.ts +53 -2
- package/src/permissions/skill-risk-classifier.test.ts +311 -0
- package/src/permissions/skill-risk-classifier.ts +214 -0
- package/src/permissions/trust-client.ts +52 -25
- package/src/permissions/trust-store-interface.ts +1 -6
- package/src/permissions/trust-store.ts +164 -65
- package/src/permissions/types.ts +23 -14
- package/src/permissions/web-risk-classifier.test.ts +170 -0
- package/src/permissions/web-risk-classifier.ts +89 -0
- package/src/permissions/workspace-policy.ts +1 -13
- package/src/platform/client.test.ts +10 -0
- package/src/platform/client.ts +19 -1
- package/src/platform/sync-identity.ts +129 -0
- package/src/prompts/persona-resolver.ts +127 -3
- package/src/prompts/system-prompt.ts +78 -38
- package/src/prompts/templates/BOOTSTRAP.md +5 -5
- package/src/prompts/templates/SOUL.md +5 -3
- package/src/prompts/templates/channels/slack.md +20 -0
- package/src/prompts/update-bulletin-job.ts +190 -0
- package/src/prompts/user-reference.ts +20 -17
- package/src/providers/__tests__/context-overflow-error.test.ts +328 -0
- package/src/providers/__tests__/provider-env-vars.test.ts +102 -0
- package/src/providers/__tests__/provider-secret-catalog.test.ts +42 -0
- package/src/providers/__tests__/retry-callsite.test.ts +424 -0
- package/src/providers/anthropic/client.ts +335 -70
- package/src/providers/call-site-routing.ts +71 -0
- package/src/providers/fireworks/client.ts +2 -2
- package/src/providers/gemini/client.ts +74 -3
- package/src/providers/managed-proxy/constants.ts +2 -1
- package/src/providers/model-catalog.ts +502 -28
- package/src/providers/model-intents.ts +8 -8
- package/src/providers/ollama/client.ts +2 -2
- package/src/providers/openai/chat-completions-provider.ts +530 -0
- package/src/providers/openai/client.ts +25 -440
- package/src/providers/openai/responses-provider.ts +579 -0
- package/src/providers/openrouter/client.ts +168 -4
- package/src/providers/provider-env-vars.ts +56 -0
- package/src/providers/provider-secret-catalog.ts +139 -0
- package/src/providers/provider-send-message.ts +22 -5
- package/src/providers/ratelimit.ts +4 -0
- package/src/providers/registry.ts +21 -10
- package/src/providers/retry.ts +185 -39
- package/src/providers/speech-to-text/__tests__/provider-catalog.test.ts +251 -0
- package/src/providers/speech-to-text/__tests__/resolve.test.ts +883 -0
- package/src/providers/speech-to-text/deepgram-realtime.test.ts +980 -0
- package/src/providers/speech-to-text/deepgram-realtime.ts +767 -0
- package/src/providers/speech-to-text/deepgram.test.ts +332 -0
- package/src/providers/speech-to-text/deepgram.ts +115 -0
- package/src/providers/speech-to-text/google-gemini-live-stream.test.ts +743 -0
- package/src/providers/speech-to-text/google-gemini-live-stream.ts +625 -0
- package/src/providers/speech-to-text/google-gemini.test.ts +226 -0
- package/src/providers/speech-to-text/google-gemini.ts +101 -0
- package/src/providers/speech-to-text/openai-whisper-stream.test.ts +564 -0
- package/src/providers/speech-to-text/openai-whisper-stream.ts +381 -0
- package/src/providers/speech-to-text/openai-whisper.test.ts +1 -37
- package/src/providers/speech-to-text/openai-whisper.ts +63 -33
- package/src/providers/speech-to-text/provider-catalog.ts +323 -0
- package/src/providers/speech-to-text/resolve.ts +393 -6
- package/src/providers/speech-to-text/xai-realtime.test.ts +578 -0
- package/src/providers/speech-to-text/xai-realtime.ts +796 -0
- package/src/providers/speech-to-text/xai.test.ts +155 -0
- package/src/providers/speech-to-text/xai.ts +97 -0
- package/src/providers/types.ts +102 -3
- package/src/runtime/AGENTS.md +45 -3
- package/src/runtime/__tests__/agent-wake.test.ts +872 -0
- package/src/runtime/__tests__/interactive-ui.test.ts +673 -0
- package/src/runtime/__tests__/runtime-mode.test.ts +62 -0
- package/src/runtime/__tests__/slack-block-formatting.test.ts +481 -0
- package/src/runtime/agent-wake.ts +553 -0
- package/src/runtime/auth/__tests__/route-policy.test.ts +40 -0
- package/src/runtime/auth/route-policy.ts +34 -5
- package/src/runtime/auth/token-service.ts +56 -1
- package/src/runtime/btw-sidechain.ts +15 -3
- package/src/runtime/capability-tokens.ts +10 -10
- package/src/runtime/channel-invite-transport.ts +1 -1
- package/src/runtime/channel-invite-transports/email.ts +14 -6
- package/src/runtime/channel-readiness-service.ts +12 -22
- package/src/runtime/channel-reply-delivery.ts +106 -2
- package/src/runtime/chrome-extension-registry.ts +38 -2
- package/src/runtime/decision-token.ts +116 -0
- package/src/runtime/gateway-client.ts +2 -2
- package/src/runtime/http-router.ts +32 -0
- package/src/runtime/http-server.ts +447 -11
- package/src/runtime/http-types.ts +29 -3
- package/src/runtime/interactive-ui.ts +362 -0
- package/src/runtime/invite-instruction-generator.ts +2 -2
- package/src/runtime/migrations/__tests__/gcs-signed-url.test.ts +176 -0
- package/src/runtime/migrations/__tests__/vbundle-import-credentials.test.ts +36 -0
- package/src/runtime/migrations/__tests__/vbundle-legacy-user-md.test.ts +360 -0
- package/src/runtime/migrations/__tests__/vbundle-metadata-merge-integration.test.ts +390 -0
- package/src/runtime/migrations/__tests__/vbundle-metadata-merge.test.ts +221 -0
- package/src/runtime/migrations/__tests__/vbundle-streaming-importer.test.ts +1540 -0
- package/src/runtime/migrations/__tests__/vbundle-streaming-validator.test.ts +453 -0
- package/src/runtime/migrations/__tests__/vbundle-tar-stream.test.ts +222 -0
- package/src/runtime/migrations/gcs-signed-url.ts +162 -0
- package/src/runtime/migrations/migration-transport.ts +1 -0
- package/src/runtime/migrations/migration-wizard.ts +1 -0
- package/src/runtime/migrations/vbundle-import-analyzer.ts +77 -1
- package/src/runtime/migrations/vbundle-importer.ts +187 -8
- package/src/runtime/migrations/vbundle-metadata-merge.ts +124 -0
- package/src/runtime/migrations/vbundle-streaming-importer.ts +2522 -0
- package/src/runtime/migrations/vbundle-streaming-validator.ts +244 -0
- package/src/runtime/migrations/vbundle-tar-stream.ts +217 -0
- package/src/runtime/migrations/vbundle-validator.ts +15 -6
- package/src/runtime/pending-interactions.ts +0 -11
- package/src/runtime/routes/__tests__/backup-routes.test.ts +967 -0
- package/src/runtime/routes/__tests__/home-feed-routes.test.ts +618 -0
- package/src/runtime/routes/__tests__/migration-import-credential-filter.test.ts +247 -0
- package/src/runtime/routes/__tests__/migration-vellum-metadata-reconcile.test.ts +246 -0
- package/src/runtime/routes/__tests__/stt-routes.test.ts +406 -0
- package/src/runtime/routes/__tests__/tts-routes.test.ts +474 -0
- package/src/runtime/routes/__tests__/user-route-dispatcher.test.ts +148 -17
- package/src/runtime/routes/app-management-routes.ts +12 -18
- package/src/runtime/routes/approval-prompt-ts-tracker.ts +58 -0
- package/src/runtime/routes/approval-routes.ts +12 -17
- package/src/runtime/routes/approval-strategies/guardian-callback-strategy.ts +9 -0
- package/src/runtime/routes/attachment-routes.test.ts +9 -3
- package/src/runtime/routes/attachment-routes.ts +216 -17
- package/src/runtime/routes/avatar-routes.ts +20 -4
- package/src/runtime/routes/backup-routes.ts +519 -0
- package/src/runtime/routes/browser-extension-pair-routes.ts +82 -23
- package/src/runtime/routes/btw-routes.ts +9 -10
- package/src/runtime/routes/contact-routes.test.ts +298 -0
- package/src/runtime/routes/contact-routes.ts +132 -5
- package/src/runtime/routes/conversation-analysis-routes.ts +22 -142
- package/src/runtime/routes/conversation-management-routes.ts +133 -0
- package/src/runtime/routes/conversation-routes.ts +487 -160
- package/src/runtime/routes/debug-routes.ts +1 -1
- package/src/runtime/routes/diagnostics-routes.ts +6 -4
- package/src/runtime/routes/events-routes.ts +16 -0
- package/src/runtime/routes/filing-routes.ts +93 -0
- package/src/runtime/routes/guardian-approval-interception.ts +33 -3
- package/src/runtime/routes/guardian-approval-prompt.ts +13 -3
- package/src/runtime/routes/home-feed-routes.ts +452 -0
- package/src/runtime/routes/home-state-routes.ts +138 -0
- package/src/runtime/routes/host-browser-routes.ts +3 -14
- package/src/runtime/routes/identity-intro-cache.ts +7 -3
- package/src/runtime/routes/identity-routes.ts +3 -17
- package/src/runtime/routes/inbound-message-handler.ts +912 -2
- package/src/runtime/routes/inbound-stages/background-dispatch.test.ts +113 -2
- package/src/runtime/routes/inbound-stages/background-dispatch.ts +61 -3
- package/src/runtime/routes/inbound-stages/edit-intercept.ts +129 -6
- package/src/runtime/routes/inbound-stages/transcribe-audio.test.ts +46 -39
- package/src/runtime/routes/inbound-stages/transcribe-audio.ts +15 -15
- package/src/runtime/routes/integrations/slack/__tests__/channel.test.ts +137 -0
- package/src/runtime/routes/integrations/slack/__tests__/share.test.ts +179 -0
- package/src/runtime/routes/integrations/slack/channel.ts +36 -6
- package/src/runtime/routes/integrations/slack/share.ts +45 -7
- package/src/runtime/routes/llm-context-normalization.ts +325 -0
- package/src/runtime/routes/memory-item-routes.test.ts +3 -2
- package/src/runtime/routes/migration-routes.ts +722 -91
- package/src/runtime/routes/settings-routes.ts +26 -7
- package/src/runtime/routes/skills-routes.ts +76 -7
- package/src/runtime/routes/stt-routes.ts +233 -0
- package/src/runtime/routes/surface-action-routes.ts +41 -2
- package/src/runtime/routes/trust-rules-routes.ts +30 -14
- package/src/runtime/routes/tts-routes.ts +108 -24
- package/src/runtime/routes/usage-routes.ts +30 -2
- package/src/runtime/routes/user-route-dispatcher.ts +50 -5
- package/src/runtime/routes/user-routes.ts +13 -1
- package/src/runtime/routes/work-items-routes.test.ts +1 -1
- package/src/runtime/routes/work-items-routes.ts +11 -3
- package/src/runtime/runtime-mode.ts +33 -0
- package/src/runtime/services/__tests__/analyze-conversation.test.ts +426 -0
- package/src/runtime/services/__tests__/analyze-deps-singleton.test.ts +67 -0
- package/src/runtime/services/__tests__/auto-analysis-prompt.test.ts +53 -0
- package/src/runtime/services/__tests__/manual-analysis-prompt.test.ts +41 -0
- package/src/runtime/services/analyze-conversation.ts +340 -0
- package/src/runtime/services/analyze-deps-singleton.ts +32 -0
- package/src/runtime/services/auto-analysis-prompt.ts +55 -0
- package/src/runtime/skill-route-registry.ts +71 -0
- package/src/runtime/slack-block-formatting.ts +437 -10
- package/src/schedule/scheduler.ts +58 -0
- package/src/security/__tests__/provider-key-env-fallback.test.ts +119 -0
- package/src/security/__tests__/untrusted-content.test.ts +109 -0
- package/src/security/oauth2.ts +122 -37
- package/src/security/secure-keys.ts +32 -10
- package/src/security/token-manager.ts +35 -13
- package/src/security/untrusted-content.ts +102 -0
- package/src/sequence/engine.ts +23 -0
- package/src/sequence/types.ts +1 -1
- package/src/skills/catalog-cache.ts +26 -7
- package/src/skills/catalog-files.ts +64 -2
- package/src/skills/catalog-install.ts +31 -3
- package/src/skills/category-inference.ts +122 -0
- package/src/skills/clawhub-files.ts +213 -0
- package/src/skills/clawhub.ts +84 -23
- package/src/skills/skill-cache-store.ts +97 -0
- package/src/skills/skill-file-provider.ts +40 -0
- package/src/skills/skillssh-files.ts +395 -0
- package/src/skills/skillssh-registry.ts +4 -4
- package/src/stt/__tests__/daemon-batch-transcriber.test.ts +468 -0
- package/src/stt/__tests__/types.test.ts +89 -0
- package/src/stt/daemon-batch-transcriber.ts +228 -0
- package/src/stt/stt-stream-session.ts +506 -0
- package/src/stt/types.ts +334 -0
- package/src/stt/wav-encoder.test.ts +373 -0
- package/src/stt/wav-encoder.ts +175 -0
- package/src/subagent/manager.ts +79 -27
- package/src/tasks/ephemeral-permissions.ts +9 -4
- package/src/telemetry/usage-telemetry-reporter.ts +27 -5
- package/src/tools/browser/__tests__/browser-mode.test.ts +119 -0
- package/src/tools/browser/__tests__/browser-status.test.ts +166 -0
- package/src/tools/browser/browser-execution.ts +1208 -41
- package/src/tools/browser/browser-manager.ts +45 -0
- package/src/tools/browser/browser-mode-constants.ts +12 -0
- package/src/tools/browser/browser-mode.ts +92 -0
- package/src/tools/browser/browser-status-constants.ts +33 -0
- package/src/tools/browser/cdp-client/__tests__/cdp-inspect-client.test.ts +393 -0
- package/src/tools/browser/cdp-client/__tests__/extension-cdp-client.test.ts +29 -0
- package/src/tools/browser/cdp-client/__tests__/factory.test.ts +1648 -32
- package/src/tools/browser/cdp-client/cdp-inspect/__tests__/discovery.test.ts +264 -0
- package/src/tools/browser/cdp-client/cdp-inspect/discovery.ts +205 -17
- package/src/tools/browser/cdp-client/cdp-inspect-client.ts +254 -21
- package/src/tools/browser/cdp-client/errors.ts +15 -0
- package/src/tools/browser/cdp-client/extension-cdp-client.ts +39 -16
- package/src/tools/browser/cdp-client/factory.ts +797 -87
- package/src/tools/browser/cdp-client/index.ts +16 -2
- package/src/tools/browser/cdp-client/types.ts +68 -0
- package/src/tools/credentials/tool-policy.ts +39 -5
- package/src/tools/credentials/vault.ts +41 -7
- package/src/tools/executor.ts +4 -0
- package/src/tools/filesystem/write.ts +52 -0
- package/src/tools/host-terminal/host-shell.ts +45 -5
- package/src/tools/memory/register.test.ts +185 -0
- package/src/tools/memory/register.ts +3 -1
- package/src/tools/network/web-fetch.ts +25 -12
- package/src/tools/network/web-search.ts +20 -2
- package/src/tools/permission-checker.ts +36 -15
- package/src/tools/policy-context.ts +25 -8
- package/src/tools/registry.ts +55 -3
- package/src/tools/shared/shell-output.ts +3 -1
- package/src/tools/side-effects.ts +0 -9
- package/src/tools/skills/execute.ts +2 -2
- package/src/tools/skills/sandbox-runner.ts +6 -2
- package/src/tools/terminal/backends/native.ts +51 -2
- package/src/tools/terminal/safe-env.ts +11 -2
- package/src/tools/terminal/shell.ts +16 -4
- package/src/tools/tool-manifest.ts +6 -0
- package/src/tools/types.ts +29 -3
- package/src/tools/ui-surface/definitions.ts +6 -1
- package/src/tools/verification-control-plane-policy.ts +1 -1
- package/src/tts/__tests__/provider-adapters.test.ts +1061 -0
- package/src/tts/__tests__/provider-catalog-consistency.test.ts +196 -0
- package/src/tts/__tests__/provider-catalog.test.ts +183 -0
- package/src/tts/__tests__/provider-registry.test.ts +90 -0
- package/src/tts/provider-catalog.ts +219 -0
- package/src/tts/provider-registry.ts +73 -0
- package/src/tts/providers/deepgram-provider.ts +219 -0
- package/src/tts/providers/elevenlabs-provider.ts +211 -0
- package/src/tts/providers/fish-audio-provider.ts +183 -0
- package/src/tts/providers/index.ts +44 -0
- package/src/tts/providers/register-builtins.ts +130 -0
- package/src/tts/providers/xai-provider.ts +224 -0
- package/src/tts/synthesize-text.ts +110 -0
- package/src/tts/tts-config-resolver.ts +78 -0
- package/src/tts/types.ts +199 -0
- package/src/types/onboarding-context.ts +7 -0
- package/src/types/tar-stream.d.ts +66 -0
- package/src/util/abort-reasons.ts +58 -0
- package/src/util/device-id.ts +32 -16
- package/src/util/errors.ts +9 -1
- package/src/util/json.ts +17 -0
- package/src/util/platform.ts +56 -12
- package/src/util/pricing.ts +78 -5
- package/src/util/spawn.ts +1 -1
- package/src/util/truncate.ts +4 -2
- package/src/util/unicode.ts +201 -0
- package/src/version.ts +19 -24
- package/src/watcher/engine.ts +24 -1
- package/src/watcher/providers/google-calendar.ts +134 -8
- package/src/watcher/providers/outlook-calendar.ts +42 -2
- package/src/watcher/watcher-store.ts +31 -0
- package/src/workspace/git-service.ts +23 -4
- package/src/workspace/migrations/003-seed-device-id.ts +9 -3
- package/src/workspace/migrations/017-seed-persona-dirs.ts +68 -4
- package/src/workspace/migrations/029-seed-pkb.ts +1 -1
- package/src/workspace/migrations/031-drop-user-md.ts +317 -0
- package/src/workspace/migrations/031-llm-log-retention-zero-to-null.ts +73 -0
- package/src/workspace/migrations/032-tts-provider-unification.ts +227 -0
- package/src/workspace/migrations/033-stt-service-explicit-config.ts +122 -0
- package/src/workspace/migrations/034-remove-calls-voice-transcription-provider.ts +215 -0
- package/src/workspace/migrations/035-seed-slack-channel-persona.ts +50 -0
- package/src/workspace/migrations/036-update-pkb-index-bar.ts +37 -0
- package/src/workspace/migrations/037-create-meets-dir.ts +61 -0
- package/src/workspace/migrations/038-unify-llm-callsite-configs.ts +516 -0
- package/src/workspace/migrations/039-drop-legacy-llm-keys.ts +171 -0
- package/src/workspace/migrations/040-seed-latency-callsite-defaults.ts +154 -0
- package/src/workspace/migrations/041-backfill-google-gmail-settings-scope.ts +57 -0
- package/src/workspace/migrations/042-fix-backfill-google-gmail-settings-scope.ts +70 -0
- package/src/workspace/migrations/043-release-notes-latex-rendering.ts +75 -0
- package/src/workspace/migrations/044-bump-stale-provider-stream-timeout.ts +51 -0
- package/src/workspace/migrations/045-release-notes-meet-avatar.ts +130 -0
- package/src/workspace/migrations/AGENTS.md +1 -1
- package/src/workspace/migrations/registry.ts +32 -0
- package/src/workspace/provider-commit-message-generator.ts +19 -38
- package/src/workspace/top-level-renderer.ts +13 -1
- package/src/workspace/turn-commit.ts +31 -0
- package/src/__tests__/email-cli.test.ts +0 -297
- package/src/__tests__/email-service-config-fallback.test.ts +0 -102
- package/src/__tests__/outlook-attachments.test.ts +0 -301
- package/src/__tests__/outlook-automation-tools.test.ts +0 -425
- package/src/__tests__/outlook-categories.test.ts +0 -212
- package/src/__tests__/outlook-compose-tools.test.ts +0 -325
- package/src/__tests__/outlook-declutter-tools.test.ts +0 -585
- package/src/__tests__/outlook-follow-up.test.ts +0 -196
- package/src/__tests__/outlook-trash.test.ts +0 -77
- package/src/__tests__/outlook-unsubscribe.test.ts +0 -250
- package/src/__tests__/update-bulletin-format.test.ts +0 -122
- package/src/__tests__/update-bulletin-state.test.ts +0 -135
- package/src/__tests__/update-bulletin.test.ts +0 -277
- package/src/__tests__/update-template-contract.test.ts +0 -29
- package/src/cli/commands/browser-relay.ts +0 -466
- package/src/cli/commands/doctor.ts +0 -341
- package/src/config/bundled-skills/browser/SKILL.md +0 -63
- package/src/config/bundled-skills/browser/TOOLS.json +0 -393
- package/src/config/bundled-skills/browser/tools/browser-click.ts +0 -12
- package/src/config/bundled-skills/browser/tools/browser-close.ts +0 -12
- package/src/config/bundled-skills/browser/tools/browser-extract.ts +0 -12
- package/src/config/bundled-skills/browser/tools/browser-fill-credential.ts +0 -12
- package/src/config/bundled-skills/browser/tools/browser-hover.ts +0 -12
- package/src/config/bundled-skills/browser/tools/browser-navigate.ts +0 -12
- package/src/config/bundled-skills/browser/tools/browser-press-key.ts +0 -12
- package/src/config/bundled-skills/browser/tools/browser-screenshot.ts +0 -12
- package/src/config/bundled-skills/browser/tools/browser-scroll.ts +0 -12
- package/src/config/bundled-skills/browser/tools/browser-select-option.ts +0 -12
- package/src/config/bundled-skills/browser/tools/browser-snapshot.ts +0 -12
- package/src/config/bundled-skills/browser/tools/browser-type.ts +0 -12
- package/src/config/bundled-skills/browser/tools/browser-wait-for-download.ts +0 -32
- package/src/config/bundled-skills/browser/tools/browser-wait-for.ts +0 -12
- package/src/config/bundled-skills/chatgpt-import/SKILL.md +0 -27
- package/src/config/bundled-skills/chatgpt-import/TOOLS.json +0 -27
- package/src/config/bundled-skills/chatgpt-import/tools/chatgpt-import.ts +0 -378
- package/src/config/bundled-skills/gmail/SKILL.md +0 -175
- package/src/config/bundled-skills/gmail/TOOLS.json +0 -558
- package/src/config/bundled-skills/gmail/tools/gmail-archive.ts +0 -149
- package/src/config/bundled-skills/gmail/tools/gmail-attachments.ts +0 -112
- package/src/config/bundled-skills/gmail/tools/gmail-draft.ts +0 -44
- package/src/config/bundled-skills/gmail/tools/gmail-filters.ts +0 -81
- package/src/config/bundled-skills/gmail/tools/gmail-follow-up.ts +0 -108
- package/src/config/bundled-skills/gmail/tools/gmail-forward.ts +0 -146
- package/src/config/bundled-skills/gmail/tools/gmail-label.ts +0 -53
- package/src/config/bundled-skills/gmail/tools/gmail-outreach-scan.ts +0 -220
- package/src/config/bundled-skills/gmail/tools/gmail-send-draft.ts +0 -26
- package/src/config/bundled-skills/gmail/tools/gmail-sender-digest.ts +0 -251
- package/src/config/bundled-skills/gmail/tools/gmail-trash.ts +0 -29
- package/src/config/bundled-skills/gmail/tools/gmail-unsubscribe.ts +0 -122
- package/src/config/bundled-skills/gmail/tools/gmail-vacation.ts +0 -67
- package/src/config/bundled-skills/gmail/tools/scan-result-store.ts +0 -100
- package/src/config/bundled-skills/gmail/tools/shared.ts +0 -47
- package/src/config/bundled-skills/google-calendar/SKILL.md +0 -51
- package/src/config/bundled-skills/google-calendar/TOOLS.json +0 -226
- package/src/config/bundled-skills/google-calendar/calendar-client.ts +0 -223
- package/src/config/bundled-skills/google-calendar/tools/calendar-check-availability.ts +0 -27
- package/src/config/bundled-skills/google-calendar/tools/calendar-create-event.ts +0 -48
- package/src/config/bundled-skills/google-calendar/tools/calendar-get-event.ts +0 -19
- package/src/config/bundled-skills/google-calendar/tools/calendar-list-events.ts +0 -36
- package/src/config/bundled-skills/google-calendar/tools/calendar-rsvp.ts +0 -58
- package/src/config/bundled-skills/google-calendar/tools/shared.ts +0 -17
- package/src/config/bundled-skills/google-calendar/types.ts +0 -97
- package/src/config/bundled-skills/outlook/SKILL.md +0 -196
- package/src/config/bundled-skills/outlook/TOOLS.json +0 -530
- package/src/config/bundled-skills/outlook/tools/outlook-attachments.ts +0 -85
- package/src/config/bundled-skills/outlook/tools/outlook-categories.ts +0 -77
- package/src/config/bundled-skills/outlook/tools/outlook-draft.ts +0 -84
- package/src/config/bundled-skills/outlook/tools/outlook-follow-up.ts +0 -94
- package/src/config/bundled-skills/outlook/tools/outlook-forward.ts +0 -49
- package/src/config/bundled-skills/outlook/tools/outlook-outreach-scan.ts +0 -237
- package/src/config/bundled-skills/outlook/tools/outlook-rules.ts +0 -161
- package/src/config/bundled-skills/outlook/tools/outlook-send-draft.ts +0 -32
- package/src/config/bundled-skills/outlook/tools/outlook-sender-digest.ts +0 -272
- package/src/config/bundled-skills/outlook/tools/outlook-trash.ts +0 -29
- package/src/config/bundled-skills/outlook/tools/outlook-unsubscribe.ts +0 -129
- package/src/config/bundled-skills/outlook/tools/outlook-vacation.ts +0 -87
- package/src/config/bundled-skills/outlook/tools/shared.ts +0 -20
- package/src/config/bundled-skills/outlook-calendar/SKILL.md +0 -51
- package/src/config/bundled-skills/outlook-calendar/TOOLS.json +0 -221
- package/src/config/bundled-skills/outlook-calendar/calendar-client.ts +0 -252
- package/src/config/bundled-skills/outlook-calendar/tools/outlook-calendar-check-availability.ts +0 -53
- package/src/config/bundled-skills/outlook-calendar/tools/outlook-calendar-create-event.ts +0 -74
- package/src/config/bundled-skills/outlook-calendar/tools/outlook-calendar-get-event.ts +0 -18
- package/src/config/bundled-skills/outlook-calendar/tools/outlook-calendar-list-events.ts +0 -46
- package/src/config/bundled-skills/outlook-calendar/tools/outlook-calendar-rsvp.ts +0 -36
- package/src/config/bundled-skills/outlook-calendar/tools/shared.ts +0 -17
- package/src/config/bundled-skills/outlook-calendar/types.ts +0 -120
- package/src/config/bundled-skills/slack/SKILL.md +0 -107
- package/src/config/bundled-skills/tasks/SKILL.md +0 -37
- package/src/config/bundled-skills/tasks/TOOLS.json +0 -353
- package/src/config/bundled-skills/tasks/icon.svg +0 -34
- package/src/config/bundled-skills/tasks/tools/task-delete.ts +0 -12
- package/src/config/bundled-skills/tasks/tools/task-list-add.ts +0 -12
- package/src/config/bundled-skills/tasks/tools/task-list-remove.ts +0 -12
- package/src/config/bundled-skills/tasks/tools/task-list-show.ts +0 -12
- package/src/config/bundled-skills/tasks/tools/task-list-update.ts +0 -12
- package/src/config/bundled-skills/tasks/tools/task-list.ts +0 -12
- package/src/config/bundled-skills/tasks/tools/task-queue-run.ts +0 -12
- package/src/config/bundled-skills/tasks/tools/task-run.ts +0 -12
- package/src/config/bundled-skills/tasks/tools/task-save.ts +0 -12
- package/src/config/bundled-skills/watcher/SKILL.md +0 -31
- package/src/config/bundled-skills/watcher/TOOLS.json +0 -167
- package/src/config/bundled-skills/watcher/tools/watcher-create.ts +0 -12
- package/src/config/bundled-skills/watcher/tools/watcher-delete.ts +0 -12
- package/src/config/bundled-skills/watcher/tools/watcher-digest.ts +0 -12
- package/src/config/bundled-skills/watcher/tools/watcher-list.ts +0 -12
- package/src/config/bundled-skills/watcher/tools/watcher-update.ts +0 -12
- package/src/email/guardrails.ts +0 -221
- package/src/email/provider.ts +0 -117
- package/src/email/providers/agentmail.ts +0 -361
- package/src/email/providers/index.ts +0 -65
- package/src/email/service.ts +0 -384
- package/src/email/types.ts +0 -126
- package/src/prompts/templates/UPDATES.md +0 -38
- package/src/prompts/templates/USER.md +0 -13
- package/src/prompts/update-bulletin-format.ts +0 -68
- package/src/prompts/update-bulletin-state.ts +0 -58
- package/src/prompts/update-bulletin-template-path.ts +0 -13
- package/src/prompts/update-bulletin.ts +0 -128
- package/src/providers/speech-to-text/types.ts +0 -17
- package/src/runtime/routes/browser-cdp-routes.ts +0 -229
- package/src/shared/provider-env-vars.ts +0 -19
- package/src/tools/watcher/create.ts +0 -86
- package/src/tools/watcher/delete.ts +0 -36
- package/src/tools/watcher/digest.ts +0 -54
- package/src/tools/watcher/list.ts +0 -83
- package/src/tools/watcher/update.ts +0 -71
package/.prettierignore
CHANGED
|
@@ -5,3 +5,8 @@
|
|
|
5
5
|
# Prompt template files use _ as a comment marker prefix. Prettier escapes
|
|
6
6
|
# these to \_ which breaks the comment-stripping preprocessor.
|
|
7
7
|
src/prompts/templates/**/*.md
|
|
8
|
+
|
|
9
|
+
# Compaction prompt assets are consumed verbatim as the LLM system prompt.
|
|
10
|
+
# Prettier's markdown formatting (blank lines between headers, trailing
|
|
11
|
+
# newlines) would change the prompt content, so we skip formatting here.
|
|
12
|
+
src/context/prompts/**/*.md
|
package/ARCHITECTURE.md
CHANGED
|
@@ -7,7 +7,7 @@ This document owns assistant-runtime architecture details. The repo-level archit
|
|
|
7
7
|
- Transport metadata arrives via `conversation_create.transport` (HTTP) or `/channels/inbound` (`channelId`, optional `hints`, optional `uxBrief`).
|
|
8
8
|
- Telegram webhook ingress injects deterministic channel-safe transport metadata (`hints` + `uxBrief`) so non-dashboard channels defer dashboard-only UI tasks cleanly.
|
|
9
9
|
- `OnboardingPlaybookManager` resolves `<channel>_onboarding.md`, checks `onboarding/playbooks/registry.json`, and applies per-channel first-time fast-path onboarding.
|
|
10
|
-
- `OnboardingOrchestrator` derives onboarding-mode guidance (post-hatch sequence,
|
|
10
|
+
- `OnboardingOrchestrator` derives onboarding-mode guidance (post-hatch sequence, `users/<slug>.md` persona capture) from playbook + transport context.
|
|
11
11
|
- Conversation runtime assembly injects both `<channel_onboarding_playbook>` and `<onboarding_mode>` context before provider calls, then strips both from persisted conversation history.
|
|
12
12
|
- Permission setup remains user-initiated and hatch + first-conversation flows avoid proactive permission asks.
|
|
13
13
|
|
|
@@ -333,11 +333,11 @@ The Slack channel provides text-based messaging via Slack's Socket Mode API. Unl
|
|
|
333
333
|
|
|
334
334
|
**Control-plane endpoints** (`/v1/integrations/slack/channel/config`):
|
|
335
335
|
|
|
336
|
-
| Endpoint | Method | Description
|
|
337
|
-
| --------------------------------------- | ------ |
|
|
338
|
-
| `/v1/integrations/slack/channel/config` | GET | Returns current config status: `hasBotToken`, `hasAppToken`, `connected`, plus workspace metadata (`teamId`, `teamName`, `botUserId`, `botUsername`)
|
|
339
|
-
| `/v1/integrations/slack/channel/config` | POST | Validates and stores credentials. Body: `{ botToken?: string, appToken?: string }`
|
|
340
|
-
| `/v1/integrations/slack/channel/config` | DELETE | Clears all Slack channel credentials from secure storage and credential metadata
|
|
336
|
+
| Endpoint | Method | Description |
|
|
337
|
+
| --------------------------------------- | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
338
|
+
| `/v1/integrations/slack/channel/config` | GET | Returns current config status: `hasBotToken`, `hasAppToken`, `hasUserToken`, `connected`, plus workspace metadata (`teamId`, `teamName`, `botUserId`, `botUsername`) |
|
|
339
|
+
| `/v1/integrations/slack/channel/config` | POST | Validates and stores credentials. Body: `{ botToken?: string, appToken?: string, userToken?: string }` |
|
|
340
|
+
| `/v1/integrations/slack/channel/config` | DELETE | Clears all Slack channel credentials (bot, app, and user tokens) from secure storage and credential metadata. Surgical user-token-only deletion is exposed internally via `clearSlackUserToken` (used by the credential vault) but is not reachable through this HTTP endpoint. |
|
|
341
341
|
|
|
342
342
|
All endpoints are JWT-authenticated via `Authorization: Bearer <jwt>`.
|
|
343
343
|
|
|
@@ -345,13 +345,18 @@ All endpoints are JWT-authenticated via `Authorization: Bearer <jwt>`.
|
|
|
345
345
|
|
|
346
346
|
Both tokens are stored in the secure key store (CES credential store with encrypted file fallback):
|
|
347
347
|
|
|
348
|
-
| Secure key
|
|
349
|
-
|
|
|
350
|
-
| `credential/slack_channel/bot_token`
|
|
351
|
-
| `credential/slack_channel/app_token`
|
|
348
|
+
| Secure key | Content |
|
|
349
|
+
| ------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
350
|
+
| `credential/slack_channel/bot_token` | Slack bot token (used for `chat.postMessage` and `auth.test`) |
|
|
351
|
+
| `credential/slack_channel/app_token` | Slack app token (`xapp-...`, used for Socket Mode `apps.connections.open`) |
|
|
352
|
+
| `credential/slack_channel/user_token` | Optional. Slack user OAuth token (`xoxp-...`). Enables reading channels and DMs the bot isn't a member of (for triage). Never used for writes. |
|
|
352
353
|
|
|
353
354
|
Workspace metadata (team ID, team name, bot user ID, bot username) is stored as JSON in the credential metadata store under `('slack_channel', 'bot_token')`.
|
|
354
355
|
|
|
356
|
+
**Read/write auth split:** The Slack adapter (`src/messaging/providers/slack/adapter.ts`) caches read and write auth separately. Reads (`listConversations`, `getHistory`, replies, search, `users.info`) prefer the user token when present via `getReadAuth()` — giving visibility into channels and DMs the bot hasn't been invited to. Writes (`postMessage`, `markRead`, and any future state-changing calls) always use the bot token via `getWriteAuth()` so posts come from the bot identity, never the user's. When no user token is stored, reads fall back to the bot token (unchanged legacy behavior).
|
|
357
|
+
|
|
358
|
+
**Workspace-consistency invariant:** The user token and bot token must be for the same Slack workspace. `setSlackChannelConfig` enforces this by calling `auth.test` on each token and comparing the returned `team_id`; if the user token's workspace does not match an already-stored bot token's workspace, the user token is rejected and not stored.
|
|
359
|
+
|
|
355
360
|
**Token validation via `auth.test`:**
|
|
356
361
|
|
|
357
362
|
When a bot token is provided via `POST /v1/integrations/slack/channel/config`, the handler calls `POST https://slack.com/api/auth.test` with the token before storing it. A successful response yields workspace metadata (`team_id`, `team`, `user_id`, `user`) that is persisted alongside the token. If `auth.test` fails, the token is rejected and not stored.
|
|
@@ -582,34 +587,191 @@ All guardian decisions for voice access requests flow through:
|
|
|
582
587
|
| `src/runtime/actor-trust-resolver.ts` | `resolveActorTrust` — caller trust classification |
|
|
583
588
|
| `src/memory/canonical-guardian-store.ts` | Canonical request persistence and CAS resolution |
|
|
584
589
|
|
|
590
|
+
### Speech-to-Text (STT) Boundaries
|
|
591
|
+
|
|
592
|
+
Audio-to-text conversion occurs in five distinct runtime boundaries, each with its own provider model and adapter layer. The `services.stt` config block is the single source of truth for STT provider selection across both daemon and telephony boundaries.
|
|
593
|
+
|
|
594
|
+
**Provider catalog model:** The daemon's canonical provider catalog (`src/providers/speech-to-text/provider-catalog.ts`) is the single source of truth for STT provider metadata — credential mappings, supported boundaries, telephony mode, and conversation streaming mode. The client-facing catalog (`meta/stt-provider-catalog.json`) carries display metadata (names, hints, setup mode, `conversationStreamingMode`) and is bundled into native apps at build time. A CI parity test (`src/__tests__/stt-catalog-parity.test.ts`) enforces that provider IDs, ordering, and credential-provider name mappings stay aligned between the two catalogs. To add a new provider, follow the checklist in `docs/stt-provider-onboarding.md`.
|
|
595
|
+
|
|
596
|
+
**Boundary overview:**
|
|
597
|
+
|
|
598
|
+
| Boundary | Runtime | Provider (current) | Adapter module | Caller |
|
|
599
|
+
| ---------------------------- | ----------------------------------------------------------------------------- | -------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------- |
|
|
600
|
+
| **Telephony (hybrid)** | Twilio-native ConversationRelay or daemon media-stream (provider-conditional) | Configured STT provider (via `services.stt`) | `src/calls/telephony-stt-routing.ts` | `src/calls/twilio-routes.ts` |
|
|
601
|
+
| **Daemon batch** | Daemon process (REST API to provider) | Configured STT provider (via `services.stt`) | `src/stt/daemon-batch-transcriber.ts` | `src/runtime/routes/inbound-stages/transcribe-audio.ts` |
|
|
602
|
+
| **Conversation streaming** | Daemon process (WebSocket-based) | Configured STT provider (via `services.stt`) | `src/stt/stt-stream-session.ts`, `src/providers/speech-to-text/deepgram-realtime.ts`, `src/providers/speech-to-text/google-gemini-live-stream.ts`, `src/providers/speech-to-text/openai-whisper-stream.ts`, `src/providers/speech-to-text/xai-realtime.ts` | `VoiceInputManager` (macOS conversation), `InputBarView` (iOS conversation) via gateway WS proxy |
|
|
603
|
+
| **Client service-first** | macOS / iOS via gateway → daemon | Configured STT provider (via `services.stt`) | `src/runtime/routes/stt-routes.ts`, `clients/shared/Network/STTClient.swift` | `VoiceInputManager` (macOS dictation), `InputBarView` (iOS), `OpenAIVoiceService` (macOS voice mode) |
|
|
604
|
+
| **Client-native (fallback)** | macOS / iOS on-device | Apple Speech (`SFSpeechRecognizer`) | `clients/macos/.../SpeechRecognizerAdapter.swift`, `clients/ios/.../SpeechRecognizerAdapter.swift` | Fallback when STT service is unconfigured or fails |
|
|
605
|
+
|
|
606
|
+
**Telephony boundary (hybrid routing):**
|
|
607
|
+
|
|
608
|
+
Telephony STT uses a provider-conditional hybrid model driven by `services.stt.provider`. The routing resolver (`src/calls/telephony-stt-routing.ts`) maps the configured provider to a discriminated strategy at call setup time:
|
|
609
|
+
|
|
610
|
+
- **`conversation-relay-native`** (Deepgram, Google) — TwiML emits `<Connect><ConversationRelay>` with `transcriptionProvider` and `speechModel` attributes. Twilio handles audio ingestion and transcription natively; the daemon receives transcribed text via the relay WebSocket. The Twilio-native provider name and default speech model are read from the provider catalog entry's `telephonyRouting.twilioNativeMapping` (e.g. Deepgram maps to `provider: "Deepgram"` with `defaultSpeechModel: "nova-3"`; Google maps to `provider: "Google"` with `defaultSpeechModel: undefined`).
|
|
611
|
+
|
|
612
|
+
- **`media-stream-custom`** (OpenAI Whisper) — TwiML emits `<Connect><Stream>` pointing to the gateway's media-stream proxy. The `<Stream url="...">` encodes `callSessionId` and auth `token` as **URL path segments** (e.g. `.../media-stream/<callSessionId>/<token>`) because Twilio Media Streams does not reliably preserve query parameters across the WebSocket upgrade. The gateway extracts metadata from path segments (with query-parameter fallback for backward compatibility) and proxies raw audio frames to the daemon, which transcribes server-side via the provider's batch API.
|
|
613
|
+
|
|
614
|
+
Key modules:
|
|
615
|
+
|
|
616
|
+
| Module | Purpose |
|
|
617
|
+
| --------------------------------------------------- | ---------------------------------------------------------------------- |
|
|
618
|
+
| `src/calls/telephony-stt-routing.ts` | Maps `services.stt.provider` to a discriminated `TelephonySttStrategy` |
|
|
619
|
+
| `src/calls/twilio-routes.ts` | Voice webhook handler; generates provider-conditional TwiML |
|
|
620
|
+
| `src/calls/media-stream-parser.ts` | Twilio Media Streams protocol parser |
|
|
621
|
+
| `src/calls/media-turn-detector.ts` | Energy-based VAD turn detector for raw audio |
|
|
622
|
+
| `src/calls/media-stream-stt-session.ts` | STT session that transcribes audio turns via `services.stt` |
|
|
623
|
+
| `src/calls/call-transport.ts` | Transport interface decoupling CallController from wire protocol |
|
|
624
|
+
| `src/calls/media-stream-output.ts` | Output adapter for sending TTS audio back via Media Streams |
|
|
625
|
+
| `src/calls/media-stream-server.ts` | WebSocket server binding media-stream lifecycle to call sessions |
|
|
626
|
+
| `gateway/src/http/routes/twilio-media-websocket.ts` | Gateway WebSocket proxy for Media Streams frames |
|
|
627
|
+
|
|
628
|
+
Guard tests in `__tests__/twilio-routes-twiml.test.ts` and `__tests__/twilio-routes.test.ts` assert that TwiML generation matches the provider-conditional strategy for each supported provider.
|
|
629
|
+
|
|
630
|
+
To add a new telephony STT provider: add a `telephonyRouting` entry to the provider's catalog entry in `provider-catalog.ts`. Set `strategyKind` to `"conversation-relay-native"` for Twilio-native providers (and include a `twilioNativeMapping` with the Twilio `provider` name and `defaultSpeechModel`), or `"media-stream-custom"` for providers that require daemon-side transcription. The routing resolver reads these fields from the catalog — no hardcoded maps to update.
|
|
631
|
+
|
|
632
|
+
**Daemon batch boundary:**
|
|
633
|
+
|
|
634
|
+
The daemon transcribes audio attachments (e.g. voice messages from channel inbound) by calling a provider's REST API directly.
|
|
635
|
+
|
|
636
|
+
- `src/stt/types.ts` defines provider-agnostic domain types: `BatchTranscriber` interface, `SttTranscribeRequest`, `SttTranscribeResult`, `SttError` with normalized categories (`auth`, `rate-limit`, `timeout`, `invalid-audio`, `provider-error`), and `SttProviderId` / `SttBoundaryId` discriminants.
|
|
637
|
+
- `createDaemonBatchTranscriber()` in `src/stt/daemon-batch-transcriber.ts` is the factory that returns a `BatchTranscriber` backed by the configured STT provider (OpenAI Whisper or Deepgram, selected via `services.stt.provider`). Returns `null` when no API key is available for the selected provider. `normalizeSttError()` maps raw provider errors to `SttError` categories.
|
|
638
|
+
- `resolveBatchTranscriber()` in `src/providers/speech-to-text/resolve.ts` is the credential-aware entry point — it reads the configured provider from `services.stt`, resolves the corresponding API key from the secure key store, and delegates to the factory.
|
|
639
|
+
- `tryTranscribeAudioAttachments()` in `src/runtime/routes/inbound-stages/transcribe-audio.ts` is the callsite that uses the facade for channel audio attachment transcription.
|
|
640
|
+
|
|
641
|
+
To add a new daemon batch STT provider, follow the full checklist in `docs/stt-provider-onboarding.md` — it covers the daemon catalog, type registration, config schema, adapter wiring, credential plumbing, client catalog, and parity tests.
|
|
642
|
+
|
|
643
|
+
**Conversation streaming boundary:**
|
|
644
|
+
|
|
645
|
+
Real-time conversation chat message capture on macOS and iOS uses a WebSocket-based streaming STT path. When the configured `services.stt` provider supports conversation streaming (determined by the `conversationStreamingMode` field in the provider catalog), native clients open a WebSocket session through the gateway to the daemon's `/v1/stt/stream` endpoint. The daemon resolves a `StreamingTranscriber` for the configured provider and streams partial/final transcript events back to the client in real time.
|
|
646
|
+
|
|
647
|
+
Two provider adapters are supported, each implementing the `StreamingTranscriber` interface from `src/stt/types.ts`:
|
|
648
|
+
|
|
649
|
+
| Provider | Adapter | Mode | Mechanism |
|
|
650
|
+
| ----------------- | ----------------------------------------------------------- | ------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
651
|
+
| **Deepgram** | `src/providers/speech-to-text/deepgram-realtime.ts` | `realtime-ws` | Opens a WebSocket to Deepgram's `/v1/listen` endpoint, forwards raw PCM audio, normalizes Deepgram's `is_final`/`speech_final` semantics into `partial`/`final` events. Uses model `nova-2`. |
|
|
652
|
+
| **Google Gemini** | `src/providers/speech-to-text/google-gemini-live-stream.ts` | `realtime-ws` | Opens a bidirectional streaming session against Gemini's Live API (`ai.live.connect`), forwards PCM audio frames, and normalizes `serverContent.inputTranscription` events into `partial`/`final` events. Uses model `gemini-live-2.5-flash-preview`. |
|
|
653
|
+
|
|
654
|
+
**Provider-specific behavior differences:**
|
|
655
|
+
|
|
656
|
+
- **Deepgram (`realtime-ws`)**: True WebSocket streaming with sub-second partial latency. Emits `partial` events for `is_final: false` frames and `final` events for `is_final: true` frames. Supports backpressure (drops audio frames when `bufferedAmount > 1 MiB`). Sends `CloseStream` message on stop with a 5-second grace period for the provider to flush remaining finals. Inactivity timeout: 30 seconds (provider-side hang detection). Connect timeout: 10 seconds. Auth errors map to close codes 1008/4001; rate limits to 1013.
|
|
657
|
+
- **Google Gemini (`realtime-ws`)**: WebSocket-backed Live API session. Partials are emitted as Gemini streams `inputTranscription.text` fragments; a `final` is emitted when the server signals `generationComplete` or `turnComplete`. On `stop()`, the adapter sends `audioStreamEnd: true` and waits up to a 5-second grace window for the server to flush remaining transcription before force-closing. Inactivity timeout: 30 seconds. Connect timeout: 10 seconds. Close codes 1008/4001 map to `auth`; 1013 maps to `rate-limit`; other codes map to `provider-error`. The model's own text turn is suppressed via a silent system instruction so we only pay for transcription.
|
|
658
|
+
|
|
659
|
+
**Session lifecycle (daemon side):**
|
|
660
|
+
|
|
661
|
+
1. Client opens a WebSocket to `/v1/stt/stream` with required query parameter `mimeType` and optional `provider` and `sampleRate`. The `provider` parameter is optional compatibility metadata — the runtime is config-authoritative and always resolves the streaming transcriber from `services.stt.provider`. When a requested provider disagrees with the configured provider, the runtime logs a mismatch warning.
|
|
662
|
+
2. `SttStreamSession` (in `src/stt/stt-stream-session.ts`) resolves a `StreamingTranscriber` via `resolveStreamingTranscriber()` from `src/providers/speech-to-text/resolve.ts`, using the configured provider (not the requested one).
|
|
663
|
+
3. The transcriber's `start()` method opens the provider session.
|
|
664
|
+
4. A `ready` event (with `provider` field) is sent to the client, signaling that audio frames are accepted.
|
|
665
|
+
5. Client sends `audio` frames (binary WebSocket frames or base64-encoded JSON) and a `stop` event when recording ends.
|
|
666
|
+
6. The transcriber emits `partial` and `final` events, forwarded to the client as JSON frames with monotonic `seq` numbers.
|
|
667
|
+
7. The session closes deterministically on: client disconnect, `stop` event followed by provider `closed`, idle timeout (60 seconds), or runtime shutdown.
|
|
668
|
+
|
|
669
|
+
**Session lifecycle (client side):**
|
|
670
|
+
|
|
671
|
+
- `STTStreamingClient` (`clients/shared/Network/STTStreamingClient.swift`) manages the WebSocket session using `URLSessionWebSocketTask`. It builds the gateway WebSocket URL via `GatewayHTTPClient.buildWebSocketRequest(path: "stt/stream", params:)`.
|
|
672
|
+
- `STTProviderRegistry` (`clients/shared/Utilities/STTProviderRegistry.swift`) exposes `isStreamingAvailable` (checks the configured provider's `conversationStreamingMode` from the bundled `stt-provider-catalog.json`) and `isServiceConfigured` (checks whether any STT provider is set).
|
|
673
|
+
- macOS: `VoiceInputManager.startStreamingSession()` creates a fresh `STTStreamingClient` per recording session. Streaming partials take priority over `SFSpeechRecognizer` partials while the stream is active and healthy. When recording stops, if the stream delivered at least one `final` event (`streamingReceivedFinal`) and has not failed (`streamingFailed`), the streaming final text is used directly. Otherwise, the batch STT path (`STTClient.transcribe()`) provides the fallback.
|
|
674
|
+
- iOS: `InputBarView.handleStreamingEvent()` applies the same priority scheme. Streaming partials update the text field while `isStreamingActive` is true and the user has not manually typed. A `.final` event commits the result via `onVoiceResult` and tears down the session. On error or close without a final, `resolveTranscriptWithServiceFirst()` triggers batch STT fallback.
|
|
675
|
+
|
|
676
|
+
**Fallback semantics:**
|
|
677
|
+
|
|
678
|
+
The conversation streaming path degrades gracefully to the existing batch STT path:
|
|
679
|
+
|
|
680
|
+
1. **Unsupported provider** (a hypothetical provider with `conversationStreamingMode: "none"`): The client checks `STTProviderRegistry.isStreamingAvailable` before attempting a streaming session. When `false`, recording proceeds with the batch-only flow (no WebSocket is opened). On the daemon side, if a streaming session is somehow opened for an unsupported provider, the session sends an `error` event followed by `closed` and closes the socket with code 1000.
|
|
681
|
+
2. **Connection failure** (network error, gateway down, auth failure): The `STTStreamingClient` reports an `STTStreamFailure` to the client's `onFailure` callback. macOS sets `streamingFailed = true`; iOS sets `isStreamingActive = false`. Both platforms then fall through to batch STT resolution when recording stops.
|
|
682
|
+
3. **Mid-session provider error** (provider WebSocket disconnect, timeout, rate limit): The daemon session emits an `error` event (with a normalized `SttErrorCategory`) followed by `closed`. The client marks the stream as failed and defers to batch STT.
|
|
683
|
+
4. **Missing credentials**: `resolveStreamingTranscriber()` returns `null` when the API key is not configured. The session sends an `error`+`closed` pair and the client falls back to batch.
|
|
684
|
+
|
|
685
|
+
**Error category mapping:**
|
|
686
|
+
|
|
687
|
+
| Category | Deepgram close codes | Google Gemini close codes | Client action |
|
|
688
|
+
| ---------------- | -------------------- | ------------------------------------ | ---------------------------------- |
|
|
689
|
+
| `auth` | 1008, 4001 | 1008, 4001 | Mark stream failed; batch fallback |
|
|
690
|
+
| `rate-limit` | 1013 | 1013 | Mark stream failed; batch fallback |
|
|
691
|
+
| `timeout` | N/A (inactivity) | N/A (inactivity) | Mark stream failed; batch fallback |
|
|
692
|
+
| `provider-error` | All other codes | All other codes / Live session error | Mark stream failed; batch fallback |
|
|
693
|
+
|
|
694
|
+
**Key source files:**
|
|
695
|
+
|
|
696
|
+
| File | Purpose |
|
|
697
|
+
| ----------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
698
|
+
| `src/stt/types.ts` | `StreamingTranscriber` interface, `SttStreamClientEvent`/`SttStreamServerEvent` discriminated unions, `ConversationStreamingMode` type |
|
|
699
|
+
| `src/stt/stt-stream-session.ts` | Runtime session orchestrator: lifecycle management, idle timeout, event forwarding with `seq` ordering |
|
|
700
|
+
| `src/providers/speech-to-text/deepgram-realtime.ts` | Deepgram realtime-ws adapter: WebSocket to Deepgram `/v1/listen`, `is_final`/`speech_final` normalization |
|
|
701
|
+
| `src/providers/speech-to-text/google-gemini-live-stream.ts` | Google Gemini realtime-ws adapter: bidirectional Live API session, `serverContent.inputTranscription` normalization |
|
|
702
|
+
| `src/providers/speech-to-text/provider-catalog.ts` | Provider catalog with `conversationStreamingMode` per entry (`realtime-ws`, `incremental-batch`, `none`) |
|
|
703
|
+
| `src/providers/speech-to-text/resolve.ts` | `resolveStreamingTranscriber()`: credential-aware factory for streaming adapters; `resolveConversationStreamingSttCapability()`: capability validator |
|
|
704
|
+
| `src/runtime/http-server.ts` | Runtime WebSocket upgrade handler for `/v1/stt/stream`, session registry (`activeSttStreamSessions`), graceful shutdown |
|
|
705
|
+
| `gateway/src/http/routes/stt-stream-websocket.ts` | Gateway WebSocket proxy: authenticates client, opens upstream WS to daemon with service token |
|
|
706
|
+
| `clients/shared/Network/STTStreamingClient.swift` | Shared Swift WebSocket client: `URLSessionWebSocketTask`-based, event parsing, failure reporting |
|
|
707
|
+
| `clients/shared/Utilities/STTProviderRegistry.swift` | Client-side provider catalog: `isStreamingAvailable`, `conversationStreamingMode` per provider |
|
|
708
|
+
| `clients/macos/.../VoiceInputManager.swift` | macOS integration: `startStreamingSession()`, streaming/batch priority, fallback on failure |
|
|
709
|
+
| `clients/ios/Views/InputBarView.swift` | iOS integration: `handleStreamingEvent()`, auto-stop coordination, batch fallback |
|
|
710
|
+
|
|
711
|
+
**Client service-first boundary:**
|
|
712
|
+
|
|
713
|
+
All product-facing dictation and voice-streaming paths on macOS and iOS use a service-first STT strategy. Clients record audio, encode it to WAV via `AudioWavEncoder` (shared utility in `clients/shared/Utilities/AudioWavEncoder.swift`), and POST it through the gateway to the daemon's `POST /v1/stt/transcribe` endpoint via `STTClient` (`clients/shared/Network/STTClient.swift`). The daemon resolves the configured STT provider through `resolveBatchTranscriber()` and returns the transcribed text.
|
|
714
|
+
|
|
715
|
+
- `STTClient` conforms to `STTClientProtocol` and returns a typed `STTResult` enum (`success`, `notConfigured`, `serviceUnavailable`, `error`). Callers pattern-match on the result to deterministically trigger native fallback.
|
|
716
|
+
- The gateway proxies the request via assistant-scoped path rewriting: `/v1/assistants/:id/stt/transcribe` is rewritten to `/v1/stt/transcribe` on the daemon.
|
|
717
|
+
- `stt-routes.ts` (`src/runtime/routes/stt-routes.ts`) defines the HTTP endpoint, validates the audio payload, and delegates to `resolveBatchTranscriber()`.
|
|
718
|
+
|
|
719
|
+
Product-facing flows using service-first STT:
|
|
720
|
+
|
|
721
|
+
| Flow | Client | Entry point |
|
|
722
|
+
| ----------------------------- | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
|
|
723
|
+
| **Push-to-talk dictation** | macOS | `VoiceInputManager.resolveTranscription()` — encodes accumulated PCM buffers to WAV, calls `sttClient.transcribe()`, falls back to native text on failure |
|
|
724
|
+
| **Conversation chat capture** | macOS | `VoiceInputManager.handleFinalTranscription()` — prefers streaming final when available; falls back to batch `sttClient.transcribe()` when streaming was not used, failed, or produced no finals |
|
|
725
|
+
| **Input bar dictation** | iOS | `InputBarView.resolveTranscriptWithServiceFirst()` — encodes captured audio buffers to WAV, calls `sttClient.transcribe()`, falls back to native transcript on failure. In conversation mode, defers to streaming final when `streamingFinalReceived` is set |
|
|
726
|
+
| **Voice mode (streaming)** | macOS | `OpenAIVoiceService.stopRecordingAndGetTranscription()` — encodes per-turn PCM to WAV, calls `sttClient.transcribe()` for turn-final transcript resolution, falls back to SFSpeechRecognizer result |
|
|
727
|
+
|
|
728
|
+
**Client-native fallback boundary:**
|
|
729
|
+
|
|
730
|
+
Apple-native on-device recognition via `SFSpeechRecognizer` serves two roles in all three product-facing flows above: (1) it provides low-latency partial transcriptions for real-time display during recording, and (2) it provides the fallback final transcription when the STT service is unconfigured (HTTP 503), temporarily unavailable (HTTP 5xx), or returns an empty result. The `SpeechRecognizerAdapter` protocols on each platform abstract Apple Speech for **testability and dependency injection**.
|
|
731
|
+
|
|
732
|
+
- macOS: `SpeechRecognizerAdapter` protocol in `clients/macos/vellum-assistant/Features/Voice/SpeechRecognizerAdapter.swift` abstracts `SFSpeechRecognizer` static APIs and instance creation. `AppleSpeechRecognizerAdapter` is the production implementation. `OpenAIVoiceService` and `VoiceInputManager` consume the adapter via dependency injection. **Note:** The macOS protocol leaks Apple Speech types through its surface — `authorizationStatus()` returns `SFSpeechRecognizerAuthorizationStatus` and `makeRecognizer(locale:)` returns `SFSpeechRecognizer?` directly. This means callers depend on the Speech framework at compile time.
|
|
733
|
+
- iOS: `SpeechRecognizerAdapter` protocol in `clients/ios/Services/SpeechRecognizerAdapter.swift` covers authorization, availability, and task construction. `AppleSpeechRecognizerAdapter` is the production implementation. `InputBarView` consumes the adapter. **Note:** The iOS protocol defines its own framework-agnostic types (`SpeechRecognizerAuthorizationStatus`, `SpeechRecognitionResult`) so callers never see `SFSpeechRecognizer` directly. However, `startRecognitionTask` still returns `SFSpeechAudioBufferRecognitionRequest` in its tuple, so full framework decoupling is not yet achieved.
|
|
734
|
+
|
|
735
|
+
Platform divergence summary:
|
|
736
|
+
|
|
737
|
+
- **Auth API:** macOS uses a callback-based `requestAuthorization(completion:)` returning `SFSpeechRecognizerAuthorizationStatus`; iOS uses `async requestAuthorization()` returning a custom `SpeechRecognizerAuthorizationStatus` enum.
|
|
738
|
+
- **Recognizer exposure:** macOS exposes the raw `SFSpeechRecognizer?` via `makeRecognizer(locale:)`; iOS fully encapsulates recognizer construction inside `startRecognitionTask`.
|
|
739
|
+
- **Concurrency model:** The iOS protocol is `@MainActor`-annotated; the macOS protocol is not.
|
|
740
|
+
|
|
741
|
+
These differences are intentional — the adapters were designed for their respective platform integration needs, not for cross-platform uniformity.
|
|
742
|
+
|
|
743
|
+
**Cross-boundary notes:**
|
|
744
|
+
|
|
745
|
+
- The `services.stt` config block is the single source of truth for STT provider selection across the daemon batch boundary, the conversation streaming boundary, the client service-first boundary, and the telephony boundary. The batch and streaming resolvers (`resolveBatchTranscriber()`, `resolveStreamingTranscriber()`) both read from `services.stt.provider` and resolve credentials through the same catalog; the telephony boundary uses `resolveTelephonySttRouting()` to determine the Twilio integration strategy. The daemon provider catalog (`src/providers/speech-to-text/provider-catalog.ts`) is the authoritative registry of supported providers; the client catalog (`meta/stt-provider-catalog.json`) mirrors it for display purposes (including `conversationStreamingMode`). The parity guard test (`src/__tests__/stt-catalog-parity.test.ts`) fails CI when the two catalogs diverge on provider IDs, ordering, or credential-provider name mappings.
|
|
746
|
+
- Conversation streaming does not replace the client service-first batch path. When streaming is available, it runs concurrently during recording and provides real-time partials and finals. The batch path remains the fallback for providers that do not support streaming, when streaming fails mid-session, or when streaming produces no final transcript.
|
|
747
|
+
- Credential mapping is catalog-driven: `provider-secret-catalog.ts` derives STT API-key provider names from the daemon catalog via `listCredentialProviderNames()`, deduplicating against the LLM/search provider list. Adding a provider to the catalog automatically includes its credential name in `API_KEY_PROVIDERS`.
|
|
748
|
+
- Terminology: "STT" and "transcription" refer to the same operation (converting audio to text). "Speech recognition" is used in client-native contexts where Apple's Speech framework terminology is canonical. All three terms map to the same conceptual operation.
|
|
749
|
+
- **Onboarding**: For a step-by-step guide to adding a new STT provider, see `docs/stt-provider-onboarding.md`.
|
|
750
|
+
|
|
585
751
|
### Update Bulletin System
|
|
586
752
|
|
|
587
|
-
Release-driven update notification system that
|
|
753
|
+
Release-driven update notification system that dispatches a background conversation to process release notes when a release lands.
|
|
588
754
|
|
|
589
755
|
**Data flow:**
|
|
590
756
|
|
|
591
|
-
1. **
|
|
592
|
-
2. **
|
|
593
|
-
3. **
|
|
594
|
-
4. **Completion by deletion** — the assistant deletes `UPDATES.md` when it has actioned all updates. Next startup detects the deletion and marks those releases as completed in checkpoint state.
|
|
595
|
-
5. **Cross-release merge** — if pending updates from a prior release exist when a new release lands, both release blocks coexist in the same file.
|
|
757
|
+
1. **Storage** — Release notes live at `<workspace>/UPDATES.md`. The file is written by workspace migrations; each release that needs to surface notes ships a dedicated migration in `src/workspace/migrations/` that appends a release-notes block to the file. The workspace-migration runner is the authoritative idempotency mechanism: `runWorkspaceMigrations()` records each migration's `WorkspaceMigration.id` in `<workspace>/data/.workspace-migrations.json` and never re-runs an ID that is already in the `applied` set.
|
|
758
|
+
2. **Dispatch** — At daemon startup (after `runWorkspaceMigrations()`), `runUpdateBulletinJobIfNeeded()` is invoked fire-and-forget. It hashes the current `UPDATES.md` content and compares against the `updates:last_processed_hash` checkpoint. When the hashes differ, it bootstraps a `conversationType: "background"` conversation and calls `wakeAgentForOpportunity()` so the agent processes the bulletin without any interactive session.
|
|
759
|
+
3. **Completion** — The agent acts on the contents and deletes `UPDATES.md` when done. The job persists the new hash to `updates:last_processed_hash` post-wake, so subsequent startups short-circuit until the file is repopulated by a future migration.
|
|
596
760
|
|
|
597
761
|
**Checkpoint keys** (in `memory_checkpoints` table):
|
|
598
762
|
|
|
599
|
-
- `updates:
|
|
600
|
-
- `updates:completed_releases` — JSON array of version strings already completed.
|
|
763
|
+
- `updates:last_processed_hash` — content hash of the `UPDATES.md` payload most recently dispatched to the background job.
|
|
601
764
|
|
|
602
765
|
**Key source files:**
|
|
603
766
|
|
|
604
|
-
| File | Purpose
|
|
605
|
-
| -------------------------------------- |
|
|
606
|
-
| `src/
|
|
607
|
-
| `src/
|
|
608
|
-
| `src/
|
|
609
|
-
| `src/
|
|
610
|
-
| `src/
|
|
611
|
-
| `src/
|
|
612
|
-
| `src/permissions/defaults.ts` | Auto-allow rules for file_read/write/edit + rm UPDATES.md |
|
|
767
|
+
| File | Purpose |
|
|
768
|
+
| -------------------------------------- | ----------------------------------------------------------------------------------------- |
|
|
769
|
+
| `src/workspace/migrations/` | Per-release migrations that append release notes to `UPDATES.md` |
|
|
770
|
+
| `src/workspace/migrations/registry.ts` | Append-only `WORKSPACE_MIGRATIONS` registry |
|
|
771
|
+
| `src/prompts/update-bulletin-job.ts` | `runUpdateBulletinJobIfNeeded()` — hash check, background dispatch, and checkpoint update |
|
|
772
|
+
| `src/daemon/lifecycle.ts` | Fire-and-forget dispatch of `runUpdateBulletinJobIfNeeded()` after DB init at startup |
|
|
773
|
+
| `src/config/schemas/updates.ts` | `updates.enabled` config toggle (defaults to `true`; disables the background dispatch) |
|
|
774
|
+
| `src/permissions/defaults.ts` | Auto-allow rules for file_read/write/edit + bare-filename `rm UPDATES.md` |
|
|
613
775
|
|
|
614
776
|
---
|
|
615
777
|
|
|
@@ -1404,18 +1566,19 @@ graph TB
|
|
|
1404
1566
|
FIND_RULE -->|"No match"| NO_MATCH{"Fallback logic"}
|
|
1405
1567
|
|
|
1406
1568
|
RISK_CHECK -->|"Low / Medium"| AUTO_ALLOW["decision: allow<br/>Auto-allowed by rule"]
|
|
1407
|
-
RISK_CHECK -->|"High"| HIGH_CHECK{"
|
|
1408
|
-
HIGH_CHECK -->|"
|
|
1409
|
-
HIGH_CHECK -->|"
|
|
1569
|
+
RISK_CHECK -->|"High"| HIGH_CHECK{"shouldAutoAllowHighRisk()<br/>(containerized bash?)"}
|
|
1570
|
+
HIGH_CHECK -->|"yes"| AUTO_ALLOW
|
|
1571
|
+
HIGH_CHECK -->|"no"| RISK_THRESHOLD{"Risk-based<br/>threshold fallback"}
|
|
1410
1572
|
|
|
1411
1573
|
NO_MATCH -->|"tool.origin === 'skill'"| PROMPT_SKILL["decision: prompt<br/>Skill tools always ask"]
|
|
1412
1574
|
NO_MATCH -->|"strict mode"| PROMPT_STRICT["decision: prompt<br/>No implicit auto-allow"]
|
|
1413
|
-
NO_MATCH -->|"workspace mode (default)"| WS_CHECK{"Workspace-scoped<br
|
|
1575
|
+
NO_MATCH -->|"workspace mode (default)"| WS_CHECK{"Workspace-scoped<br/>+ Low risk?"}
|
|
1414
1576
|
WS_CHECK -->|"yes"| AUTO_WS["decision: allow<br/>Workspace-scoped auto-allow"]
|
|
1415
|
-
WS_CHECK -->|"no"|
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
1577
|
+
WS_CHECK -->|"no"| RISK_THRESHOLD
|
|
1578
|
+
|
|
1579
|
+
RISK_THRESHOLD{"risk ≤ autoApproveUpTo<br/>threshold?"}
|
|
1580
|
+
RISK_THRESHOLD -->|"yes"| AUTO_THRESHOLD["decision: allow<br/>within auto-approve threshold"]
|
|
1581
|
+
RISK_THRESHOLD -->|"no"| PROMPT_THRESHOLD["decision: prompt<br/>above auto-approve threshold"]
|
|
1419
1582
|
```
|
|
1420
1583
|
|
|
1421
1584
|
### Permission Modes: Workspace and Strict
|
|
@@ -1433,7 +1596,7 @@ The `permissions.mode` config option (`workspace` or `strict`) controls the defa
|
|
|
1433
1596
|
| `browser_*` skill tools with system default rules | Auto-allowed (priority 100 allow rules) | Auto-allowed (priority 100 allow rules) |
|
|
1434
1597
|
| Skill-origin tools with no matching rule | Prompted | Prompted |
|
|
1435
1598
|
| Allow rules for non-high-risk tools | Auto-allowed | Auto-allowed |
|
|
1436
|
-
| Allow rules
|
|
1599
|
+
| Allow rules + containerized bash (high risk) | Auto-allowed (runtime check) | Auto-allowed (runtime check) |
|
|
1437
1600
|
| Deny rules | Blocked | Blocked |
|
|
1438
1601
|
|
|
1439
1602
|
**Workspace mode** (default) auto-allows operations scoped to the workspace (file reads/writes/edits within the workspace directory, sandboxed bash) without prompting. Host operations, network requests, and operations outside the workspace still follow the normal approval flow. Explicit deny and ask rules override auto-allow.
|
|
@@ -1455,7 +1618,6 @@ Rules are stored in `~/.vellum/protected/trust.json` with version `3`. Each rule
|
|
|
1455
1618
|
| `decision` | `allow \| deny \| ask` | What to do when the rule matches |
|
|
1456
1619
|
| `priority` | `number` | Higher priority wins; deny wins ties at equal priority |
|
|
1457
1620
|
| `executionTarget` | `string?` | `sandbox` or `host` — restricts by execution context |
|
|
1458
|
-
| `allowHighRisk` | `boolean?` | When true, auto-allows even high-risk invocations |
|
|
1459
1621
|
|
|
1460
1622
|
Missing optional fields act as wildcards. A rule with no `executionTarget` matches any target.
|
|
1461
1623
|
|
|
@@ -1549,7 +1711,7 @@ When a permission prompt is sent to the client (via `confirmation_request` SSE e
|
|
|
1549
1711
|
| `allowlistOptions` | Suggested patterns for "always allow" rules |
|
|
1550
1712
|
| `scopeOptions` | Suggested scopes for rule persistence |
|
|
1551
1713
|
|
|
1552
|
-
The user can respond with: `allow` (one-time), `always_allow` (create allow rule), `
|
|
1714
|
+
The user can respond with: `allow` (one-time), `always_allow` (create allow rule), `deny` (one-time), or `always_deny` (create deny rule). High-risk operations with an allow rule in containerized environments are auto-allowed at runtime by `DefaultApprovalPolicy.shouldAutoAllowHighRisk()` without requiring persisted state. All other risk-based decisions use the `autoApproveUpTo` threshold (default: `"low"`) -- tools at or below the threshold are auto-allowed, those above are prompted.
|
|
1553
1715
|
|
|
1554
1716
|
### Canonical Paths
|
|
1555
1717
|
|
|
@@ -1928,7 +2090,7 @@ Connected channels are resolved at signal emission time: vellum is always includ
|
|
|
1928
2090
|
|
|
1929
2091
|
**Audit trail (SQLite):** `notification_events` → `notification_decisions` (with `conversationActions` in validation results) → `notification_deliveries` (with `conversation_id`, `message_id`, `conversation_strategy`, `conversation_action`, `conversation_target_id`, `conversation_fallback_used`)
|
|
1930
2092
|
|
|
1931
|
-
**Configuration:** `
|
|
2093
|
+
**Configuration:** `llm.callSites.notificationDecision` (decision engine) and `llm.callSites.preferenceExtraction` (preference extractor) in `config.json`. Both fall back to `llm.default` when unset.
|
|
1932
2094
|
|
|
1933
2095
|
---
|
|
1934
2096
|
|
|
@@ -2026,6 +2188,76 @@ The guardian trust system uses a three-valued `TrustClass` — `'guardian'`, `'t
|
|
|
2026
2188
|
| `src/memory/channel-verification-sessions.ts` | `GuardianBinding` with required `guardianPrincipalId` |
|
|
2027
2189
|
| `src/__tests__/trust-context-guards.test.ts` | Guard tests enforcing trust-context type invariants |
|
|
2028
2190
|
|
|
2191
|
+
### TTS Provider Abstraction (`services.tts`)
|
|
2192
|
+
|
|
2193
|
+
All text-to-speech functionality (in-app message playback and phone call voice) routes through a catalog-driven, provider-agnostic TTS abstraction. The architecture consists of six layers: a canonical provider catalog, a config schema, a config resolver, a provider registry, an explicit call-strategy abstraction, and a top-level synthesis orchestrator.
|
|
2194
|
+
|
|
2195
|
+
**Canonical provider catalog (`provider-catalog.ts`):** The provider catalog is the **single source of truth** for TTS provider identity and metadata on the assistant side. Every provider the system supports is declared as a `TtsProviderCatalogEntry` in the `CATALOG` array. Each entry captures the provider's unique ID (`TtsProviderId`), display name, telephony call mode (`TtsCallMode`: `"native-twilio"` or `"synthesized-play"`), static capabilities (`supportsStreaming`, `supportedFormats`), and secret requirements (credential store keys, display names, setup commands). Downstream modules query the catalog via `getCatalogProvider()`, `listCatalogProviders()`, or `listCatalogProviderIds()` instead of hardcoding provider IDs.
|
|
2196
|
+
|
|
2197
|
+
A parallel **client artifact** (`meta/tts-provider-catalog.json`) captures the subset of provider metadata needed by native clients (macOS, iOS) for display and setup UX. The client artifact must list exactly the same provider IDs as the assistant catalog. A CI consistency guard test (`src/tts/__tests__/provider-catalog-consistency.test.ts`) compares the two sets and fails if they drift.
|
|
2198
|
+
|
|
2199
|
+
**Config schema (`services.tts`):** The canonical config block lives at `services.tts` in the assistant config. The set of valid provider IDs and provider-specific config objects is catalog-driven — the Zod schema reads from the catalog rather than maintaining a separate hardcoded enum. It contains:
|
|
2200
|
+
|
|
2201
|
+
| Field | Type | Default | Description |
|
|
2202
|
+
| ----------------------------- | ------ | -------------- | --------------------------------------------------------- |
|
|
2203
|
+
| `services.tts.mode` | enum | `"your-own"` | Service mode (only `"your-own"` is supported) |
|
|
2204
|
+
| `services.tts.provider` | enum | `"elevenlabs"` | Active TTS provider (must be a catalog-known provider ID) |
|
|
2205
|
+
| `services.tts.providers.<id>` | object | _(defaults)_ | Provider-specific settings, one block per catalog entry |
|
|
2206
|
+
|
|
2207
|
+
Provider-specific config is nested under `services.tts.providers.<id>`. All legacy top-level keys (`elevenlabs.*`, `fishAudio.*`) were removed by workspace migration 032 — only canonical `services.tts` paths are supported at runtime.
|
|
2208
|
+
|
|
2209
|
+
**Config resolver (`tts-config-resolver.ts`):** `resolveTtsConfig(config)` reads `services.tts.provider` to determine the active provider and returns a `ResolvedTtsConfig` containing the provider ID and its provider-specific config object from `services.tts.providers.<id>`. No legacy fallback logic exists.
|
|
2210
|
+
|
|
2211
|
+
**Provider registry (`provider-registry.ts`):** A runtime registry where provider adapters self-register at startup via `registerTtsProvider()`. Callers resolve a provider by ID with `getTtsProvider()`, which throws for unknown IDs so misconfiguration surfaces immediately. Built-in providers are registered in `providers/register-builtins.ts` during daemon initialization. The registration is catalog-checked — `register-builtins.ts` validates that each adapter's ID exists in the catalog.
|
|
2212
|
+
|
|
2213
|
+
**Provider interface (`types.ts`):** Every provider implements the `TtsProvider` interface:
|
|
2214
|
+
|
|
2215
|
+
- `id` — unique provider identifier (matches `TtsProviderId`)
|
|
2216
|
+
- `capabilities` — static capability advertisement (`supportsStreaming`, `supportedFormats`)
|
|
2217
|
+
- `synthesize(request)` — buffer-oriented synthesis (required for all providers)
|
|
2218
|
+
- `synthesizeStream?(request, onChunk)` — optional chunk-level streaming for real-time use cases
|
|
2219
|
+
|
|
2220
|
+
The `TtsUseCase` discriminator (`"phone-call"` or `"message-playback"`) lets providers tailor format, latency, and quality trade-offs per product surface.
|
|
2221
|
+
|
|
2222
|
+
**Synthesis orchestrator (`synthesize-text.ts`):** `synthesizeText()` is the top-level entry point. It resolves the globally configured provider via the config resolver, looks up the adapter in the registry, and delegates synthesis. Provider selection is always global — per-use-case policy only gates capabilities (e.g. format checks), never overrides the chosen provider.
|
|
2223
|
+
|
|
2224
|
+
**Call strategy abstraction (`tts-call-strategy.ts`):** The call strategy layer determines how a TTS provider integrates with the Twilio ConversationRelay telephony path. Instead of inferring call behavior from runtime capabilities, `resolveCallStrategy(config)` reads the provider's `callMode` from the canonical catalog and returns a `TtsCallStrategy` with the provider ID and call mode. Two modes exist:
|
|
2225
|
+
|
|
2226
|
+
- **`native-twilio`** — Twilio handles TTS natively via ConversationRelay. The profile needs a real `ttsProvider` name (e.g. `"ElevenLabs"`) and a provider-specific voice spec string. New native providers plug in by registering a `NativeTwilioVoiceSpecBuilder` via `registerNativeTwilioVoiceSpec()` — no edits to core call routing logic required.
|
|
2227
|
+
- **`synthesized-play`** — The assistant synthesises audio via the provider's HTTP API and streams chunks to Twilio via `play` messages. Uses a placeholder TTS provider (`"Google"`) and an empty voice string because Twilio never drives TTS itself on this path.
|
|
2228
|
+
|
|
2229
|
+
**Phone call integration:** `resolveVoiceQualityProfile()` in `voice-quality.ts` uses `resolveCallStrategy()` to determine the call mode, then dispatches to the appropriate path. For `native-twilio`, it looks up the registered `NativeTwilioVoiceSpec` to build the voice string. For `synthesized-play`, it uses the placeholder profile. This replaces the previous `supportsStreaming`-based branching with explicit catalog-declared modes.
|
|
2230
|
+
|
|
2231
|
+
**Adding a new TTS provider (catalog-first checklist):**
|
|
2232
|
+
|
|
2233
|
+
1. **Catalog entry** — Add a new `TtsProviderCatalogEntry` to the `CATALOG` array in `src/tts/provider-catalog.ts`. Declare the provider's ID, display name, call mode, capabilities, and secret requirements.
|
|
2234
|
+
2. **Client artifact** — Add a corresponding entry to `meta/tts-provider-catalog.json` with the same provider ID, display name, and client-facing metadata (subtitle, setup mode, setup hint). The CI consistency guard will fail if this is skipped.
|
|
2235
|
+
3. **Config schema** — Add a new Zod object under `TtsProvidersSchema` in `src/config/schemas/tts.ts` for the provider's settings. The valid provider ID enum is catalog-driven.
|
|
2236
|
+
4. **Provider adapter** — Create `src/tts/providers/<id>-provider.ts` implementing `TtsProvider` with the appropriate `capabilities` and `synthesize`/`synthesizeStream` methods.
|
|
2237
|
+
5. **Register the adapter** — Add a factory entry for the provider to the `providerFactories` map in `src/tts/providers/index.ts`. The `register-builtins.ts` module iterates the catalog at startup and looks up each ID in this map — a missing entry is a fatal error.
|
|
2238
|
+
6. **Optional: native Twilio voice builder** — If the provider uses `native-twilio` call mode, add a `NativeTwilioVoiceSpec` entry to the `nativeVoiceSpecs` map in `src/tts/providers/register-builtins.ts`. Synthesized-play providers skip this step entirely.
|
|
2239
|
+
|
|
2240
|
+
No hardcoded enum edits are required — the `TtsProviderId` union in `types.ts` uses an open string union (`(string & {})`), the config schema reads valid IDs from the catalog, and the call strategy dispatches based on the catalog's `callMode` field. The registry, resolver, orchestrator, and call strategy all automatically pick up the new provider when selected via `services.tts.provider`.
|
|
2241
|
+
|
|
2242
|
+
**Key source files:**
|
|
2243
|
+
|
|
2244
|
+
| File | Purpose |
|
|
2245
|
+
| ---------------------------------------------------------- | -------------------------------------------------------------------------------------------- |
|
|
2246
|
+
| `src/tts/provider-catalog.ts` | Canonical provider catalog: single source of truth for provider IDs and metadata |
|
|
2247
|
+
| `src/tts/types.ts` | Core domain types: `TtsProvider`, `TtsProviderId`, `TtsCallMode`, `TtsUseCase`, capabilities |
|
|
2248
|
+
| `src/tts/provider-registry.ts` | Runtime provider registry: register, lookup, list |
|
|
2249
|
+
| `src/tts/tts-config-resolver.ts` | Config resolver: `resolveTtsConfig()` reads `services.tts` and returns resolved |
|
|
2250
|
+
| `src/tts/synthesize-text.ts` | Top-level orchestrator: `synthesizeText()` entry point |
|
|
2251
|
+
| `src/tts/providers/register-builtins.ts` | Startup registration of built-in providers (catalog-checked) |
|
|
2252
|
+
| `src/tts/providers/elevenlabs-provider.ts` | ElevenLabs adapter implementation |
|
|
2253
|
+
| `src/tts/providers/fish-audio-provider.ts` | Fish Audio adapter implementation |
|
|
2254
|
+
| `src/config/schemas/tts.ts` | Zod schema for `services.tts` config block (catalog-driven valid provider IDs) |
|
|
2255
|
+
| `src/calls/tts-call-strategy.ts` | Explicit call strategy: resolves call mode from catalog, native voice spec registry |
|
|
2256
|
+
| `src/calls/voice-quality.ts` | Phone call integration: `resolveVoiceQualityProfile()` uses call strategy |
|
|
2257
|
+
| `meta/tts-provider-catalog.json` | Client artifact: provider metadata for macOS/iOS settings UI |
|
|
2258
|
+
| `src/tts/__tests__/provider-catalog-consistency.test.ts` | CI guard: catalog vs client artifact provider ID consistency |
|
|
2259
|
+
| `src/workspace/migrations/032-tts-provider-unification.ts` | Migration that materialised canonical `services.tts` fields |
|
|
2260
|
+
|
|
2029
2261
|
### Managed Profiler Runtime
|
|
2030
2262
|
|
|
2031
2263
|
Managed cloud assistants use Bun's built-in CPU and heap profiling to capture runtime performance data. The profiler subsystem consists of a persistent on-disk run store, a retention/pruning sweep, and HTTP routes for remote management.
|
|
@@ -2078,3 +2310,30 @@ These endpoints allow the platform (via vembda proxy) to enumerate, inspect, exp
|
|
|
2078
2310
|
| `src/util/platform.ts` | Profiler directory path helpers |
|
|
2079
2311
|
| `src/__tests__/profiler-run-store.test.ts` | Profiler store unit tests |
|
|
2080
2312
|
| `src/__tests__/profiler-routes.test.ts` | Profiler HTTP route tests |
|
|
2313
|
+
|
|
2314
|
+
### LLM Provider Transport — OpenAI Responses API
|
|
2315
|
+
|
|
2316
|
+
OpenAI inference uses the **Responses API** (`client.responses.stream()`), not the Chat Completions API. OpenAI-compatible providers (OpenRouter, Fireworks, Ollama) continue to use the chat-completions transport.
|
|
2317
|
+
|
|
2318
|
+
**Transport split:**
|
|
2319
|
+
|
|
2320
|
+
| Provider key | Transport class | API surface |
|
|
2321
|
+
| ------------ | ------------------------- | --------------------------- |
|
|
2322
|
+
| `openai` | `OpenAIResponsesProvider` | `client.responses.stream()` |
|
|
2323
|
+
| `openrouter` | `OpenRouterProvider` | `chat.completions.create()` |
|
|
2324
|
+
| `fireworks` | `FireworksProvider` | `chat.completions.create()` |
|
|
2325
|
+
| `ollama` | `OllamaProvider` | `chat.completions.create()` |
|
|
2326
|
+
|
|
2327
|
+
The registry (`src/providers/registry.ts`) imports `OpenAIResponsesProvider` from `openai/client.ts` and wires it to the `openai` key. The chat-completions transport (`OpenAIChatCompletionsProvider`) remains available for OpenAI-compatible providers that implement the Chat Completions API.
|
|
2328
|
+
|
|
2329
|
+
Both transports produce the same `ProviderResponse` contract so downstream code (agent loop, context management, conversation history) is transport-agnostic.
|
|
2330
|
+
|
|
2331
|
+
**Key files:**
|
|
2332
|
+
|
|
2333
|
+
| File | Purpose |
|
|
2334
|
+
| ------------------------------------------------------ | -------------------------------------------------------------------- |
|
|
2335
|
+
| `src/providers/openai/responses-provider.ts` | Responses API transport (streaming, tool calls, usage mapping) |
|
|
2336
|
+
| `src/providers/openai/chat-completions-provider.ts` | Chat Completions transport (OpenAI-compatible providers) |
|
|
2337
|
+
| `src/providers/openai/client.ts` | Re-exports both transports + `validateOpenAIApiKey()` |
|
|
2338
|
+
| `src/providers/registry.ts` | Provider initialization (wires `openai` → `OpenAIResponsesProvider`) |
|
|
2339
|
+
| `src/__tests__/openai-responses-cutover-guard.test.ts` | CI guard preventing chat-completions regression in OpenAI path |
|
package/Dockerfile
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
# Use debian as base image
|
|
2
|
-
FROM debian:trixie@sha256:
|
|
2
|
+
FROM debian:trixie-slim@sha256:4ffb3a1511099754cddc70eb1b12e50ffdb67619aa0ab6c13fcd800a78ef7c7a AS builder
|
|
3
3
|
|
|
4
4
|
WORKDIR /app
|
|
5
5
|
|
|
@@ -20,16 +20,27 @@ ENV PATH="/root/.bun/bin:${PATH}"
|
|
|
20
20
|
COPY packages/ces-contracts ./packages/ces-contracts
|
|
21
21
|
COPY packages/credential-storage ./packages/credential-storage
|
|
22
22
|
COPY packages/egress-proxy ./packages/egress-proxy
|
|
23
|
-
|
|
24
23
|
# Install assistant dependencies first for cache reuse
|
|
25
24
|
COPY assistant/package.json assistant/bun.lock ./assistant/
|
|
26
25
|
RUN cd /app/assistant && bun install --frozen-lockfile
|
|
27
26
|
|
|
27
|
+
# Copy bundled first-party skills and install their deps. The repo-root
|
|
28
|
+
# .dockerignore is the single source of truth for which skills (and which
|
|
29
|
+
# files within them) ship in the assistant image -- adding or removing a
|
|
30
|
+
# bundled skill is a .dockerignore edit, not a Dockerfile edit.
|
|
31
|
+
COPY skills ./skills
|
|
32
|
+
RUN set -eu; for pkg in /app/skills/*/package.json; do \
|
|
33
|
+
[ -e "$pkg" ] || continue; \
|
|
34
|
+
dir="$(dirname "$pkg")"; \
|
|
35
|
+
echo "Installing dependencies for $dir"; \
|
|
36
|
+
(cd "$dir" && (bun install --frozen-lockfile 2>/dev/null || bun install)); \
|
|
37
|
+
done
|
|
38
|
+
|
|
28
39
|
# Copy source
|
|
29
40
|
COPY assistant ./assistant
|
|
30
41
|
|
|
31
42
|
# Final stage
|
|
32
|
-
FROM debian:trixie@sha256:
|
|
43
|
+
FROM debian:trixie-slim@sha256:4ffb3a1511099754cddc70eb1b12e50ffdb67619aa0ab6c13fcd800a78ef7c7a AS runner
|
|
33
44
|
|
|
34
45
|
WORKDIR /app/assistant
|
|
35
46
|
|
package/README.md
CHANGED
|
@@ -47,9 +47,9 @@ cp .env.example .env
|
|
|
47
47
|
|
|
48
48
|
## Update Bulletin
|
|
49
49
|
|
|
50
|
-
|
|
50
|
+
Release notes are surfaced via a background conversation dispatched at daemon startup. Workspace migrations write release notes to `<workspace>/UPDATES.md`; `runUpdateBulletinJobIfNeeded()` then spawns a `conversationType: "background"` conversation (via `wakeAgentForOpportunity()`) whenever the file's content hash changes. The agent uses judgment to surface updates to the user when relevant, and deletes the file when done.
|
|
51
51
|
|
|
52
|
-
**For release maintainers:**
|
|
52
|
+
**For release maintainers:** Add a new migration under `assistant/src/workspace/migrations/0XX-release-notes-<slug>.ts` with the release notes inline as a string literal, and append the export to `WORKSPACE_MIGRATIONS` in `assistant/src/workspace/migrations/registry.ts`. Migrations are append-only. Idempotency is handled by the workspace-migration runner — `runWorkspaceMigrations()` records each migration's `WorkspaceMigration.id` in `<workspace>/data/.workspace-migrations.json` and never re-runs an ID that is already in the `applied` set, so release-notes migrations do not need an in-file guard. Skip the migration entirely for releases with no user/assistant-facing changes.
|
|
53
53
|
|
|
54
54
|
## Usage
|
|
55
55
|
|
|
@@ -86,7 +86,6 @@ bun run src/index.ts # interactive CLI session
|
|
|
86
86
|
| `assistant config set\|get\|list` | Manage configuration |
|
|
87
87
|
| `assistant keys set\|list\|delete` | Manage API keys in secure storage |
|
|
88
88
|
| `assistant trust list\|remove\|clear` | Manage trust rules |
|
|
89
|
-
| `assistant doctor` | Run diagnostic checks |
|
|
90
89
|
|
|
91
90
|
## Project Structure
|
|
92
91
|
|
|
@@ -204,7 +203,7 @@ The runtime exposes a RESTful HTTP API for Twilio configuration, credential mana
|
|
|
204
203
|
| Method | Path | Description |
|
|
205
204
|
| ------ | ------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
206
205
|
| GET | `/v1/integrations/twilio/config` | Returns current state: `hasCredentials` (boolean) and `phoneNumber` (if assigned) |
|
|
207
|
-
| POST | `/v1/integrations/twilio/credentials` | Validates and stores Account SID and Auth Token in secure storage (CES / encrypted file store)
|
|
206
|
+
| POST | `/v1/integrations/twilio/credentials` | Validates and stores Account SID and Auth Token in secure storage (CES / encrypted file store) |
|
|
208
207
|
| DELETE | `/v1/integrations/twilio/credentials` | Removes stored credentials. Preserves the phone number in config so re-entering credentials resumes working without reassigning the number. |
|
|
209
208
|
| GET | `/v1/integrations/twilio/numbers` | Lists all incoming phone numbers on the Twilio account with their capabilities |
|
|
210
209
|
| POST | `/v1/integrations/twilio/numbers/provision` | Purchases a new phone number. Accepts optional `areaCode` and `country`. Auto-assigns and configures webhooks when ingress is available. |
|