@vellumai/assistant 0.6.2 → 0.6.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +273 -10
- package/Dockerfile +2 -3
- package/bun.lock +41 -49
- package/bunfig.toml +3 -0
- package/docs/architecture/memory.md +1 -1
- package/docs/backup-troubleshooting.md +52 -0
- package/docs/browser-use-architecture-phase2.md +174 -0
- package/docs/stt-provider-onboarding.md +120 -0
- package/knip.json +12 -2
- package/node_modules/@vellumai/ces-contracts/bun.lock +8 -6
- package/node_modules/@vellumai/ces-contracts/package.json +3 -3
- package/node_modules/@vellumai/ces-contracts/src/rpc.ts +42 -0
- package/openapi.yaml +1111 -86
- package/package.json +40 -42
- package/scripts/generate-openapi.ts +0 -2
- package/scripts/test.sh +73 -18
- package/src/__tests__/acp-session.test.ts +43 -0
- package/src/__tests__/agent-image-optimize.test.ts +28 -0
- package/src/__tests__/agent-loop.test.ts +123 -0
- package/src/__tests__/anthropic-provider.test.ts +263 -10
- package/src/__tests__/app-builder-tool-scripts.test.ts +1 -0
- package/src/__tests__/app-executors.test.ts +1 -0
- package/src/__tests__/app-source-watcher.test.ts +37 -11
- package/src/__tests__/approval-routes-http.test.ts +178 -1
- package/src/__tests__/auto-analysis-end-to-end.test.ts +550 -0
- package/src/__tests__/auto-analysis-prompt.test.ts +50 -0
- package/src/__tests__/browser-fill-credential.test.ts +240 -94
- package/src/__tests__/browser-manager.test.ts +40 -27
- package/src/__tests__/browser-skill-baseline-tool-payload.test.ts +2 -2
- package/src/__tests__/browser-skill-endstate.test.ts +31 -7
- package/src/__tests__/btw-routes.test.ts +7 -0
- package/src/__tests__/call-controller.test.ts +581 -20
- package/src/__tests__/catalog-files.test.ts +1000 -0
- package/src/__tests__/channel-approvals.test.ts +53 -0
- package/src/__tests__/channel-invite-transport.test.ts +2 -2
- package/src/__tests__/channel-readiness-routes.test.ts +16 -20
- package/src/__tests__/channel-readiness-service.test.ts +12 -7
- package/src/__tests__/checker.test.ts +157 -10
- package/src/__tests__/clawhub-files.test.ts +347 -0
- package/src/__tests__/commit-message-enrichment-service.test.ts +36 -19
- package/src/__tests__/config-analysis.test.ts +100 -0
- package/src/__tests__/config-managed-gemini-defaults.test.ts +326 -0
- package/src/__tests__/config-schema-cmd.test.ts +2 -2
- package/src/__tests__/config-schema.test.ts +1248 -224
- package/src/__tests__/config-watcher-cleanup-throttle.test.ts +339 -0
- package/src/__tests__/config-watcher.test.ts +43 -8
- package/src/__tests__/confirmation-request-guardian-bridge.test.ts +23 -0
- package/src/__tests__/contact-store-user-file.test.ts +512 -0
- package/src/__tests__/contacts-write.test.ts +197 -0
- package/src/__tests__/context-overflow-approval.test.ts +16 -1
- package/src/__tests__/context-window-manager.test.ts +88 -0
- package/src/__tests__/conversation-abort-tool-results.test.ts +2 -0
- package/src/__tests__/conversation-agent-loop-overflow.test.ts +2 -1
- package/src/__tests__/conversation-agent-loop.test.ts +99 -3
- package/src/__tests__/conversation-analysis-routes.test.ts +2 -2
- package/src/__tests__/conversation-attachments.test.ts +80 -4
- package/src/__tests__/conversation-confirmation-signals.test.ts +290 -0
- package/src/__tests__/conversation-error.test.ts +70 -0
- package/src/__tests__/conversation-fork-crud.test.ts +17 -0
- package/src/__tests__/conversation-history-web-search.test.ts +12 -4
- package/src/__tests__/conversation-host-access-routes.test.ts +229 -0
- package/src/__tests__/conversation-init.benchmark.test.ts +6 -1
- package/src/__tests__/conversation-inject-context.test.ts +103 -0
- package/src/__tests__/conversation-launcher-skill-regression.test.ts +51 -0
- package/src/__tests__/conversation-list-source.test.ts +145 -0
- package/src/__tests__/conversation-pre-run-repair.test.ts +2 -0
- package/src/__tests__/conversation-provider-retry-repair.test.ts +2 -0
- package/src/__tests__/conversation-queue.test.ts +946 -62
- package/src/__tests__/conversation-routes-disk-view.test.ts +275 -0
- package/src/__tests__/conversation-routes-guardian-reply.test.ts +16 -0
- package/src/__tests__/conversation-routes-slash-commands.test.ts +1 -0
- package/src/__tests__/conversation-runtime-assembly.test.ts +324 -46
- package/src/__tests__/conversation-skill-tools.test.ts +7 -4
- package/src/__tests__/conversation-slash-commands.test.ts +33 -0
- package/src/__tests__/conversation-slash-queue.test.ts +89 -18
- package/src/__tests__/conversation-slash-unknown.test.ts +2 -0
- package/src/__tests__/conversation-starter-routes.test.ts +126 -0
- package/src/__tests__/conversation-starters-cadence.test.ts +161 -0
- package/src/__tests__/conversation-store.test.ts +195 -0
- package/src/__tests__/conversation-tool-setup-batch-authorized.test.ts +226 -0
- package/src/__tests__/conversation-workspace-cache-state.test.ts +193 -0
- package/src/__tests__/conversation-workspace-injection.test.ts +2 -0
- package/src/__tests__/conversation-workspace-tool-tracking.test.ts +2 -0
- package/src/__tests__/credential-execution-approval-bridge.test.ts +32 -1
- package/src/__tests__/credential-health-service.test.ts +352 -0
- package/src/__tests__/credential-security-invariants.test.ts +6 -3
- package/src/__tests__/credential-vault-unit.test.ts +383 -7
- package/src/__tests__/credential-vault.test.ts +152 -13
- package/src/__tests__/credentials-cli.test.ts +42 -18
- package/src/__tests__/cross-provider-web-search.test.ts +146 -35
- package/src/__tests__/date-context.test.ts +4 -4
- package/src/__tests__/deterministic-verification-control-plane.test.ts +10 -1
- package/src/__tests__/device-id.test.ts +112 -0
- package/src/__tests__/docker-signing-key-bootstrap.test.ts +167 -4
- package/src/__tests__/dynamic-skill-workflow-prompt.test.ts +1 -3
- package/src/__tests__/email-html-renderer.test.ts +71 -0
- package/src/__tests__/email-invite-adapter.test.ts +36 -32
- package/src/__tests__/embedding-managed-proxy-selection.test.ts +256 -0
- package/src/__tests__/emit-event-signal.test.ts +71 -0
- package/src/__tests__/extension-id-sync-guard.test.ts +222 -0
- package/src/__tests__/fixtures/mock-chrome-extension.ts +386 -0
- package/src/__tests__/gateway-only-enforcement.test.ts +206 -1
- package/src/__tests__/gateway-only-guard.test.ts +2 -0
- package/src/__tests__/gemini-provider.test.ts +66 -2
- package/src/__tests__/get-skill-detail-audit.test.ts +325 -0
- package/src/__tests__/gmail-archive-fallback.test.ts +193 -0
- package/src/__tests__/gmail-archive-gate.test.ts +246 -0
- package/src/__tests__/gmail-preferences.test.ts +117 -0
- package/src/__tests__/guardian-routing-invariants.test.ts +70 -2
- package/src/__tests__/headless-browser-interactions.test.ts +738 -359
- package/src/__tests__/headless-browser-mode.test.ts +614 -0
- package/src/__tests__/headless-browser-navigate.test.ts +528 -49
- package/src/__tests__/headless-browser-read-tools.test.ts +274 -100
- package/src/__tests__/headless-browser-snapshot.test.ts +250 -77
- package/src/__tests__/heartbeat-service.test.ts +70 -17
- package/src/__tests__/home-state-routes.test.ts +162 -0
- package/src/__tests__/host-bash-proxy.test.ts +145 -1
- package/src/__tests__/host-browser-e2e-cloud.test.ts +596 -0
- package/src/__tests__/host-browser-e2e-self-hosted-capability.test.ts +286 -0
- package/src/__tests__/host-browser-e2e-self-hosted.test.ts +374 -0
- package/src/__tests__/host-browser-event-routes.test.ts +350 -0
- package/src/__tests__/host-browser-proxy.test.ts +444 -0
- package/src/__tests__/host-browser-routes.test.ts +198 -0
- package/src/__tests__/host-browser-ws-events-e2e.test.ts +423 -0
- package/src/__tests__/host-cu-proxy.test.ts +166 -1
- package/src/__tests__/host-file-proxy.test.ts +185 -1
- package/src/__tests__/host-file-read-tool.test.ts +52 -0
- package/src/__tests__/host-proxy-interface.test.ts +165 -0
- package/src/__tests__/host-shell-tool.test.ts +1 -11
- package/src/__tests__/http-user-message-parity.test.ts +1 -0
- package/src/__tests__/identity-intro-cache.test.ts +40 -10
- package/src/__tests__/init-feature-flag-overrides.test.ts +38 -112
- package/src/__tests__/integration-status.test.ts +6 -7
- package/src/__tests__/jobs-store-upsert-debounced.test.ts +141 -0
- package/src/__tests__/list-messages-tool-merge.test.ts +37 -12
- package/src/__tests__/llm-context-normalization.test.ts +488 -0
- package/src/__tests__/llm-context-route-provider.test.ts +86 -5
- package/src/__tests__/llm-usage-store.test.ts +363 -0
- package/src/__tests__/mcp-client-auth.test.ts +40 -4
- package/src/__tests__/mcp-health-check.test.ts +10 -3
- package/src/__tests__/media-stream-output.test.ts +555 -0
- package/src/__tests__/media-stream-parser.test.ts +374 -0
- package/src/__tests__/media-stream-server-integration.test.ts +1234 -0
- package/src/__tests__/media-stream-stt-session.test.ts +588 -0
- package/src/__tests__/media-turn-detector.test.ts +440 -0
- package/src/__tests__/message-queue.test.ts +125 -0
- package/src/__tests__/migration-cross-version-compatibility.test.ts +3 -1
- package/src/__tests__/migration-export-http.test.ts +67 -8
- package/src/__tests__/migration-export-streaming.test.ts +66 -0
- package/src/__tests__/migration-import-commit-http.test.ts +109 -7
- package/src/__tests__/migration-import-preflight-http.test.ts +6 -5
- package/src/__tests__/migration-validate-http.test.ts +3 -3
- package/src/__tests__/mock-gateway-ipc.ts +151 -0
- package/src/__tests__/model-intents.test.ts +2 -2
- package/src/__tests__/native-host-marker-sync-guard.test.ts +157 -0
- package/src/__tests__/oauth-apps-routes.test.ts +18 -12
- package/src/__tests__/oauth-cli.test.ts +709 -60
- package/src/__tests__/oauth-connect-orchestrator.test.ts +118 -24
- package/src/__tests__/oauth-provider-seed-logos.test.ts +23 -0
- package/src/__tests__/oauth-provider-serializer.test.ts +147 -10
- package/src/__tests__/oauth-provider-visibility.test.ts +19 -21
- package/src/__tests__/oauth-providers-routes.test.ts +52 -14
- package/src/__tests__/oauth-store.test.ts +1465 -176
- package/src/__tests__/oauth2-gateway-transport.test.ts +460 -26
- package/src/__tests__/onboarding-template-contract.test.ts +81 -70
- package/src/__tests__/openai-provider.test.ts +178 -2
- package/src/__tests__/openai-responses-cutover-guard.test.ts +184 -0
- package/src/__tests__/openai-responses-provider.test.ts +1105 -0
- package/src/__tests__/openrouter-token-estimation.test.ts +100 -0
- package/src/__tests__/outlook-categories.test.ts +1 -1
- package/src/__tests__/outlook-client-automation.test.ts +1 -1
- package/src/__tests__/outlook-compose-tools.test.ts +1 -1
- package/src/__tests__/outlook-email-watcher.test.ts +1 -1
- package/src/__tests__/outlook-follow-up.test.ts +1 -1
- package/src/__tests__/outlook-messaging-provider.test.ts +2 -2
- package/src/__tests__/outlook-trash.test.ts +1 -1
- package/src/__tests__/outlook-unsubscribe.test.ts +32 -3
- package/src/__tests__/permission-checker-host-gate.test.ts +74 -14
- package/src/__tests__/permission-mode.test.ts +28 -56
- package/src/__tests__/persona-resolver.test.ts +251 -0
- package/src/__tests__/platform-bash-auto-approve.test.ts +4 -0
- package/src/__tests__/platform-callback-registration.test.ts +19 -0
- package/src/__tests__/platform.test.ts +92 -1
- package/src/__tests__/post-turn-tool-result-truncation.test.ts +343 -0
- package/src/__tests__/prechat-onboarding-contract.test.ts +267 -0
- package/src/__tests__/pricing.test.ts +174 -0
- package/src/__tests__/proxy-approval-callback.test.ts +18 -0
- package/src/__tests__/qdrant-manager.test.ts +29 -8
- package/src/__tests__/regenerate-fire-and-forget-trace.test.ts +194 -0
- package/src/__tests__/relationship-state-contract.test.ts +175 -0
- package/src/__tests__/relay-server.test.ts +423 -5
- package/src/__tests__/require-fresh-approval.test.ts +40 -1
- package/src/__tests__/sanitize-config-for-transfer.test.ts +132 -0
- package/src/__tests__/schedule-routes.test.ts +162 -0
- package/src/__tests__/search-skills-unified.test.ts +118 -0
- package/src/__tests__/secret-detection-handler.test.ts +84 -0
- package/src/__tests__/secret-ingress-http.test.ts +1 -0
- package/src/__tests__/secret-scanner-executor.test.ts +4 -0
- package/src/__tests__/secure-keys.test.ts +107 -0
- package/src/__tests__/send-endpoint-busy.test.ts +8 -1
- package/src/__tests__/sequence-store.test.ts +1 -1
- package/src/__tests__/server-history-render.test.ts +49 -0
- package/src/__tests__/set-permission-mode.test.ts +13 -250
- package/src/__tests__/settings-routes.test.ts +201 -0
- package/src/__tests__/skill-load-feature-flag.test.ts +1 -0
- package/src/__tests__/skills-file-content-endpoint.test.ts +801 -0
- package/src/__tests__/skills-files-catalog-fallback.test.ts +738 -0
- package/src/__tests__/skills.test.ts +5 -2
- package/src/__tests__/skillssh-files.test.ts +446 -0
- package/src/__tests__/slack-block-formatting.test.ts +110 -0
- package/src/__tests__/slack-channel-config.test.ts +576 -16
- package/src/__tests__/stt-catalog-parity.test.ts +282 -0
- package/src/__tests__/stt-stream-session.test.ts +535 -0
- package/src/__tests__/subagent-detail.test.ts +44 -2
- package/src/__tests__/subagent-disposal.test.ts +1 -0
- package/src/__tests__/subagent-fork-notifications.test.ts +291 -0
- package/src/__tests__/subagent-fork-spawn.test.ts +384 -0
- package/src/__tests__/subagent-manager-notify.test.ts +1 -0
- package/src/__tests__/subagent-notify-parent.test.ts +1 -0
- package/src/__tests__/subagent-spawn-tool-fork.test.ts +411 -0
- package/src/__tests__/subagent-tools.test.ts +1 -0
- package/src/__tests__/subagent-types.test.ts +1 -0
- package/src/__tests__/system-prompt-ask-mode.test.ts +27 -71
- package/src/__tests__/system-prompt.test.ts +184 -27
- package/src/__tests__/task-scheduler.test.ts +32 -6
- package/src/__tests__/telegram-config.test.ts +10 -13
- package/src/__tests__/telephony-stt-routing.test.ts +329 -0
- package/src/__tests__/terminal-tools.test.ts +25 -5
- package/src/__tests__/test-preload.ts +18 -0
- package/src/__tests__/test-support/browser-skill-harness.ts +4 -1
- package/src/__tests__/tool-approval-handler.test.ts +73 -0
- package/src/__tests__/tool-executor-lifecycle-events.test.ts +9 -5
- package/src/__tests__/tool-executor-shell-integration.test.ts +4 -0
- package/src/__tests__/tool-executor.test.ts +33 -24
- package/src/__tests__/tool-result-truncation.test.ts +36 -0
- package/src/__tests__/tool-side-effects-slack-dm.test.ts +22 -0
- package/src/__tests__/top-level-renderer.test.ts +73 -1
- package/src/__tests__/transport-hints-queue.test.ts +14 -29
- package/src/__tests__/trust-store.test.ts +7 -1
- package/src/__tests__/trusted-contact-approval-notifier.test.ts +1 -1
- package/src/__tests__/trusted-contact-inline-approval-integration.test.ts +109 -0
- package/src/__tests__/tts-catalog-parity.test.ts +345 -0
- package/src/__tests__/twilio-routes-twiml.test.ts +512 -114
- package/src/__tests__/twilio-routes.test.ts +376 -0
- package/src/__tests__/unicode.test.ts +293 -0
- package/src/__tests__/update-bulletin-format.test.ts +59 -0
- package/src/__tests__/update-bulletin.test.ts +206 -5
- package/src/__tests__/usage-routes.test.ts +25 -4
- package/src/__tests__/user-reference.test.ts +46 -61
- package/src/__tests__/v2-consent-policy.test.ts +103 -0
- package/src/__tests__/verification-control-plane-policy.test.ts +4 -0
- package/src/__tests__/voice-config-update.test.ts +403 -0
- package/src/__tests__/voice-quality.test.ts +434 -19
- package/src/__tests__/workspace-heartbeat-service.test.ts +7 -0
- package/src/__tests__/workspace-migration-033-stt-service-explicit-config.test.ts +547 -0
- package/src/__tests__/workspace-migration-034-remove-calls-voice-transcription-provider.test.ts +596 -0
- package/src/__tests__/workspace-migration-drop-user-md.test.ts +368 -0
- package/src/__tests__/workspace-migration-meets.test.ts +244 -0
- package/src/__tests__/workspace-migration-seed-device-id.test.ts +14 -20
- package/src/__tests__/workspace-policy.test.ts +2 -0
- package/src/acp/client-handler.ts +30 -4
- package/src/agent/image-optimize.ts +24 -12
- package/src/agent/loop.ts +55 -9
- package/src/approvals/guardian-request-resolvers.ts +21 -15
- package/src/backup/__tests__/backup-key.test.ts +152 -0
- package/src/backup/__tests__/backup-worker.test.ts +767 -0
- package/src/backup/__tests__/list-snapshots.test.ts +87 -0
- package/src/backup/__tests__/local-writer.test.ts +218 -0
- package/src/backup/__tests__/offsite-writer.test.ts +641 -0
- package/src/backup/__tests__/paths.test.ts +300 -0
- package/src/backup/__tests__/restore.test.ts +498 -0
- package/src/backup/__tests__/snapshot-lock.test.ts +352 -0
- package/src/backup/__tests__/stream-crypt.test.ts +228 -0
- package/src/backup/backup-key.ts +137 -0
- package/src/backup/backup-worker.ts +459 -0
- package/src/backup/list-snapshots.ts +147 -0
- package/src/backup/local-writer.ts +133 -0
- package/src/backup/offsite-writer.ts +222 -0
- package/src/backup/paths.ts +226 -0
- package/src/backup/restore.ts +322 -0
- package/src/backup/snapshot-lock.ts +431 -0
- package/src/backup/stream-crypt.ts +263 -0
- package/src/browser-session/__tests__/manager.test.ts +297 -0
- package/src/browser-session/backends/cdp-inspect.ts +30 -0
- package/src/browser-session/backends/extension.ts +26 -0
- package/src/browser-session/backends/local.ts +24 -0
- package/src/browser-session/events.ts +164 -0
- package/src/browser-session/index.ts +27 -0
- package/src/browser-session/manager.ts +159 -0
- package/src/browser-session/types.ts +28 -0
- package/src/bundler/package-resolver.ts +4 -0
- package/src/calls/audio-store.ts +11 -5
- package/src/calls/call-controller.ts +226 -71
- package/src/calls/call-domain.ts +9 -0
- package/src/calls/call-speech-output.ts +190 -0
- package/src/calls/call-transport.ts +77 -0
- package/src/calls/media-stream-audio-transcode.ts +173 -0
- package/src/calls/media-stream-output.ts +660 -0
- package/src/calls/media-stream-parser.ts +300 -0
- package/src/calls/media-stream-protocol.ts +166 -0
- package/src/calls/media-stream-server.ts +592 -0
- package/src/calls/media-stream-stt-session.ts +460 -0
- package/src/calls/media-turn-detector.ts +230 -0
- package/src/calls/relay-server.ts +90 -75
- package/src/calls/resolve-call-tts-provider.ts +136 -0
- package/src/calls/telephony-stt-routing.ts +145 -0
- package/src/calls/tts-call-strategy.ts +161 -0
- package/src/calls/tts-text-sanitizer.ts +32 -16
- package/src/calls/twilio-routes.ts +281 -17
- package/src/calls/voice-quality.ts +78 -35
- package/src/calls/voice-session-bridge.ts +8 -1
- package/src/channels/__tests__/types.test.ts +134 -0
- package/src/channels/types.ts +69 -3
- package/src/cli/__tests__/run-assistant-command.ts +11 -1
- package/src/cli/commands/__tests__/backup.test.ts +1165 -0
- package/src/cli/commands/__tests__/domain-register.test.ts +234 -0
- package/src/cli/commands/__tests__/domain-status.test.ts +132 -0
- package/src/cli/commands/__tests__/email-attachment.test.ts +422 -0
- package/src/cli/commands/__tests__/email-download.test.ts +16 -1
- package/src/cli/commands/__tests__/email-list.test.ts +22 -4
- package/src/cli/commands/__tests__/email-register.test.ts +4 -4
- package/src/cli/commands/__tests__/email-send.test.ts +37 -4
- package/src/cli/commands/__tests__/email-status.test.ts +5 -1
- package/src/cli/commands/__tests__/email-unregister.test.ts +34 -5
- package/src/cli/commands/backup.ts +993 -0
- package/src/cli/commands/conversations.ts +77 -0
- package/src/cli/commands/credentials.ts +3 -4
- package/src/cli/commands/domain.ts +210 -0
- package/src/cli/commands/email.ts +273 -16
- package/src/cli/commands/mcp.ts +16 -4
- package/src/cli/commands/oauth/__tests__/connect.test.ts +56 -44
- package/src/cli/commands/oauth/__tests__/disconnect.test.ts +21 -21
- package/src/cli/commands/oauth/__tests__/mode.test.ts +17 -17
- package/src/cli/commands/oauth/__tests__/ping.test.ts +16 -16
- package/src/cli/commands/oauth/__tests__/providers-delete.test.ts +32 -33
- package/src/cli/commands/oauth/__tests__/providers-register.test.ts +330 -0
- package/src/cli/commands/oauth/__tests__/providers-update.test.ts +117 -12
- package/src/cli/commands/oauth/__tests__/status.test.ts +10 -10
- package/src/cli/commands/oauth/__tests__/token.test.ts +7 -7
- package/src/cli/commands/oauth/apps.ts +7 -4
- package/src/cli/commands/oauth/connect.ts +6 -3
- package/src/cli/commands/oauth/disconnect.ts +1 -1
- package/src/cli/commands/oauth/mode.ts +12 -3
- package/src/cli/commands/oauth/providers.ts +215 -36
- package/src/cli/commands/oauth/shared.ts +7 -6
- package/src/cli/commands/platform/__tests__/callback-routes-list.test.ts +254 -0
- package/src/cli/commands/platform/__tests__/connect.test.ts +6 -0
- package/src/cli/commands/platform/__tests__/disconnect.test.ts +7 -1
- package/src/cli/commands/platform/__tests__/status.test.ts +6 -0
- package/src/cli/commands/platform/index.ts +107 -10
- package/src/cli/commands/usage.ts +10 -9
- package/src/cli/lib/daemon-credential-client.ts +4 -0
- package/src/cli/program.ts +30 -4
- package/src/config/__tests__/backup-schema.test.ts +134 -0
- package/src/config/assistant-feature-flags.ts +61 -62
- package/src/config/bundled-skills/app-builder/SKILL.md +26 -249
- package/src/config/bundled-skills/app-builder/references/CUSTOM_ROUTES.md +141 -0
- package/src/config/bundled-skills/app-builder/references/INTERACTION_HOOKS.md +56 -0
- package/src/config/bundled-skills/app-builder/references/WIDGETS.md +125 -0
- package/src/config/bundled-skills/browser/SKILL.md +30 -5
- package/src/config/bundled-skills/browser/TOOLS.json +123 -0
- package/src/config/bundled-skills/browser/tools/browser-attach.ts +12 -0
- package/src/config/bundled-skills/browser/tools/browser-detach.ts +12 -0
- package/src/config/bundled-skills/browser/tools/browser-status.ts +12 -0
- package/src/config/bundled-skills/browser/tools/browser-wait-for-download.ts +17 -0
- package/src/config/bundled-skills/contacts/SKILL.md +5 -2
- package/src/config/bundled-skills/document/SKILL.md +4 -0
- package/src/config/bundled-skills/gmail/SKILL.md +54 -8
- package/src/config/bundled-skills/gmail/TOOLS.json +33 -3
- package/src/config/bundled-skills/gmail/tools/gmail-archive.ts +116 -9
- package/src/config/bundled-skills/gmail/tools/gmail-outreach-scan.ts +138 -11
- package/src/config/bundled-skills/gmail/tools/gmail-preferences-tool.ts +59 -0
- package/src/config/bundled-skills/gmail/tools/gmail-preferences.ts +82 -0
- package/src/config/bundled-skills/gmail/tools/gmail-sender-digest.ts +113 -17
- package/src/config/bundled-skills/gmail/tools/gmail-unsubscribe.ts +2 -2
- package/src/config/bundled-skills/media-processing/SKILL.md +3 -9
- package/src/config/bundled-skills/media-processing/TOOLS.json +1 -6
- package/src/config/bundled-skills/media-processing/__tests__/audio-transcribe.test.ts +125 -0
- package/src/config/bundled-skills/media-processing/__tests__/extract-keyframes.test.ts +181 -0
- package/src/config/bundled-skills/media-processing/__tests__/preprocess-audio.test.ts +141 -0
- package/src/config/bundled-skills/media-processing/services/audio-transcribe.ts +32 -87
- package/src/config/bundled-skills/media-processing/services/preprocess.ts +8 -4
- package/src/config/bundled-skills/media-processing/tools/extract-keyframes.ts +0 -10
- package/src/config/bundled-skills/messaging/SKILL.md +3 -3
- package/src/config/bundled-skills/messaging/tools/messaging-archive-by-sender.ts +2 -2
- package/src/config/bundled-skills/outlook/SKILL.md +9 -2
- package/src/config/bundled-skills/outlook/tools/outlook-unsubscribe.ts +2 -2
- package/src/config/bundled-skills/phone-calls/SKILL.md +2 -2
- package/src/config/bundled-skills/phone-calls/references/CONFIG.md +27 -18
- package/src/config/bundled-skills/phone-calls/references/TROUBLESHOOTING.md +3 -3
- package/src/config/bundled-skills/settings/TOOLS.json +3 -3
- package/src/config/bundled-skills/settings/tools/voice-config-update.ts +26 -22
- package/src/config/bundled-skills/slack/SKILL.md +1 -0
- package/src/config/bundled-skills/subagent/SKILL.md +21 -0
- package/src/config/bundled-skills/subagent/TOOLS.json +8 -4
- package/src/config/bundled-skills/tasks/SKILL.md +5 -0
- package/src/config/bundled-skills/transcribe/SKILL.md +9 -14
- package/src/config/bundled-skills/transcribe/TOOLS.json +2 -7
- package/src/config/bundled-skills/transcribe/tools/transcribe-media.test.ts +256 -0
- package/src/config/bundled-skills/transcribe/tools/transcribe-media.ts +38 -188
- package/src/config/bundled-tool-registry.ts +8 -0
- package/src/config/env-registry.ts +38 -0
- package/src/config/env.ts +49 -4
- package/src/config/feature-flag-registry.json +85 -14
- package/src/config/loader.ts +82 -13
- package/src/config/sanitize-for-transfer.ts +47 -0
- package/src/config/schema.ts +81 -15
- package/src/config/schemas/__tests__/stt.test.ts +43 -0
- package/src/config/schemas/analysis.ts +51 -0
- package/src/config/schemas/backup.ts +72 -0
- package/src/config/schemas/calls.ts +1 -26
- package/src/config/schemas/elevenlabs.ts +0 -59
- package/src/config/schemas/filing.ts +47 -7
- package/src/config/schemas/heartbeat.ts +27 -5
- package/src/config/schemas/host-browser.ts +112 -0
- package/src/config/schemas/inference.ts +1 -1
- package/src/config/schemas/memory-lifecycle.ts +14 -2
- package/src/config/schemas/memory-retrieval.ts +103 -0
- package/src/config/schemas/security.ts +0 -6
- package/src/config/schemas/services.ts +52 -0
- package/src/config/schemas/stt.ts +59 -0
- package/src/config/schemas/tts.ts +230 -0
- package/src/config/schemas/updates.ts +14 -0
- package/src/config/skills.ts +4 -0
- package/src/config/types.ts +4 -1
- package/src/contacts/contact-store.ts +56 -11
- package/src/contacts/contacts-write.ts +38 -1
- package/src/context/post-turn-tool-result-truncation.ts +177 -0
- package/src/context/tool-result-truncation.ts +2 -1
- package/src/context/window-manager.ts +61 -10
- package/src/credential-execution/approval-bridge.ts +49 -15
- package/src/credential-execution/executable-discovery.ts +12 -2
- package/src/credential-execution/process-manager.ts +33 -2
- package/src/credential-health/credential-health-service.ts +366 -0
- package/src/daemon/__tests__/conversation-lifecycle-auto-analyze.test.ts +324 -0
- package/src/daemon/__tests__/conversation-surfaces-launch.test.ts +497 -0
- package/src/daemon/__tests__/conversation-tool-setup.test.ts +195 -0
- package/src/daemon/__tests__/lifecycle-startup-ordering.test.ts +127 -0
- package/src/daemon/app-source-watcher.ts +35 -0
- package/src/daemon/config-watcher.ts +99 -5
- package/src/daemon/context-overflow-approval.ts +5 -0
- package/src/daemon/conversation-agent-loop-handlers.ts +23 -2
- package/src/daemon/conversation-agent-loop.ts +153 -42
- package/src/daemon/conversation-attachments.ts +40 -0
- package/src/daemon/conversation-error.ts +11 -0
- package/src/daemon/conversation-history.ts +40 -6
- package/src/daemon/conversation-launch.ts +220 -0
- package/src/daemon/conversation-lifecycle.ts +59 -9
- package/src/daemon/conversation-messaging.ts +37 -3
- package/src/daemon/conversation-notifiers.ts +5 -0
- package/src/daemon/conversation-process.ts +622 -13
- package/src/daemon/conversation-queue-manager.ts +24 -0
- package/src/daemon/conversation-runtime-assembly.ts +128 -36
- package/src/daemon/conversation-slash.ts +36 -0
- package/src/daemon/conversation-surfaces.ts +131 -40
- package/src/daemon/conversation-tool-setup.ts +99 -8
- package/src/daemon/conversation-usage.ts +7 -4
- package/src/daemon/conversation-workspace.ts +12 -0
- package/src/daemon/conversation.ts +292 -16
- package/src/daemon/date-context.ts +10 -10
- package/src/daemon/first-greeting.ts +3 -2
- package/src/daemon/handlers/config-slack-channel.ts +269 -94
- package/src/daemon/handlers/conversations.ts +13 -141
- package/src/daemon/handlers/shared.ts +80 -0
- package/src/daemon/handlers/skills.ts +483 -44
- package/src/daemon/host-bash-proxy.ts +48 -13
- package/src/daemon/host-browser-proxy.ts +192 -0
- package/src/daemon/host-cu-proxy.ts +36 -11
- package/src/daemon/host-file-proxy.ts +57 -9
- package/src/daemon/lifecycle.ts +179 -28
- package/src/daemon/message-protocol.ts +13 -0
- package/src/daemon/message-types/conversations.ts +89 -14
- package/src/daemon/message-types/home.ts +40 -0
- package/src/daemon/message-types/host-browser.ts +100 -0
- package/src/daemon/message-types/meet.ts +143 -0
- package/src/daemon/message-types/messages.ts +19 -5
- package/src/daemon/message-types/schedules.ts +34 -2
- package/src/daemon/message-types/skills.ts +26 -0
- package/src/daemon/message-types/subagents.ts +2 -0
- package/src/daemon/message-types/surfaces.ts +2 -0
- package/src/daemon/server.ts +439 -14
- package/src/daemon/shutdown-handlers.ts +32 -4
- package/src/daemon/shutdown-registry.ts +40 -0
- package/src/daemon/tool-side-effects.ts +15 -0
- package/src/daemon/transport-hints.ts +5 -24
- package/src/email/html-renderer.ts +76 -0
- package/src/heartbeat/heartbeat-service.ts +93 -7
- package/src/home/__tests__/assistant-feed-authoring.test.ts +156 -0
- package/src/home/__tests__/emit-feed-event.test.ts +169 -0
- package/src/home/__tests__/feed-scheduler.test.ts +194 -0
- package/src/home/__tests__/feed-types.test.ts +275 -0
- package/src/home/__tests__/feed-writer.test.ts +688 -0
- package/src/home/__tests__/phase5-exit-criteria.test.ts +212 -0
- package/src/home/__tests__/platform-gmail-digest.test.ts +222 -0
- package/src/home/__tests__/progress-formula.test.ts +213 -0
- package/src/home/__tests__/relationship-state-writer.test.ts +740 -0
- package/src/home/__tests__/rollup-producer.test.ts +398 -0
- package/src/home/assistant-feed-authoring.ts +124 -0
- package/src/home/emit-feed-event.ts +158 -0
- package/src/home/feed-scheduler.ts +247 -0
- package/src/home/feed-types.ts +181 -0
- package/src/home/feed-writer.ts +469 -0
- package/src/home/platform-gmail-digest.ts +163 -0
- package/src/home/progress-formula.ts +86 -0
- package/src/home/relationship-state-writer.ts +824 -0
- package/src/home/relationship-state.ts +143 -0
- package/src/home/rollup-producer.ts +384 -0
- package/src/hooks/runner.ts +7 -0
- package/src/inbound/platform-callback-registration.ts +30 -20
- package/src/inbound/public-ingress-urls.ts +12 -0
- package/src/instrument.ts +1 -1
- package/src/ipc/__tests__/cli-ipc.test.ts +200 -0
- package/src/ipc/cli-client.ts +151 -0
- package/src/ipc/cli-server.ts +234 -0
- package/src/ipc/gateway-client.ts +180 -0
- package/src/ipc/routes/index.ts +5 -0
- package/src/ipc/routes/wake-conversation.ts +19 -0
- package/src/mcp/client.ts +59 -24
- package/src/memory/__tests__/auto-analysis-enqueue.test.ts +356 -0
- package/src/memory/__tests__/auto-analysis-guard.test.ts +57 -0
- package/src/memory/__tests__/conversation-analyze-job.test.ts +232 -0
- package/src/memory/__tests__/find-analysis-conversation.test.ts +196 -0
- package/src/memory/app-store.ts +31 -1
- package/src/memory/attachments-store.ts +70 -0
- package/src/memory/auto-analysis-enqueue.ts +127 -0
- package/src/memory/auto-analysis-guard.ts +27 -0
- package/src/memory/cleanup-schedule-state.ts +37 -0
- package/src/memory/conversation-analyze-job.ts +73 -0
- package/src/memory/conversation-crud.ts +122 -0
- package/src/memory/conversation-disk-view.ts +7 -0
- package/src/memory/conversation-group-migration.ts +34 -2
- package/src/memory/conversation-queries.ts +6 -5
- package/src/memory/conversation-starters-cadence.ts +76 -0
- package/src/memory/conversation-title-service.ts +5 -2
- package/src/memory/db-init.ts +18 -0
- package/src/memory/db-maintenance.ts +108 -0
- package/src/memory/db.ts +1 -0
- package/src/memory/embedding-backend.test.ts +75 -0
- package/src/memory/embedding-backend.ts +131 -5
- package/src/memory/embedding-gemini.test.ts +54 -0
- package/src/memory/embedding-gemini.ts +20 -9
- package/src/memory/embedding-local.ts +176 -17
- package/src/memory/graph/consolidation.ts +10 -23
- package/src/memory/graph/conversation-graph-memory.ts +15 -0
- package/src/memory/graph/extraction-job.ts +15 -0
- package/src/memory/graph/extraction.test.ts +23 -0
- package/src/memory/graph/extraction.ts +8 -0
- package/src/memory/graph/retriever.ts +67 -40
- package/src/memory/graph/scoring.test.ts +186 -0
- package/src/memory/graph/scoring.ts +31 -1
- package/src/memory/graph/store.test.ts +7 -3
- package/src/memory/graph/store.ts +47 -12
- package/src/memory/graph/tools.ts +1 -1
- package/src/memory/group-crud.ts +6 -1
- package/src/memory/indexer.ts +95 -16
- package/src/memory/job-handlers/cleanup.ts +11 -8
- package/src/memory/job-handlers/conversation-starters.ts +16 -10
- package/src/memory/jobs-store.ts +64 -4
- package/src/memory/jobs-worker.ts +22 -9
- package/src/memory/llm-usage-store.ts +137 -60
- package/src/memory/migrations/213-oauth-providers-scope-separator.ts +13 -0
- package/src/memory/migrations/214-oauth-providers-refresh-url.ts +11 -0
- package/src/memory/migrations/215-oauth-providers-revoke.ts +14 -0
- package/src/memory/migrations/216-oauth-providers-token-auth-method.ts +30 -0
- package/src/memory/migrations/217-conversation-host-access.ts +40 -0
- package/src/memory/migrations/218-oauth-providers-logo-url.ts +11 -0
- package/src/memory/migrations/219-oauth-providers-token-exchange-body-format.ts +15 -0
- package/src/memory/migrations/220-normalize-user-file-by-principal.ts +190 -0
- package/src/memory/migrations/221-conversations-archived-at.ts +16 -0
- package/src/memory/migrations/index.ts +12 -0
- package/src/memory/migrations/registry.ts +16 -0
- package/src/memory/qdrant-manager.ts +43 -16
- package/src/memory/schema/conversations.ts +3 -0
- package/src/memory/schema/oauth.ts +21 -13
- package/src/memory/usage-buckets.ts +396 -0
- package/src/messaging/providers/gmail/client.ts +57 -6
- package/src/messaging/providers/slack/__tests__/adapter-token-routing.test.ts +282 -0
- package/src/messaging/providers/slack/adapter.ts +143 -38
- package/src/messaging/providers/slack/client.ts +16 -0
- package/src/messaging/providers/slack/types.ts +4 -0
- package/src/notifications/decision-engine.ts +3 -3
- package/src/notifications/signal.ts +5 -0
- package/src/oauth/AGENTS.md +76 -0
- package/src/oauth/__tests__/identity-verifier.test.ts +25 -19
- package/src/oauth/__tests__/seed-providers-managed.test.ts +32 -0
- package/src/oauth/byo-connection.test.ts +26 -9
- package/src/oauth/byo-connection.ts +10 -8
- package/src/oauth/connect-orchestrator.ts +25 -21
- package/src/oauth/connect-types.ts +3 -3
- package/src/oauth/connection-resolver.test.ts +17 -4
- package/src/oauth/connection-resolver.ts +22 -18
- package/src/oauth/connection.ts +3 -1
- package/src/oauth/manual-token-connection.ts +13 -13
- package/src/oauth/oauth-store.ts +223 -100
- package/src/oauth/platform-connection.test.ts +101 -3
- package/src/oauth/platform-connection.ts +56 -35
- package/src/oauth/provider-serializer.ts +31 -5
- package/src/oauth/revoke.ts +76 -0
- package/src/oauth/seed-providers.ts +133 -87
- package/src/oauth/token-persistence.ts +1 -1
- package/src/permissions/checker.ts +16 -6
- package/src/permissions/defaults.ts +49 -1
- package/src/permissions/permission-mode.ts +4 -11
- package/src/permissions/prompter.ts +13 -1
- package/src/permissions/trust-store.ts +3 -3
- package/src/permissions/v2-consent-policy.ts +87 -0
- package/src/permissions/workspace-policy.ts +3 -0
- package/src/platform/client.test.ts +10 -0
- package/src/platform/sync-identity.ts +129 -0
- package/src/prompts/persona-resolver.ts +126 -2
- package/src/prompts/system-prompt.ts +76 -38
- package/src/prompts/templates/BOOTSTRAP-REFERENCE.md +3 -65
- package/src/prompts/templates/BOOTSTRAP.md +59 -105
- package/src/prompts/templates/SOUL.md +3 -1
- package/src/prompts/templates/UPDATES.md +12 -0
- package/src/prompts/templates/channels/slack.md +20 -0
- package/src/prompts/update-bulletin-format.ts +26 -9
- package/src/prompts/update-bulletin.ts +34 -23
- package/src/prompts/user-reference.ts +20 -17
- package/src/providers/__tests__/provider-secret-catalog.test.ts +42 -0
- package/src/providers/anthropic/client.ts +157 -60
- package/src/providers/fireworks/client.ts +2 -2
- package/src/providers/gemini/client.ts +9 -1
- package/src/providers/model-catalog.ts +6 -0
- package/src/providers/model-intents.ts +4 -4
- package/src/providers/ollama/client.ts +2 -2
- package/src/providers/openai/chat-completions-provider.ts +474 -0
- package/src/providers/openai/client.ts +25 -440
- package/src/providers/openai/responses-provider.ts +502 -0
- package/src/providers/openrouter/client.ts +101 -4
- package/src/providers/provider-secret-catalog.ts +139 -0
- package/src/providers/registry.ts +2 -2
- package/src/providers/retry.ts +14 -3
- package/src/providers/speech-to-text/__tests__/provider-catalog.test.ts +251 -0
- package/src/providers/speech-to-text/__tests__/resolve.test.ts +828 -0
- package/src/providers/speech-to-text/deepgram-realtime.test.ts +980 -0
- package/src/providers/speech-to-text/deepgram-realtime.ts +767 -0
- package/src/providers/speech-to-text/deepgram.test.ts +332 -0
- package/src/providers/speech-to-text/deepgram.ts +115 -0
- package/src/providers/speech-to-text/google-gemini-live-stream.test.ts +743 -0
- package/src/providers/speech-to-text/google-gemini-live-stream.ts +625 -0
- package/src/providers/speech-to-text/google-gemini.test.ts +226 -0
- package/src/providers/speech-to-text/google-gemini.ts +101 -0
- package/src/providers/speech-to-text/openai-whisper-stream.test.ts +564 -0
- package/src/providers/speech-to-text/openai-whisper-stream.ts +381 -0
- package/src/providers/speech-to-text/openai-whisper.test.ts +1 -37
- package/src/providers/speech-to-text/openai-whisper.ts +63 -33
- package/src/providers/speech-to-text/provider-catalog.ts +306 -0
- package/src/providers/speech-to-text/resolve.ts +386 -6
- package/src/providers/types.ts +10 -1
- package/src/runtime/AGENTS.md +65 -0
- package/src/runtime/__tests__/agent-wake.test.ts +831 -0
- package/src/runtime/__tests__/browser-extension-pair-routes.test.ts +715 -0
- package/src/runtime/__tests__/capability-tokens.test.ts +258 -0
- package/src/runtime/__tests__/chrome-extension-registry.test.ts +518 -0
- package/src/runtime/__tests__/runtime-mode.test.ts +62 -0
- package/src/runtime/__tests__/slack-block-formatting.test.ts +481 -0
- package/src/runtime/agent-wake.ts +512 -0
- package/src/runtime/assistant-event-hub.ts +2 -2
- package/src/runtime/auth/__tests__/guard-tests.test.ts +1 -0
- package/src/runtime/auth/__tests__/middleware.test.ts +116 -1
- package/src/runtime/auth/__tests__/route-policy.test.ts +48 -0
- package/src/runtime/auth/middleware.ts +98 -0
- package/src/runtime/auth/route-policy.ts +33 -9
- package/src/runtime/auth/token-service.ts +56 -1
- package/src/runtime/btw-sidechain.ts +2 -0
- package/src/runtime/capability-tokens.ts +414 -0
- package/src/runtime/channel-approvals.ts +18 -5
- package/src/runtime/channel-invite-transport.ts +1 -1
- package/src/runtime/channel-invite-transports/email.ts +14 -6
- package/src/runtime/channel-readiness-service.ts +12 -22
- package/src/runtime/chrome-extension-registry.ts +368 -0
- package/src/runtime/confirmation-request-guardian-bridge.ts +6 -0
- package/src/runtime/guardian-decision-types.ts +7 -0
- package/src/runtime/http-server.ts +815 -75
- package/src/runtime/http-types.ts +6 -2
- package/src/runtime/migrations/__tests__/rebind-secrets-credentials.test.ts +172 -0
- package/src/runtime/migrations/__tests__/vbundle-builder-credentials.test.ts +276 -0
- package/src/runtime/migrations/__tests__/vbundle-import-credentials.test.ts +198 -0
- package/src/runtime/migrations/__tests__/vbundle-legacy-user-md.test.ts +360 -0
- package/src/runtime/migrations/migration-transport.ts +7 -0
- package/src/runtime/migrations/migration-wizard.ts +23 -2
- package/src/runtime/migrations/rebind-secrets-screen.ts +76 -15
- package/src/runtime/migrations/vbundle-builder.ts +145 -38
- package/src/runtime/migrations/vbundle-import-analyzer.ts +96 -1
- package/src/runtime/migrations/vbundle-importer.ts +89 -5
- package/src/runtime/pending-interactions.ts +18 -13
- package/src/runtime/routes/__tests__/backup-routes.test.ts +967 -0
- package/src/runtime/routes/__tests__/home-feed-routes.test.ts +507 -0
- package/src/runtime/routes/__tests__/migration-import-credential-filter.test.ts +208 -0
- package/src/runtime/routes/__tests__/stt-routes.test.ts +406 -0
- package/src/runtime/routes/__tests__/tts-routes.test.ts +474 -0
- package/src/runtime/routes/__tests__/user-route-dispatcher.test.ts +148 -17
- package/src/runtime/routes/app-management-routes.ts +12 -18
- package/src/runtime/routes/approval-routes.ts +90 -16
- package/src/runtime/routes/attachment-routes.test.ts +9 -3
- package/src/runtime/routes/attachment-routes.ts +216 -17
- package/src/runtime/routes/backup-routes.ts +519 -0
- package/src/runtime/routes/browser-extension-pair-routes.ts +556 -0
- package/src/runtime/routes/btw-routes.ts +8 -6
- package/src/runtime/routes/contact-routes.test.ts +298 -0
- package/src/runtime/routes/contact-routes.ts +132 -5
- package/src/runtime/routes/conversation-analysis-routes.ts +22 -141
- package/src/runtime/routes/conversation-management-routes.ts +223 -0
- package/src/runtime/routes/conversation-routes.ts +598 -103
- package/src/runtime/routes/conversation-starter-routes.ts +78 -16
- package/src/runtime/routes/filing-routes.ts +93 -0
- package/src/runtime/routes/guardian-action-routes.ts +24 -13
- package/src/runtime/routes/home-feed-routes.ts +334 -0
- package/src/runtime/routes/home-state-routes.ts +138 -0
- package/src/runtime/routes/host-browser-routes.ts +268 -0
- package/src/runtime/routes/host-file-routes.ts +9 -1
- package/src/runtime/routes/identity-intro-cache.ts +7 -3
- package/src/runtime/routes/identity-routes.ts +262 -33
- package/src/runtime/routes/inbound-stages/transcribe-audio.test.ts +46 -39
- package/src/runtime/routes/inbound-stages/transcribe-audio.ts +15 -15
- package/src/runtime/routes/integrations/slack/__tests__/channel.test.ts +137 -0
- package/src/runtime/routes/integrations/slack/__tests__/share.test.ts +179 -0
- package/src/runtime/routes/integrations/slack/channel.ts +11 -3
- package/src/runtime/routes/integrations/slack/share.ts +45 -7
- package/src/runtime/routes/llm-context-normalization.ts +303 -0
- package/src/runtime/routes/log-export-routes.ts +42 -22
- package/src/runtime/routes/memory-item-routes.test.ts +3 -2
- package/src/runtime/routes/memory-item-routes.ts +1 -7
- package/src/runtime/routes/migration-routes.ts +122 -2
- package/src/runtime/routes/oauth-apps.ts +15 -17
- package/src/runtime/routes/oauth-providers.ts +4 -0
- package/src/runtime/routes/schedule-routes.ts +24 -11
- package/src/runtime/routes/settings-routes.ts +31 -102
- package/src/runtime/routes/skills-routes.ts +128 -9
- package/src/runtime/routes/stt-routes.ts +233 -0
- package/src/runtime/routes/subagents-routes.ts +14 -10
- package/src/runtime/routes/surface-action-routes.ts +41 -2
- package/src/runtime/routes/tts-routes.ts +108 -24
- package/src/runtime/routes/usage-routes.ts +38 -9
- package/src/runtime/routes/user-route-dispatcher.ts +50 -5
- package/src/runtime/routes/user-routes.ts +13 -1
- package/src/runtime/routes/work-items-routes.ts +8 -1
- package/src/runtime/routes/workspace-routes.test.ts +22 -0
- package/src/runtime/routes/workspace-routes.ts +8 -1
- package/src/runtime/routes/workspace-utils.ts +2 -0
- package/src/runtime/runtime-mode.ts +33 -0
- package/src/runtime/services/__tests__/analyze-conversation.test.ts +444 -0
- package/src/runtime/services/__tests__/analyze-deps-singleton.test.ts +67 -0
- package/src/runtime/services/__tests__/auto-analysis-prompt.test.ts +53 -0
- package/src/runtime/services/__tests__/manual-analysis-prompt.test.ts +41 -0
- package/src/runtime/services/analyze-conversation.ts +344 -0
- package/src/runtime/services/analyze-deps-singleton.ts +32 -0
- package/src/runtime/services/auto-analysis-prompt.ts +55 -0
- package/src/runtime/skill-route-registry.ts +49 -0
- package/src/runtime/slack-block-formatting.ts +437 -10
- package/src/schedule/scheduler.ts +57 -5
- package/src/security/ces-credential-client.ts +20 -0
- package/src/security/ces-rpc-credential-backend.ts +17 -0
- package/src/security/credential-backend.ts +5 -0
- package/src/security/oauth2.ts +68 -29
- package/src/security/secure-keys.ts +143 -27
- package/src/security/token-manager.ts +31 -10
- package/src/sequence/engine.ts +23 -0
- package/src/sequence/types.ts +1 -1
- package/src/skills/catalog-files.ts +554 -0
- package/src/skills/category-inference.ts +122 -0
- package/src/skills/clawhub-files.ts +213 -0
- package/src/skills/clawhub.ts +84 -23
- package/src/skills/skill-file-provider.ts +40 -0
- package/src/skills/skillssh-files.ts +395 -0
- package/src/skills/skillssh-registry.ts +4 -4
- package/src/stt/__tests__/daemon-batch-transcriber.test.ts +392 -0
- package/src/stt/__tests__/types.test.ts +89 -0
- package/src/stt/daemon-batch-transcriber.ts +195 -0
- package/src/stt/stt-stream-session.ts +499 -0
- package/src/stt/types.ts +330 -0
- package/src/stt/wav-encoder.test.ts +373 -0
- package/src/stt/wav-encoder.ts +175 -0
- package/src/subagent/manager.ts +169 -40
- package/src/subagent/types.ts +19 -0
- package/src/tools/apps/executors.ts +11 -2
- package/src/tools/browser/__tests__/auth-detector.test.ts +202 -108
- package/src/tools/browser/__tests__/browser-mode.test.ts +119 -0
- package/src/tools/browser/__tests__/browser-status.test.ts +123 -0
- package/src/tools/browser/auth-detector.ts +43 -12
- package/src/tools/browser/browser-execution.ts +1787 -342
- package/src/tools/browser/browser-manager.ts +81 -12
- package/src/tools/browser/browser-mode-constants.ts +12 -0
- package/src/tools/browser/browser-mode.ts +92 -0
- package/src/tools/browser/browser-status-constants.ts +33 -0
- package/src/tools/browser/cdp-client/__tests__/accessibility-snapshot.test.ts +318 -0
- package/src/tools/browser/cdp-client/__tests__/cdp-dom-helpers.test.ts +1175 -0
- package/src/tools/browser/cdp-client/__tests__/cdp-inspect-client.test.ts +1263 -0
- package/src/tools/browser/cdp-client/__tests__/extension-cdp-client.test.ts +359 -0
- package/src/tools/browser/cdp-client/__tests__/factory.test.ts +1993 -0
- package/src/tools/browser/cdp-client/__tests__/fixtures/ax-tree-nested-frames.json +64 -0
- package/src/tools/browser/cdp-client/__tests__/fixtures/ax-tree-simple.json +69 -0
- package/src/tools/browser/cdp-client/__tests__/local-cdp-client.test.ts +310 -0
- package/src/tools/browser/cdp-client/__tests__/types.test.ts +96 -0
- package/src/tools/browser/cdp-client/accessibility-snapshot.ts +387 -0
- package/src/tools/browser/cdp-client/cdp-dom-helpers.ts +695 -0
- package/src/tools/browser/cdp-client/cdp-inspect/__tests__/discovery.test.ts +1007 -0
- package/src/tools/browser/cdp-client/cdp-inspect/__tests__/ws-transport.test.ts +580 -0
- package/src/tools/browser/cdp-client/cdp-inspect/discovery.ts +744 -0
- package/src/tools/browser/cdp-client/cdp-inspect/ws-transport.ts +579 -0
- package/src/tools/browser/cdp-client/cdp-inspect-client.ts +868 -0
- package/src/tools/browser/cdp-client/errors.ts +49 -0
- package/src/tools/browser/cdp-client/extension-cdp-client.ts +148 -0
- package/src/tools/browser/cdp-client/factory.ts +914 -0
- package/src/tools/browser/cdp-client/index.ts +28 -0
- package/src/tools/browser/cdp-client/local-cdp-client.ts +187 -0
- package/src/tools/browser/cdp-client/types.ts +120 -0
- package/src/tools/credentials/vault.ts +35 -6
- package/src/tools/filesystem/edit.ts +1 -1
- package/src/tools/filesystem/list.ts +1 -1
- package/src/tools/filesystem/read.ts +1 -1
- package/src/tools/filesystem/write.ts +2 -1
- package/src/tools/host-filesystem/edit.ts +1 -1
- package/src/tools/host-filesystem/read.ts +12 -15
- package/src/tools/host-filesystem/write.ts +1 -1
- package/src/tools/host-terminal/host-shell.ts +21 -16
- package/src/tools/network/web-fetch.ts +5 -2
- package/src/tools/network/web-search.ts +5 -2
- package/src/tools/permission-checker.ts +77 -82
- package/src/tools/registry.ts +0 -2
- package/src/tools/secret-detection-handler.ts +34 -0
- package/src/tools/shared/filesystem/image-read.ts +61 -40
- package/src/tools/shared/shell-output.ts +3 -1
- package/src/tools/side-effects.ts +2 -0
- package/src/tools/skills/sandbox-runner.ts +3 -2
- package/src/tools/subagent/spawn.ts +47 -3
- package/src/tools/subagent/status.ts +2 -0
- package/src/tools/system/register.ts +2 -16
- package/src/tools/terminal/safe-env.ts +15 -0
- package/src/tools/terminal/shell.ts +36 -20
- package/src/tools/tool-approval-handler.ts +48 -2
- package/src/tools/tool-manifest.ts +21 -0
- package/src/tools/types.ts +19 -0
- package/src/tools/ui-surface/definitions.ts +6 -1
- package/src/tts/__tests__/provider-adapters.test.ts +834 -0
- package/src/tts/__tests__/provider-catalog-consistency.test.ts +196 -0
- package/src/tts/__tests__/provider-catalog.test.ts +183 -0
- package/src/tts/__tests__/provider-registry.test.ts +90 -0
- package/src/tts/provider-catalog.ts +201 -0
- package/src/tts/provider-registry.ts +73 -0
- package/src/tts/providers/deepgram-provider.ts +219 -0
- package/src/tts/providers/elevenlabs-provider.ts +211 -0
- package/src/tts/providers/fish-audio-provider.ts +183 -0
- package/src/tts/providers/index.ts +42 -0
- package/src/tts/providers/register-builtins.ts +130 -0
- package/src/tts/synthesize-text.ts +110 -0
- package/src/tts/tts-config-resolver.ts +78 -0
- package/src/tts/types.ts +153 -0
- package/src/types/onboarding-context.ts +7 -0
- package/src/util/abort-reasons.ts +58 -0
- package/src/util/device-id.ts +32 -16
- package/src/util/errors.ts +9 -1
- package/src/util/platform.ts +63 -24
- package/src/util/pricing.ts +66 -3
- package/src/util/spawn.ts +1 -1
- package/src/util/truncate.ts +4 -2
- package/src/util/unicode.ts +201 -0
- package/src/version.ts +19 -24
- package/src/watcher/engine.ts +23 -0
- package/src/watcher/watcher-store.ts +31 -0
- package/src/workspace/migrations/003-seed-device-id.ts +9 -3
- package/src/workspace/migrations/017-seed-persona-dirs.ts +68 -4
- package/src/workspace/migrations/029-seed-pkb.ts +1 -1
- package/src/workspace/migrations/031-drop-user-md.ts +317 -0
- package/src/workspace/migrations/031-llm-log-retention-zero-to-null.ts +73 -0
- package/src/workspace/migrations/032-tts-provider-unification.ts +227 -0
- package/src/workspace/migrations/033-stt-service-explicit-config.ts +122 -0
- package/src/workspace/migrations/034-remove-calls-voice-transcription-provider.ts +215 -0
- package/src/workspace/migrations/035-seed-slack-channel-persona.ts +50 -0
- package/src/workspace/migrations/036-update-pkb-index-bar.ts +37 -0
- package/src/workspace/migrations/037-create-meets-dir.ts +61 -0
- package/src/workspace/migrations/registry.ts +16 -0
- package/src/workspace/top-level-renderer.ts +31 -1
- package/src/workspace/turn-commit.ts +31 -0
- package/src/__tests__/chrome-cdp.test.ts +0 -419
- package/src/__tests__/email-cli.test.ts +0 -297
- package/src/__tests__/email-service-config-fallback.test.ts +0 -102
- package/src/__tests__/permission-mode-sse.test.ts +0 -418
- package/src/__tests__/permission-mode-store.test.ts +0 -277
- package/src/browser-extension-relay/protocol.ts +0 -63
- package/src/browser-extension-relay/server.ts +0 -203
- package/src/cli/commands/browser-relay.ts +0 -536
- package/src/config/schemas/sandbox.ts +0 -14
- package/src/email/guardrails.ts +0 -221
- package/src/email/provider.ts +0 -117
- package/src/email/providers/agentmail.ts +0 -361
- package/src/email/providers/index.ts +0 -65
- package/src/email/service.ts +0 -384
- package/src/email/types.ts +0 -126
- package/src/permissions/permission-mode-store.ts +0 -180
- package/src/prompts/templates/USER.md +0 -13
- package/src/providers/speech-to-text/types.ts +0 -17
- package/src/tools/browser/chrome-cdp.ts +0 -239
- package/src/tools/system/set-permission-mode.ts +0 -103
|
@@ -1,17 +1,10 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
|
-
import {
|
|
3
|
-
access,
|
|
4
|
-
mkdir,
|
|
5
|
-
readdir,
|
|
6
|
-
readFile,
|
|
7
|
-
unlink,
|
|
8
|
-
writeFile,
|
|
9
|
-
} from "node:fs/promises";
|
|
2
|
+
import { access, mkdir, readdir, readFile, unlink } from "node:fs/promises";
|
|
10
3
|
import { tmpdir } from "node:os";
|
|
11
4
|
import { extname, join } from "node:path";
|
|
12
5
|
|
|
13
|
-
import {
|
|
14
|
-
import {
|
|
6
|
+
import { resolveBatchTranscriber } from "../../../../providers/speech-to-text/resolve.js";
|
|
7
|
+
import type { BatchTranscriber } from "../../../../stt/types.js";
|
|
15
8
|
import type {
|
|
16
9
|
ToolContext,
|
|
17
10
|
ToolExecutionResult,
|
|
@@ -44,17 +37,14 @@ const AUDIO_EXTENSIONS = new Set([
|
|
|
44
37
|
".wma",
|
|
45
38
|
]);
|
|
46
39
|
|
|
47
|
-
/** Max file size for a single
|
|
48
|
-
const
|
|
49
|
-
|
|
50
|
-
/** Duration per chunk when splitting for the API (10 minutes - stays well under 25MB as WAV). */
|
|
51
|
-
const API_CHUNK_DURATION_SECS = 600;
|
|
40
|
+
/** Max file size for a single STT chunk request (25MB). */
|
|
41
|
+
const STT_CHUNK_MAX_BYTES = 25 * 1024 * 1024;
|
|
52
42
|
|
|
53
|
-
/**
|
|
54
|
-
const
|
|
43
|
+
/** Duration per chunk when splitting for large files (10 minutes - stays well under 25MB as WAV). */
|
|
44
|
+
const CHUNK_DURATION_SECS = 600;
|
|
55
45
|
|
|
56
|
-
/** Timeout for a single
|
|
57
|
-
const
|
|
46
|
+
/** Timeout for a single STT transcription request. */
|
|
47
|
+
const STT_REQUEST_TIMEOUT_MS = 300_000;
|
|
58
48
|
|
|
59
49
|
// ---------------------------------------------------------------------------
|
|
60
50
|
// Helpers
|
|
@@ -161,34 +151,30 @@ async function toWav(inputPath: string, isVideo: boolean): Promise<string> {
|
|
|
161
151
|
}
|
|
162
152
|
|
|
163
153
|
// ---------------------------------------------------------------------------
|
|
164
|
-
//
|
|
154
|
+
// Transcription via resolved STT provider
|
|
165
155
|
// ---------------------------------------------------------------------------
|
|
166
156
|
|
|
167
|
-
async function
|
|
157
|
+
async function transcribeWithProvider(
|
|
168
158
|
audioPath: string,
|
|
169
|
-
|
|
159
|
+
transcriber: BatchTranscriber,
|
|
170
160
|
context: ToolContext,
|
|
171
161
|
): Promise<string> {
|
|
172
|
-
const provider = new OpenAIWhisperProvider(apiKey);
|
|
173
162
|
const duration = await getAudioDuration(audioPath);
|
|
174
163
|
const fileSize = Bun.file(audioPath).size;
|
|
175
164
|
|
|
176
165
|
// If small enough, send directly
|
|
177
|
-
if (fileSize <=
|
|
166
|
+
if (fileSize <= STT_CHUNK_MAX_BYTES) {
|
|
178
167
|
const audioBuffer = await readFile(audioPath);
|
|
179
|
-
const result = await
|
|
180
|
-
audioBuffer,
|
|
181
|
-
"audio/wav",
|
|
182
|
-
AbortSignal.timeout(
|
|
183
|
-
);
|
|
168
|
+
const result = await transcriber.transcribe({
|
|
169
|
+
audio: audioBuffer,
|
|
170
|
+
mimeType: "audio/wav",
|
|
171
|
+
signal: AbortSignal.timeout(STT_REQUEST_TIMEOUT_MS),
|
|
172
|
+
});
|
|
184
173
|
return result.text;
|
|
185
174
|
}
|
|
186
175
|
|
|
187
176
|
// Split into chunks for large files
|
|
188
|
-
const chunkDir = join(
|
|
189
|
-
tmpdir(),
|
|
190
|
-
`vellum-transcribe-api-chunks-${randomUUID()}`,
|
|
191
|
-
);
|
|
177
|
+
const chunkDir = join(tmpdir(), `vellum-transcribe-chunks-${randomUUID()}`);
|
|
192
178
|
await mkdir(chunkDir, { recursive: true });
|
|
193
179
|
|
|
194
180
|
try {
|
|
@@ -197,22 +183,18 @@ async function transcribeViaApi(
|
|
|
197
183
|
duration / 60,
|
|
198
184
|
)}min) - splitting into chunks...\n`,
|
|
199
185
|
);
|
|
200
|
-
const chunks = await splitAudio(
|
|
201
|
-
audioPath,
|
|
202
|
-
chunkDir,
|
|
203
|
-
API_CHUNK_DURATION_SECS,
|
|
204
|
-
);
|
|
186
|
+
const chunks = await splitAudio(audioPath, chunkDir, CHUNK_DURATION_SECS);
|
|
205
187
|
const parts: string[] = [];
|
|
206
188
|
|
|
207
189
|
for (let i = 0; i < chunks.length; i++) {
|
|
208
190
|
if (context.signal?.aborted) throw new Error("Cancelled");
|
|
209
191
|
context.onOutput?.(` Transcribing chunk ${i + 1}/${chunks.length}...\n`);
|
|
210
192
|
const audioBuffer = await readFile(chunks[i]);
|
|
211
|
-
const result = await
|
|
212
|
-
audioBuffer,
|
|
213
|
-
"audio/wav",
|
|
214
|
-
AbortSignal.timeout(
|
|
215
|
-
);
|
|
193
|
+
const result = await transcriber.transcribe({
|
|
194
|
+
audio: audioBuffer,
|
|
195
|
+
mimeType: "audio/wav",
|
|
196
|
+
signal: AbortSignal.timeout(STT_REQUEST_TIMEOUT_MS),
|
|
197
|
+
});
|
|
216
198
|
if (result.text) parts.push(result.text);
|
|
217
199
|
}
|
|
218
200
|
|
|
@@ -226,131 +208,6 @@ async function transcribeViaApi(
|
|
|
226
208
|
}
|
|
227
209
|
}
|
|
228
210
|
|
|
229
|
-
// ---------------------------------------------------------------------------
|
|
230
|
-
// Local mode - whisper.cpp
|
|
231
|
-
// ---------------------------------------------------------------------------
|
|
232
|
-
|
|
233
|
-
async function transcribeViaLocal(
|
|
234
|
-
audioPath: string,
|
|
235
|
-
context: ToolContext,
|
|
236
|
-
): Promise<string> {
|
|
237
|
-
// Check if whisper-cpp is installed
|
|
238
|
-
const whichResult = await spawnWithTimeout(["which", "whisper-cpp"], 5_000);
|
|
239
|
-
if (whichResult.exitCode !== 0) {
|
|
240
|
-
throw new Error(
|
|
241
|
-
"whisper-cpp is not installed. Install it with: brew install whisper-cpp",
|
|
242
|
-
);
|
|
243
|
-
}
|
|
244
|
-
|
|
245
|
-
// Resolve model path - use the base model, download if needed
|
|
246
|
-
const modelPath = await resolveWhisperModel(context);
|
|
247
|
-
|
|
248
|
-
const duration = await getAudioDuration(audioPath);
|
|
249
|
-
|
|
250
|
-
if (duration > 0 && duration <= 1800) {
|
|
251
|
-
// Under 30 minutes - transcribe directly (whisper.cpp handles long files well)
|
|
252
|
-
context.onOutput?.(
|
|
253
|
-
`Transcribing ${Math.round(duration / 60)}min of audio locally...\n`,
|
|
254
|
-
);
|
|
255
|
-
return await whisperCppRun(audioPath, modelPath);
|
|
256
|
-
}
|
|
257
|
-
|
|
258
|
-
// Very long files - split into 10-minute chunks to show progress
|
|
259
|
-
const chunkDir = join(
|
|
260
|
-
tmpdir(),
|
|
261
|
-
`vellum-transcribe-local-chunks-${randomUUID()}`,
|
|
262
|
-
);
|
|
263
|
-
await mkdir(chunkDir, { recursive: true });
|
|
264
|
-
|
|
265
|
-
try {
|
|
266
|
-
context.onOutput?.(
|
|
267
|
-
`Large file (${Math.round(
|
|
268
|
-
duration / 60,
|
|
269
|
-
)}min) - splitting into chunks...\n`,
|
|
270
|
-
);
|
|
271
|
-
const chunks = await splitAudio(audioPath, chunkDir, 600);
|
|
272
|
-
const parts: string[] = [];
|
|
273
|
-
|
|
274
|
-
for (let i = 0; i < chunks.length; i++) {
|
|
275
|
-
if (context.signal?.aborted) throw new Error("Cancelled");
|
|
276
|
-
context.onOutput?.(` Transcribing chunk ${i + 1}/${chunks.length}...\n`);
|
|
277
|
-
const text = await whisperCppRun(chunks[i], modelPath);
|
|
278
|
-
if (text) parts.push(text);
|
|
279
|
-
}
|
|
280
|
-
|
|
281
|
-
return parts.join(" ");
|
|
282
|
-
} finally {
|
|
283
|
-
const { rm } = await import("node:fs/promises");
|
|
284
|
-
await silentlyWithLog(
|
|
285
|
-
rm(chunkDir, { recursive: true, force: true }),
|
|
286
|
-
"transcribe chunk cleanup",
|
|
287
|
-
);
|
|
288
|
-
}
|
|
289
|
-
}
|
|
290
|
-
|
|
291
|
-
async function resolveWhisperModel(context: ToolContext): Promise<string> {
|
|
292
|
-
// Check common locations for the base model
|
|
293
|
-
const homeDir = process.env.HOME ?? "/tmp";
|
|
294
|
-
const candidates = [
|
|
295
|
-
join(homeDir, ".vellum", "models", "ggml-base.en.bin"),
|
|
296
|
-
join(homeDir, ".vellum", "models", "ggml-base.bin"),
|
|
297
|
-
"/usr/local/share/whisper-cpp/models/ggml-base.en.bin",
|
|
298
|
-
"/opt/homebrew/share/whisper-cpp/models/ggml-base.en.bin",
|
|
299
|
-
];
|
|
300
|
-
|
|
301
|
-
for (const p of candidates) {
|
|
302
|
-
try {
|
|
303
|
-
await access(p);
|
|
304
|
-
return p;
|
|
305
|
-
} catch {
|
|
306
|
-
/* next */
|
|
307
|
-
}
|
|
308
|
-
}
|
|
309
|
-
|
|
310
|
-
// Download the base.en model (~140MB)
|
|
311
|
-
const modelDir = join(homeDir, ".vellum", "models");
|
|
312
|
-
await mkdir(modelDir, { recursive: true });
|
|
313
|
-
const modelPath = join(modelDir, "ggml-base.en.bin");
|
|
314
|
-
const modelUrl =
|
|
315
|
-
"https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin";
|
|
316
|
-
|
|
317
|
-
context.onOutput?.("Downloading Whisper base.en model (~140MB)...\n");
|
|
318
|
-
|
|
319
|
-
const response = await fetch(modelUrl);
|
|
320
|
-
if (!response.ok) {
|
|
321
|
-
throw new Error(`Failed to download model: ${response.status}`);
|
|
322
|
-
}
|
|
323
|
-
|
|
324
|
-
const data = Buffer.from(await response.arrayBuffer());
|
|
325
|
-
await writeFile(modelPath, data);
|
|
326
|
-
context.onOutput?.("Model downloaded.\n");
|
|
327
|
-
|
|
328
|
-
return modelPath;
|
|
329
|
-
}
|
|
330
|
-
|
|
331
|
-
async function whisperCppRun(
|
|
332
|
-
audioPath: string,
|
|
333
|
-
modelPath: string,
|
|
334
|
-
): Promise<string> {
|
|
335
|
-
const result = await spawnWithTimeout(
|
|
336
|
-
["whisper-cpp", "-m", modelPath, "-f", audioPath, "--no-timestamps"],
|
|
337
|
-
LOCAL_CHUNK_TIMEOUT_MS,
|
|
338
|
-
);
|
|
339
|
-
|
|
340
|
-
if (result.exitCode !== 0) {
|
|
341
|
-
throw new Error(`whisper-cpp failed: ${result.stderr.slice(0, 300)}`);
|
|
342
|
-
}
|
|
343
|
-
|
|
344
|
-
// whisper-cpp outputs transcription to stderr with some logging, and
|
|
345
|
-
// the actual text lines to stdout. Clean up whitespace.
|
|
346
|
-
return result.stdout
|
|
347
|
-
.split("\n")
|
|
348
|
-
.map((l) => l.trim())
|
|
349
|
-
.filter((l) => l.length > 0)
|
|
350
|
-
.join(" ")
|
|
351
|
-
.trim();
|
|
352
|
-
}
|
|
353
|
-
|
|
354
211
|
// ---------------------------------------------------------------------------
|
|
355
212
|
// Main entry point
|
|
356
213
|
// ---------------------------------------------------------------------------
|
|
@@ -359,26 +216,24 @@ export async function run(
|
|
|
359
216
|
input: Record<string, unknown>,
|
|
360
217
|
context: ToolContext,
|
|
361
218
|
): Promise<ToolExecutionResult> {
|
|
362
|
-
|
|
363
|
-
|
|
219
|
+
// Reject legacy callers that pass the now-removed `mode` parameter to avoid
|
|
220
|
+
// silently routing audio through a different provider than expected.
|
|
221
|
+
if ("mode" in input) {
|
|
364
222
|
return {
|
|
365
223
|
content:
|
|
366
|
-
"
|
|
224
|
+
"The `mode` parameter is no longer supported. Transcription now uses the configured speech-to-text service.",
|
|
367
225
|
isError: true,
|
|
368
226
|
};
|
|
369
227
|
}
|
|
370
228
|
|
|
371
|
-
//
|
|
372
|
-
|
|
373
|
-
if (
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
isError: true,
|
|
380
|
-
};
|
|
381
|
-
}
|
|
229
|
+
// Resolve the configured STT provider
|
|
230
|
+
const transcriber = await resolveBatchTranscriber();
|
|
231
|
+
if (!transcriber) {
|
|
232
|
+
return {
|
|
233
|
+
content:
|
|
234
|
+
"No speech-to-text provider is configured. Set up an STT provider (e.g. OpenAI Whisper or Deepgram) in your assistant settings to enable transcription.",
|
|
235
|
+
isError: true,
|
|
236
|
+
};
|
|
382
237
|
}
|
|
383
238
|
|
|
384
239
|
const source = await resolveSource(input);
|
|
@@ -391,12 +246,7 @@ export async function run(
|
|
|
391
246
|
// Convert to WAV
|
|
392
247
|
wavPath = await toWav(inputPath, isVideo);
|
|
393
248
|
|
|
394
|
-
|
|
395
|
-
if (mode === "api") {
|
|
396
|
-
text = await transcribeViaApi(wavPath, openaiKey!, context);
|
|
397
|
-
} else {
|
|
398
|
-
text = await transcribeViaLocal(wavPath, context);
|
|
399
|
-
}
|
|
249
|
+
const text = await transcribeWithProvider(wavPath, transcriber, context);
|
|
400
250
|
|
|
401
251
|
if (!text.trim()) {
|
|
402
252
|
return { content: "No speech detected in the audio.", isError: false };
|
|
@@ -23,8 +23,10 @@ import * as appDelete from "./bundled-skills/app-builder/tools/app-delete.js";
|
|
|
23
23
|
import * as appGenerateIcon from "./bundled-skills/app-builder/tools/app-generate-icon.js";
|
|
24
24
|
import * as appRefresh from "./bundled-skills/app-builder/tools/app-refresh.js";
|
|
25
25
|
// ── browser ────────────────────────────────────────────────────────────────────
|
|
26
|
+
import * as browserAttach from "./bundled-skills/browser/tools/browser-attach.js";
|
|
26
27
|
import * as browserClick from "./bundled-skills/browser/tools/browser-click.js";
|
|
27
28
|
import * as browserClose from "./bundled-skills/browser/tools/browser-close.js";
|
|
29
|
+
import * as browserDetach from "./bundled-skills/browser/tools/browser-detach.js";
|
|
28
30
|
import * as browserExtract from "./bundled-skills/browser/tools/browser-extract.js";
|
|
29
31
|
import * as browserFillCredential from "./bundled-skills/browser/tools/browser-fill-credential.js";
|
|
30
32
|
import * as browserHover from "./bundled-skills/browser/tools/browser-hover.js";
|
|
@@ -34,6 +36,7 @@ import * as browserScreenshot from "./bundled-skills/browser/tools/browser-scree
|
|
|
34
36
|
import * as browserScroll from "./bundled-skills/browser/tools/browser-scroll.js";
|
|
35
37
|
import * as browserSelectOption from "./bundled-skills/browser/tools/browser-select-option.js";
|
|
36
38
|
import * as browserSnapshot from "./bundled-skills/browser/tools/browser-snapshot.js";
|
|
39
|
+
import * as browserStatus from "./bundled-skills/browser/tools/browser-status.js";
|
|
37
40
|
import * as browserType from "./bundled-skills/browser/tools/browser-type.js";
|
|
38
41
|
import * as browserWaitFor from "./bundled-skills/browser/tools/browser-wait-for.js";
|
|
39
42
|
import * as browserWaitForDownload from "./bundled-skills/browser/tools/browser-wait-for-download.js";
|
|
@@ -74,6 +77,7 @@ import * as gmailFollowUp from "./bundled-skills/gmail/tools/gmail-follow-up.js"
|
|
|
74
77
|
import * as gmailForward from "./bundled-skills/gmail/tools/gmail-forward.js";
|
|
75
78
|
import * as gmailLabel from "./bundled-skills/gmail/tools/gmail-label.js";
|
|
76
79
|
import * as gmailOutreachScan from "./bundled-skills/gmail/tools/gmail-outreach-scan.js";
|
|
80
|
+
import * as gmailPreferencesTool from "./bundled-skills/gmail/tools/gmail-preferences-tool.js";
|
|
77
81
|
import * as gmailSendDraft from "./bundled-skills/gmail/tools/gmail-send-draft.js";
|
|
78
82
|
import * as gmailSenderDigest from "./bundled-skills/gmail/tools/gmail-sender-digest.js";
|
|
79
83
|
import * as gmailTrash from "./bundled-skills/gmail/tools/gmail-trash.js";
|
|
@@ -207,6 +211,8 @@ export const bundledToolRegistry = new Map<string, SkillToolScript>([
|
|
|
207
211
|
["browser:tools/browser-snapshot.ts", browserSnapshot],
|
|
208
212
|
["browser:tools/browser-screenshot.ts", browserScreenshot],
|
|
209
213
|
["browser:tools/browser-close.ts", browserClose],
|
|
214
|
+
["browser:tools/browser-attach.ts", browserAttach],
|
|
215
|
+
["browser:tools/browser-detach.ts", browserDetach],
|
|
210
216
|
["browser:tools/browser-click.ts", browserClick],
|
|
211
217
|
["browser:tools/browser-type.ts", browserType],
|
|
212
218
|
["browser:tools/browser-press-key.ts", browserPressKey],
|
|
@@ -217,6 +223,7 @@ export const bundledToolRegistry = new Map<string, SkillToolScript>([
|
|
|
217
223
|
["browser:tools/browser-extract.ts", browserExtract],
|
|
218
224
|
["browser:tools/browser-wait-for-download.ts", browserWaitForDownload],
|
|
219
225
|
["browser:tools/browser-fill-credential.ts", browserFillCredential],
|
|
226
|
+
["browser:tools/browser-status.ts", browserStatus],
|
|
220
227
|
|
|
221
228
|
// chatgpt-import
|
|
222
229
|
["chatgpt-import:tools/chatgpt-import.ts", chatgptImport],
|
|
@@ -269,6 +276,7 @@ export const bundledToolRegistry = new Map<string, SkillToolScript>([
|
|
|
269
276
|
["gmail:tools/gmail-vacation.ts", gmailVacation],
|
|
270
277
|
["gmail:tools/gmail-sender-digest.ts", gmailSenderDigest],
|
|
271
278
|
["gmail:tools/gmail-outreach-scan.ts", gmailOutreachScan],
|
|
279
|
+
["gmail:tools/gmail-preferences-tool.ts", gmailPreferencesTool],
|
|
272
280
|
|
|
273
281
|
// google-calendar
|
|
274
282
|
["google-calendar:tools/calendar-list-events.ts", calendarListEvents],
|
|
@@ -81,11 +81,41 @@ export function getWorkspaceDirOverride(): string | undefined {
|
|
|
81
81
|
return str("VELLUM_WORKSPACE_DIR");
|
|
82
82
|
}
|
|
83
83
|
|
|
84
|
+
/**
|
|
85
|
+
* VELLUM_BACKUP_DIR — string, default: undefined
|
|
86
|
+
* Overrides the default backup root directory (~/.vellum/backups/).
|
|
87
|
+
* Used in containerized deployments where the backup directory must be
|
|
88
|
+
* on a persistent volume.
|
|
89
|
+
*/
|
|
90
|
+
export function getBackupDirOverride(): string | undefined {
|
|
91
|
+
return str("VELLUM_BACKUP_DIR");
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* VELLUM_BACKUP_KEY_PATH — string, default: undefined
|
|
96
|
+
* Overrides the default backup encryption key path (~/.vellum/protected/backup.key).
|
|
97
|
+
* Used in containerized deployments where the key must be on a persistent volume.
|
|
98
|
+
*/
|
|
99
|
+
export function getBackupKeyPathOverride(): string | undefined {
|
|
100
|
+
return str("VELLUM_BACKUP_KEY_PATH");
|
|
101
|
+
}
|
|
102
|
+
|
|
84
103
|
// ── Profiler env vars ───────────────────────────────────────────────────
|
|
85
104
|
// These are injected by the platform when running a managed assistant in
|
|
86
105
|
// profiler mode. The runtime uses them to locate, scope, and budget-limit
|
|
87
106
|
// profiler output on the workspace volume.
|
|
88
107
|
|
|
108
|
+
/**
|
|
109
|
+
* VELLUM_CPU_LIMIT — string (K8s resource format), default: undefined
|
|
110
|
+
* The CPU resource limit for the container (e.g. "2000m", "2").
|
|
111
|
+
* Set by the platform StatefulSet template to the exact K8s CPU limit.
|
|
112
|
+
* Used by the health endpoint to report accurate CPU core count inside
|
|
113
|
+
* gVisor sandboxes where cgroup files may expose the host node's CPUs.
|
|
114
|
+
*/
|
|
115
|
+
export function getCpuLimit(): string | undefined {
|
|
116
|
+
return str("VELLUM_CPU_LIMIT");
|
|
117
|
+
}
|
|
118
|
+
|
|
89
119
|
/**
|
|
90
120
|
* VELLUM_PROFILER_RUN_ID — string, default: undefined
|
|
91
121
|
* Unique identifier for the current profiler run. When set, the profiler
|
|
@@ -140,15 +170,21 @@ export function getProfilerMinFreeMb(): number | undefined {
|
|
|
140
170
|
*/
|
|
141
171
|
const KNOWN_VELLUM_VARS = new Set([
|
|
142
172
|
"VELLUM_ASSISTANT_NAME",
|
|
173
|
+
"VELLUM_ASSISTANT_PLATFORM_URL",
|
|
143
174
|
"VELLUM_AWS_ROLE_ARN",
|
|
175
|
+
"VELLUM_BACKUP_DIR",
|
|
176
|
+
"VELLUM_BACKUP_KEY_PATH",
|
|
144
177
|
"VELLUM_CLOUD",
|
|
145
178
|
"VELLUM_CUSTOM_QR_CODE_PATH",
|
|
146
179
|
"VELLUM_DAEMON_AUTOSTART",
|
|
147
180
|
"VELLUM_DAEMON_NOAUTH",
|
|
148
181
|
"VELLUM_DATA_DIR",
|
|
182
|
+
"VELLUM_DEBUG",
|
|
149
183
|
"VELLUM_DESKTOP_APP",
|
|
150
184
|
"VELLUM_DEV",
|
|
185
|
+
"VELLUM_DOCS_BASE_URL",
|
|
151
186
|
"VELLUM_ENABLE_INSECURE_LAN_PAIRING",
|
|
187
|
+
"VELLUM_ENVIRONMENT",
|
|
152
188
|
"VELLUM_HATCHED_BY",
|
|
153
189
|
"VELLUM_HOOK_EVENT",
|
|
154
190
|
"VELLUM_HOOK_NAME",
|
|
@@ -164,6 +200,8 @@ const KNOWN_VELLUM_VARS = new Set([
|
|
|
164
200
|
"VELLUM_SSH_USER",
|
|
165
201
|
"VELLUM_UNSAFE_AUTH_BYPASS",
|
|
166
202
|
"VELLUM_WORKSPACE_DIR",
|
|
203
|
+
"VELLUM_CPU_LIMIT",
|
|
204
|
+
"VELLUM_MEMORY_LIMIT",
|
|
167
205
|
]);
|
|
168
206
|
|
|
169
207
|
/**
|
package/src/config/env.ts
CHANGED
|
@@ -152,10 +152,18 @@ export function getPlatformBaseUrl(): string {
|
|
|
152
152
|
} catch {
|
|
153
153
|
// Config not yet available (early bootstrap) — fall through
|
|
154
154
|
}
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
155
|
+
// Resolve the default platform URL from VELLUM_ENVIRONMENT.
|
|
156
|
+
// Default to dev-platform for safety; only production and staging
|
|
157
|
+
// opt into their respective platforms explicitly.
|
|
158
|
+
const env = str("VELLUM_ENVIRONMENT")?.trim();
|
|
159
|
+
let defaultUrl: string;
|
|
160
|
+
if (env === "production") {
|
|
161
|
+
defaultUrl = "https://platform.vellum.ai";
|
|
162
|
+
} else if (env === "staging") {
|
|
163
|
+
defaultUrl = "https://staging-platform.vellum.ai";
|
|
164
|
+
} else {
|
|
165
|
+
defaultUrl = "https://dev-platform.vellum.ai";
|
|
166
|
+
}
|
|
159
167
|
return (
|
|
160
168
|
configUrl ||
|
|
161
169
|
str("VELLUM_PLATFORM_URL") ||
|
|
@@ -164,6 +172,43 @@ export function getPlatformBaseUrl(): string {
|
|
|
164
172
|
);
|
|
165
173
|
}
|
|
166
174
|
|
|
175
|
+
/**
|
|
176
|
+
* Derive the assistant service domain from the platform base URL.
|
|
177
|
+
*
|
|
178
|
+
* Known platform URLs map directly (via regex stripping of `platform.vellum.ai`):
|
|
179
|
+
* - `dev-platform.vellum.ai` → `dev.vellum.me`
|
|
180
|
+
* - `staging-platform.vellum.ai` → `staging.vellum.me` (derived automatically from the staging URL)
|
|
181
|
+
* - `platform.vellum.ai` → `vellum.me`
|
|
182
|
+
*
|
|
183
|
+
* Non-vellum.ai hosts (localhost, host.docker.internal, etc.) use
|
|
184
|
+
* VELLUM_ENVIRONMENT to derive the subdomain, defaulting to `local`:
|
|
185
|
+
* - local → `local.vellum.me`
|
|
186
|
+
*/
|
|
187
|
+
export function getAssistantDomain(): string {
|
|
188
|
+
try {
|
|
189
|
+
const url = getPlatformBaseUrl();
|
|
190
|
+
const host = new URL(url).hostname;
|
|
191
|
+
|
|
192
|
+
if (host.endsWith("platform.vellum.ai")) {
|
|
193
|
+
const prefix = host.replace(/[-.]?platform\.vellum\.ai$/, "");
|
|
194
|
+
if (prefix) {
|
|
195
|
+
return `${prefix}.vellum.me`;
|
|
196
|
+
}
|
|
197
|
+
return "vellum.me";
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
// Non-vellum.ai host (local dev, Docker, etc.) — derive from environment
|
|
201
|
+
const env = str("VELLUM_ENVIRONMENT")?.trim();
|
|
202
|
+
if (env && env !== "production") {
|
|
203
|
+
return `${env}.vellum.me`;
|
|
204
|
+
}
|
|
205
|
+
return "local.vellum.me";
|
|
206
|
+
} catch {
|
|
207
|
+
// Fall through to default
|
|
208
|
+
}
|
|
209
|
+
return "vellum.me";
|
|
210
|
+
}
|
|
211
|
+
|
|
167
212
|
let _platformAssistantIdOverride: string | undefined;
|
|
168
213
|
|
|
169
214
|
export function setPlatformAssistantId(value: string | undefined): void {
|
|
@@ -1,6 +1,14 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 1,
|
|
3
3
|
"flags": [
|
|
4
|
+
{
|
|
5
|
+
"id": "auto-analyze",
|
|
6
|
+
"scope": "assistant",
|
|
7
|
+
"key": "auto-analyze",
|
|
8
|
+
"label": "Auto-analyze conversations",
|
|
9
|
+
"description": "Automatically trigger conversation analysis on the same cadence as memory extraction (batch threshold, idle debounce, end-of-conversation). The analysis agent has full tool access and writes back to memory and skills without user approval.",
|
|
10
|
+
"defaultEnabled": false
|
|
11
|
+
},
|
|
4
12
|
{
|
|
5
13
|
"id": "browser",
|
|
6
14
|
"scope": "assistant",
|
|
@@ -153,14 +161,6 @@
|
|
|
153
161
|
"description": "Enable the Deploy to Vercel / Publish option in the app workspace header share menu",
|
|
154
162
|
"defaultEnabled": false
|
|
155
163
|
},
|
|
156
|
-
{
|
|
157
|
-
"id": "managed-google-oauth",
|
|
158
|
-
"scope": "assistant",
|
|
159
|
-
"key": "managed-google-oauth",
|
|
160
|
-
"label": "Managed Google OAuth",
|
|
161
|
-
"description": "Show the Google OAuth service card in Models & Services settings",
|
|
162
|
-
"defaultEnabled": true
|
|
163
|
-
},
|
|
164
164
|
{
|
|
165
165
|
"id": "settings-embedding-provider",
|
|
166
166
|
"scope": "assistant",
|
|
@@ -178,11 +178,11 @@
|
|
|
178
178
|
"defaultEnabled": false
|
|
179
179
|
},
|
|
180
180
|
{
|
|
181
|
-
"id": "
|
|
181
|
+
"id": "multi-platform-assistant",
|
|
182
182
|
"scope": "assistant",
|
|
183
|
-
"key": "
|
|
184
|
-
"label": "
|
|
185
|
-
"description": "
|
|
183
|
+
"key": "multi-platform-assistant",
|
|
184
|
+
"label": "Multi-Platform Assistant Switcher",
|
|
185
|
+
"description": "Enable the menu-bar assistant switcher for managing multiple platform-hosted assistants (new/switch/retire)",
|
|
186
186
|
"defaultEnabled": false
|
|
187
187
|
},
|
|
188
188
|
{
|
|
@@ -262,7 +262,7 @@
|
|
|
262
262
|
"scope": "assistant",
|
|
263
263
|
"key": "fast-mode",
|
|
264
264
|
"label": "Fast Mode",
|
|
265
|
-
"description": "Enable Anthropic fast mode for Opus 4.6, delivering up to 2.5x higher output tokens per second at premium pricing",
|
|
265
|
+
"description": "Enable Anthropic fast mode for Opus models (4.6, 4.7), delivering up to 2.5x higher output tokens per second at premium pricing",
|
|
266
266
|
"defaultEnabled": false
|
|
267
267
|
},
|
|
268
268
|
{
|
|
@@ -288,7 +288,78 @@
|
|
|
288
288
|
"label": "Permission Controls V2",
|
|
289
289
|
"description": "Replace risk-level permission system with two independent controls: 'Ask before acting' (LLM behavior toggle) and 'Host access' (system-enforced gate)",
|
|
290
290
|
"defaultEnabled": false
|
|
291
|
+
},
|
|
292
|
+
{
|
|
293
|
+
"id": "apple-container",
|
|
294
|
+
"scope": "macos",
|
|
295
|
+
"key": "apple-container",
|
|
296
|
+
"label": "Apple Container",
|
|
297
|
+
"description": "Enable assistant sandboxing via Apple Containers on macOS 26+, providing a lightweight native sandbox without third-party dependencies",
|
|
298
|
+
"defaultEnabled": false
|
|
299
|
+
},
|
|
300
|
+
{
|
|
301
|
+
"id": "tool-result-truncation",
|
|
302
|
+
"scope": "assistant",
|
|
303
|
+
"key": "tool-result-truncation",
|
|
304
|
+
"label": "Post-Turn Tool Result Truncation",
|
|
305
|
+
"description": "Truncate large tool results after each assistant turn, persisting full content to disk for on-demand re-reads",
|
|
306
|
+
"defaultEnabled": true
|
|
307
|
+
},
|
|
308
|
+
{
|
|
309
|
+
"id": "managed-gemini-embeddings-enabled",
|
|
310
|
+
"scope": "assistant",
|
|
311
|
+
"key": "managed-gemini-embeddings-enabled",
|
|
312
|
+
"label": "Managed Gemini Embeddings Enabled",
|
|
313
|
+
"description": "Route embedding requests through the platform runtime proxy using Vellum-managed Gemini credentials when available",
|
|
314
|
+
"defaultEnabled": false
|
|
315
|
+
},
|
|
316
|
+
{
|
|
317
|
+
"id": "fork-from-message",
|
|
318
|
+
"scope": "macos",
|
|
319
|
+
"key": "fork-from-message",
|
|
320
|
+
"label": "Fork from Message",
|
|
321
|
+
"description": "Show the 'Fork from here' option in message overflow menus",
|
|
322
|
+
"defaultEnabled": false
|
|
323
|
+
},
|
|
324
|
+
{
|
|
325
|
+
"id": "managed-x-oauth-integration",
|
|
326
|
+
"scope": "assistant",
|
|
327
|
+
"key": "managed-x-oauth-integration",
|
|
328
|
+
"label": "Managed X/Twitter OAuth",
|
|
329
|
+
"description": "Enable platform-managed X/Twitter OAuth integration in Models & Services settings",
|
|
330
|
+
"defaultEnabled": false
|
|
331
|
+
},
|
|
332
|
+
{
|
|
333
|
+
"id": "onboarding-pre-chat",
|
|
334
|
+
"scope": "macos",
|
|
335
|
+
"key": "onboarding-pre-chat",
|
|
336
|
+
"label": "Pre-Chat Onboarding Flow",
|
|
337
|
+
"description": "Gates the 3-screen pre-chat onboarding flow (tools, tasks/tone, name exchange) shown before the first conversation",
|
|
338
|
+
"defaultEnabled": false
|
|
339
|
+
},
|
|
340
|
+
{
|
|
341
|
+
"id": "home-tab",
|
|
342
|
+
"scope": "macos",
|
|
343
|
+
"key": "home-tab",
|
|
344
|
+
"label": "Home Tab",
|
|
345
|
+
"description": "Replace the knowledge graph top-level tab with a new Home page showing relationship progression, facts, and capability tiers",
|
|
346
|
+
"defaultEnabled": false
|
|
347
|
+
},
|
|
348
|
+
{
|
|
349
|
+
"id": "channel-avatar-sync",
|
|
350
|
+
"scope": "assistant",
|
|
351
|
+
"key": "channel-avatar-sync",
|
|
352
|
+
"label": "Channel Avatar Sync",
|
|
353
|
+
"description": "Automatically sync the assistant's avatar to connected channels (Slack) when the avatar changes or a new channel is connected",
|
|
354
|
+
"defaultEnabled": true
|
|
355
|
+
},
|
|
356
|
+
{
|
|
357
|
+
"id": "meet",
|
|
358
|
+
"scope": "assistant",
|
|
359
|
+
"key": "meet",
|
|
360
|
+
"label": "Google Meet",
|
|
361
|
+
"description": "Enables the Google Meet joining bot and the meet-join skill.",
|
|
362
|
+
"defaultEnabled": false
|
|
291
363
|
}
|
|
292
364
|
]
|
|
293
365
|
}
|
|
294
|
-
|