@vellumai/assistant 0.11.6 → 0.11.7-staging.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +31 -19
- package/Dockerfile +1 -0
- package/docs/architecture/security.md +3 -3
- package/docs/flux-turn-detection-spike.md +8 -7
- package/docs/runbook-trusted-contacts.md +14 -0
- package/docs/stt-provider-onboarding.md +8 -4
- package/knip.json +1 -0
- package/node_modules/@vellumai/avatar-manifest/package.json +18 -0
- package/node_modules/@vellumai/avatar-manifest/src/__tests__/manifest.test.ts +173 -0
- package/node_modules/@vellumai/avatar-manifest/src/__tests__/read.test.ts +114 -0
- package/node_modules/@vellumai/avatar-manifest/src/index.ts +29 -0
- package/node_modules/@vellumai/avatar-manifest/src/layout.ts +18 -0
- package/node_modules/@vellumai/avatar-manifest/src/manifest.ts +156 -0
- package/node_modules/@vellumai/avatar-manifest/src/read.ts +109 -0
- package/node_modules/@vellumai/avatar-manifest/tsconfig.json +20 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/__tests__/ingress.test.ts +37 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/__tests__/remote-web-pairing.test.ts +389 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/ingress.ts +22 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/remote-web-pairing.ts +508 -18
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/rpc.ts +174 -0
- package/node_modules/@vellumai/ces-client/src/http-credentials.ts +161 -0
- package/node_modules/@vellumai/ces-client/src/index.ts +1 -0
- package/node_modules/@vellumai/environments/package.json +2 -1
- package/node_modules/@vellumai/environments/src/shell.test.ts +56 -0
- package/node_modules/@vellumai/environments/src/shell.ts +65 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/__tests__/ingress.test.ts +37 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/__tests__/remote-web-pairing.test.ts +389 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/ingress.ts +22 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/remote-web-pairing.ts +508 -18
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/rpc.ts +174 -0
- package/node_modules/@vellumai/gateway-client/src/__tests__/contact-read-contracts.test.ts +32 -0
- package/node_modules/@vellumai/gateway-client/src/__tests__/gateway-client.test.ts +1 -0
- package/node_modules/@vellumai/gateway-client/src/__tests__/inbound-event-kind.test.ts +82 -0
- package/node_modules/@vellumai/gateway-client/src/__tests__/plugin-admission-denied-contract.test.ts +56 -0
- package/node_modules/@vellumai/gateway-client/src/__tests__/trust-verdict-contract.test.ts +18 -0
- package/node_modules/@vellumai/gateway-client/src/admission-policy-contract.ts +5 -11
- package/node_modules/@vellumai/gateway-client/src/channel-permission-contract.ts +3 -1
- package/node_modules/@vellumai/gateway-client/src/gateway-ipc-contracts.ts +9 -0
- package/node_modules/@vellumai/gateway-client/src/inbound-contract.ts +4 -0
- package/node_modules/@vellumai/gateway-client/src/inbound-event-kind.ts +75 -0
- package/node_modules/@vellumai/gateway-client/src/index.ts +17 -0
- package/node_modules/@vellumai/gateway-client/src/outbound-contract.ts +8 -0
- package/node_modules/@vellumai/gateway-client/src/plugin-admission-denied-contract.ts +47 -0
- package/node_modules/@vellumai/gateway-client/src/trust-verdict-contract.ts +11 -0
- package/node_modules/@vellumai/service-contracts/src/__tests__/ingress.test.ts +37 -0
- package/node_modules/@vellumai/service-contracts/src/__tests__/remote-web-pairing.test.ts +389 -0
- package/node_modules/@vellumai/service-contracts/src/ingress.ts +22 -0
- package/node_modules/@vellumai/service-contracts/src/remote-web-pairing.ts +508 -18
- package/node_modules/@vellumai/service-contracts/src/rpc.ts +174 -0
- package/openapi.yaml +419 -55
- package/package.json +4 -1
- package/scripts/sync-llm-catalog.ts +6 -0
- package/src/__tests__/approval-card-edit-inbound-handoff.test.ts +199 -0
- package/src/__tests__/approval-card-edit.test.ts +153 -0
- package/src/__tests__/attachments-store-heic-normalize.test.ts +1 -1
- package/src/__tests__/available-skills.test.ts +54 -0
- package/src/__tests__/avatar-identity-sync.test.ts +5 -1
- package/src/__tests__/catalog-install-normalize.test.ts +9 -1
- package/src/__tests__/channel-policy.test.ts +1 -3
- package/src/__tests__/channel-readiness-discord.test.ts +9 -70
- package/src/__tests__/compactor-call-site-logging.test.ts +61 -0
- package/src/__tests__/config-loader-backfill.test.ts +69 -0
- package/src/__tests__/config-schema.test.ts +2 -3
- package/src/__tests__/conversation-agent-loop.test.ts +238 -0
- package/src/__tests__/conversation-attachments.test.ts +0 -1
- package/src/__tests__/conversation-error.test.ts +44 -0
- package/src/__tests__/conversation-evictor.test.ts +19 -0
- package/src/__tests__/conversation-fork-referential.test.ts +73 -63
- package/src/__tests__/conversation-fork-retrospective.test.ts +99 -74
- package/src/__tests__/conversation-runtime-assembly.test.ts +23 -0
- package/src/__tests__/conversation-slash.test.ts +9 -0
- package/src/__tests__/conversation-title-service.test.ts +97 -0
- package/src/__tests__/conversation-tool-setup-attribution.test.ts +16 -0
- package/src/__tests__/credential-broker-server-use.test.ts +0 -218
- package/src/__tests__/credential-broker.test.ts +43 -420
- package/src/__tests__/credential-record-write-through.test.ts +78 -0
- package/src/__tests__/credential-routes.test.ts +17 -1
- package/src/__tests__/credential-security-invariants.test.ts +8 -9
- package/src/__tests__/default-profile-catalog-fallback.test.ts +289 -0
- package/src/__tests__/delete-propagation.test.ts +92 -2
- package/src/__tests__/discord-callback-round-trip.test.ts +93 -0
- package/src/__tests__/discord-channel-config.test.ts +237 -0
- package/src/__tests__/edit-propagation.test.ts +42 -2
- package/src/__tests__/evict-conversations-for-reload.test.ts +51 -1
- package/src/__tests__/fallback-breaker.test.ts +1335 -0
- package/src/__tests__/gateway-threshold-reader-mock.ts +6 -0
- package/src/__tests__/guardian-card-withdrawal.test.ts +30 -0
- package/src/__tests__/handlers-user-message-approval-consumption.test.ts +0 -2
- package/src/__tests__/image-conversion.test.ts +38 -4
- package/src/__tests__/install-skill-routing.test.ts +25 -0
- package/src/__tests__/llm-catalog-parity.test.ts +4 -0
- package/src/__tests__/llm-resolver.test.ts +74 -0
- package/src/__tests__/llm-schema-fallback-profile.test.ts +359 -0
- package/src/__tests__/managed-fallback-dispatch.test.ts +430 -0
- package/src/__tests__/managed-profile-guard.test.ts +28 -0
- package/src/__tests__/managed-speech-defaults.test.ts +196 -1
- package/src/__tests__/mcp-cli.test.ts +23 -1
- package/src/__tests__/mcp-tool-annotations-risk.test.ts +43 -34
- package/src/__tests__/media-stream-stt-session.test.ts +2 -0
- package/src/__tests__/notification-decision-fallback.test.ts +106 -0
- package/src/__tests__/notification-discord-adapter.test.ts +162 -0
- package/src/__tests__/oauth-connect-orchestrator.test.ts +95 -0
- package/src/__tests__/persist-unsendable-image-downscale.test.ts +6 -8
- package/src/__tests__/persist-unsendable-image.test.ts +3 -3
- package/src/__tests__/platform.test.ts +19 -1
- package/src/__tests__/plugin-import-boundary-guard.test.ts +5 -1
- package/src/__tests__/provider-catalog-visibility.test.ts +17 -0
- package/src/__tests__/provider-platform-proxy-integration.test.ts +8 -1
- package/src/__tests__/provider-usage-tracking.test.ts +56 -0
- package/src/__tests__/require-fresh-approval.test.ts +5 -2
- package/src/__tests__/retry-fallback-escalation.test.ts +1093 -0
- package/src/__tests__/schedule-retry.test.ts +183 -4
- package/src/__tests__/scheduler-reuse-conversation.test.ts +6 -6
- package/src/__tests__/secret-routes-platform-proxy.test.ts +55 -0
- package/src/__tests__/skill-load-tool.test.ts +34 -0
- package/src/__tests__/skill-tool-factory.test.ts +6 -1
- package/src/__tests__/skill-tool-manifest.test.ts +20 -0
- package/src/__tests__/skills.test.ts +25 -0
- package/src/__tests__/subagent-manager-notify.test.ts +38 -0
- package/src/__tests__/terminal-tools.test.ts +2 -0
- package/src/__tests__/tool-approval-handler.test.ts +146 -2
- package/src/__tests__/tool-approval-seed-content-blocks.test.ts +108 -2
- package/src/__tests__/tool-audit.test.ts +111 -40
- package/src/__tests__/tool-executor-lifecycle-events.test.ts +228 -18
- package/src/__tests__/ui-shape-teaching.test.ts +1 -1
- package/src/__tests__/verification-control-plane-policy.test.ts +6 -2
- package/src/__tests__/voice-config-update.test.ts +43 -0
- package/src/__tests__/workspace-migration-147-rename-colliding-backup-profile-names.test.ts +480 -0
- package/src/__tests__/workspace-migration-148-strip-unsupported-fallback-profiles.test.ts +167 -0
- package/src/__tests__/workspace-migration-149-repoint-backup-profile-selections.test.ts +144 -0
- package/src/agent/loop.ts +31 -2
- package/src/approvals/guardian-card-withdrawal.ts +22 -0
- package/src/avatar/__tests__/ensure-raster.test.ts +291 -0
- package/src/avatar/avatar-manifest.ts +48 -100
- package/src/avatar/avatar-store.ts +7 -7
- package/src/avatar/ensure-raster.ts +146 -0
- package/src/avatar/resvg-lazy.test.ts +0 -1
- package/src/avatar/traits-png-sync.ts +9 -10
- package/src/background-wake/wake-intent-hooks.test.ts +0 -1
- package/src/backup/__tests__/paths.test.ts +6 -2
- package/src/backup/paths.ts +6 -4
- package/src/bundler/app-compiler.ts +2 -2
- package/src/bundler/compiler-tools.test.ts +36 -0
- package/src/bundler/compiler-tools.ts +54 -18
- package/src/calls/__tests__/telephony-synthesis-language.test.ts +81 -0
- package/src/calls/__tests__/voice-session-bridge.test.ts +57 -0
- package/src/calls/media-stream-stt-session.ts +2 -1
- package/src/calls/telephony-synthesis-language.ts +13 -5
- package/src/calls/voice-session-bridge.ts +19 -4
- package/src/channels/config.ts +23 -12
- package/src/cli/AGENTS.md +3 -1
- package/src/cli/commands/backup.help.ts +10 -13
- package/src/cli/commands/contacts.help.ts +16 -4
- package/src/cli/commands/contacts.ts +5 -1
- package/src/cli/commands/credentials.help.ts +1 -1
- package/src/cli/commands/inference-providers.ts +18 -53
- package/src/cli/commands/mcp.help.ts +6 -5
- package/src/cli/commands/mcp.ts +1 -1
- package/src/cli/commands/memory/memory-retrospective.ts +1 -0
- package/src/cli/commands/stt.help.ts +2 -2
- package/src/config/__tests__/backup-schema.test.ts +1 -1
- package/src/config/__tests__/default-profile-catalog.test.ts +83 -2
- package/src/config/__tests__/loader-sparse-array-cleanup.test.ts +1 -1
- package/src/config/__tests__/memory-retrospective-schema.test.ts +12 -15
- package/src/config/bundled-skills/computer-use/SKILL.md +6 -6
- package/src/config/bundled-skills/computer-use/TOOLS.json +9 -6
- package/src/config/bundled-skills/media-processing/services/audio-transcribe.ts +3 -1
- package/src/config/bundled-skills/media-processing/services/preprocess.ts +1 -1
- package/src/config/bundled-skills/phone-calls/SKILL.md +1 -1
- package/src/config/bundled-skills/phone-calls/references/CONFIG.md +16 -15
- package/src/config/bundled-skills/phone-calls/references/TROUBLESHOOTING.md +2 -1
- package/src/config/bundled-skills/settings/TOOLS.json +11 -5
- package/src/config/bundled-skills/settings/tools/open-system-settings.test.ts +85 -0
- package/src/config/bundled-skills/settings/tools/open-system-settings.ts +33 -8
- package/src/config/bundled-skills/settings/tools/voice-config-update.ts +54 -3
- package/src/config/bundled-skills/transcribe/tools/transcribe-media.ts +1 -1
- package/src/config/default-profile-catalog.ts +195 -10
- package/src/config/default-profile-names.ts +41 -0
- package/src/config/feature-flag-registry.json +34 -2
- package/src/config/inference-profile-validation.ts +11 -2
- package/src/config/llm-resolver.ts +25 -0
- package/src/config/loader.ts +3 -0
- package/src/config/managed-speech-defaults.ts +160 -10
- package/src/config/schema.ts +2 -0
- package/src/config/schemas/__tests__/live-voice.test.ts +7 -6
- package/src/config/schemas/__tests__/stt.test.ts +94 -0
- package/src/config/schemas/backup.ts +1 -1
- package/src/config/schemas/channels.ts +32 -0
- package/src/config/schemas/live-voice.ts +13 -4
- package/src/config/schemas/llm.ts +263 -15
- package/src/config/schemas/mcp.ts +10 -2
- package/src/config/schemas/memory-retrospective.ts +1 -19
- package/src/config/schemas/services.ts +24 -0
- package/src/config/schemas/stt.ts +128 -5
- package/src/config/seed-inference-profiles.ts +32 -3
- package/src/config/skill-state.ts +5 -0
- package/src/config/skills.ts +21 -0
- package/src/contacts/notify-contacts-changed.ts +5 -2
- package/src/context/compactor.ts +74 -1
- package/src/daemon/__tests__/conversation-tool-setup.test.ts +138 -1
- package/src/daemon/config-watcher.ts +2 -1
- package/src/daemon/conversation-agent-loop-handlers.ts +27 -0
- package/src/daemon/conversation-agent-loop.ts +86 -10
- package/src/daemon/conversation-error.ts +25 -0
- package/src/daemon/conversation-evictor.ts +2 -4
- package/src/daemon/conversation-runtime-assembly.ts +19 -7
- package/src/daemon/conversation-skill-tools.ts +5 -1
- package/src/daemon/conversation-slash.ts +2 -2
- package/src/daemon/conversation-store.ts +8 -1
- package/src/daemon/conversation-tool-setup.ts +37 -0
- package/src/daemon/conversation.ts +0 -4
- package/src/daemon/handlers/channel-config-result.ts +30 -0
- package/src/daemon/handlers/config-discord-channel.ts +301 -0
- package/src/daemon/handlers/config-ingress.ts +9 -0
- package/src/daemon/handlers/config-slack-channel.ts +13 -15
- package/src/daemon/handlers/config-telegram.ts +2 -5
- package/src/daemon/handlers/conversations.ts +0 -2
- package/src/daemon/handlers/skills.ts +25 -7
- package/src/daemon/lifecycle.ts +3 -1
- package/src/daemon/message-types/messages.ts +0 -2
- package/src/daemon/tool-setup-types.ts +5 -3
- package/src/daemon/trust-context-types.test.ts +33 -0
- package/src/heartbeat/__tests__/heartbeat-service.test.ts +0 -1
- package/src/live-voice/__tests__/live-voice-flux-turn-end.test.ts +58 -4
- package/src/live-voice/__tests__/live-voice-managed-speech-fallback.test.ts +8 -3
- package/src/live-voice/__tests__/live-voice-session-manager.test.ts +73 -11
- package/src/live-voice/__tests__/live-voice-session-preflight.test.ts +148 -2
- package/src/live-voice/__tests__/live-voice-session-telemetry.test.ts +64 -0
- package/src/live-voice/__tests__/live-voice-stt.test.ts +3 -0
- package/src/live-voice/__tests__/live-voice-text-turn.test.ts +391 -0
- package/src/live-voice/__tests__/protocol.test.ts +224 -0
- package/src/live-voice/live-voice-credential-preflight.ts +5 -1
- package/src/live-voice/live-voice-session-manager.ts +39 -2
- package/src/live-voice/live-voice-session.ts +224 -9
- package/src/live-voice/protocol.ts +187 -1
- package/src/messaging/providers/discord/api.ts +2 -1
- package/src/messaging/providers/discord/withdraw.ts +62 -0
- package/src/messaging/read-provider-metadata.ts +32 -0
- package/src/monitoring/__tests__/file-descriptors.test.ts +32 -0
- package/src/monitoring/__tests__/process-memory.test.ts +34 -1
- package/src/monitoring/db-integrity-sample.ts +1 -0
- package/src/monitoring/file-descriptors.ts +39 -6
- package/src/monitoring/process-memory.ts +37 -5
- package/src/monitoring/resource-sampler.ts +25 -8
- package/src/notifications/__tests__/activity-failed-dedupe.test.ts +161 -0
- package/src/notifications/__tests__/background-failure-signal.test.ts +107 -0
- package/src/notifications/__tests__/connected-channels.test.ts +12 -0
- package/src/notifications/__tests__/copy-composer.test.ts +83 -0
- package/src/notifications/__tests__/destination-resolver.test.ts +38 -0
- package/src/notifications/activity-failed-dedupe.ts +95 -0
- package/src/notifications/adapters/discord.ts +132 -0
- package/src/notifications/adapters/shared.ts +16 -0
- package/src/notifications/adapters/slack.ts +6 -1
- package/src/notifications/adapters/telegram.ts +5 -8
- package/src/notifications/approval-card-data.ts +94 -1
- package/src/notifications/background-failure-signal.ts +116 -0
- package/src/notifications/broadcaster.ts +3 -9
- package/src/notifications/copy-composer.ts +41 -8
- package/src/notifications/decision-engine.ts +55 -0
- package/src/notifications/destination-resolver.ts +45 -5
- package/src/notifications/emit-signal.ts +19 -5
- package/src/notifications/guardian-question-mode.ts +57 -0
- package/src/notifications/types.ts +2 -8
- package/src/oauth/__tests__/seed-providers-managed.test.ts +18 -0
- package/src/oauth/connect-orchestrator.ts +16 -0
- package/src/oauth/seed-providers.ts +262 -0
- package/src/permissions/checker.test.ts +21 -0
- package/src/permissions/checker.ts +6 -0
- package/src/permissions/gateway-threshold-reader.test.ts +184 -0
- package/src/permissions/gateway-threshold-reader.ts +144 -10
- package/src/permissions/prompter.ts +0 -6
- package/src/permissions/types.ts +5 -4
- package/src/persistence/__tests__/bulk-write-gate.test.ts +0 -38
- package/src/persistence/bulk-write-gate.ts +9 -10
- package/src/persistence/conversation-crud.ts +88 -134
- package/src/persistence/conversation-title-service.ts +43 -29
- package/src/persistence/embeddings/embedding-local.ts +14 -85
- package/src/persistence/embeddings/embedding-runtime-manager.ts +1 -0
- package/src/persistence/lifecycle-events-store.test.ts +50 -0
- package/src/persistence/lifecycle-events-store.ts +75 -5
- package/src/platform/platform-patch-queue.test.ts +368 -0
- package/src/platform/platform-patch-queue.ts +190 -0
- package/src/platform/sync-avatar.test.ts +527 -0
- package/src/platform/sync-avatar.ts +209 -0
- package/src/platform/sync-identity.test.ts +75 -0
- package/src/platform/sync-identity.ts +17 -110
- package/src/plugin-api/model-profiles.test.ts +2 -2
- package/src/plugin-api/model-profiles.ts +7 -7
- package/src/plugin-api/transcription-session.ts +1 -1
- package/src/plugins/defaults/compaction/window-manager.ts +4 -0
- package/src/plugins/defaults/image-recovery/recover.ts +2 -2
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-job.test.ts +133 -34
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-prompt.test.ts +21 -31
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-provider-path.test.ts +13 -9
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-wake-chain.test.ts +31 -4
- package/src/plugins/defaults/memory/memory-retrospective-constants.ts +0 -11
- package/src/plugins/defaults/memory/memory-retrospective-fork-boundary.ts +27 -13
- package/src/plugins/defaults/memory/memory-retrospective-job.ts +183 -113
- package/src/plugins/defaults/memory/memory-retrospective-prompt.ts +13 -13
- package/src/plugins/defaults/memory/src/memory-v2-routes.ts +2 -2
- package/src/plugins/defaults/memory/substrate/__tests__/consolidation-job.test.ts +1 -1
- package/src/plugins/defaults/memory/substrate/consolidation-job.ts +4 -2
- package/src/plugins/defaults/memory/substrate/sweep-job.ts +9 -42
- package/src/plugins/defaults/memory/v2/rerank-local.ts +1 -0
- package/src/plugins/defaults/memory/worker.ts +7 -0
- package/src/providers/__tests__/provider-secret-catalog.test.ts +1 -0
- package/src/providers/__tests__/retry-callsite.test.ts +55 -0
- package/src/providers/__tests__/vellum-connection-routing.test.ts +21 -3
- package/src/providers/connection-resolution.ts +2 -17
- package/src/providers/fallback-breaker.ts +389 -0
- package/src/providers/inference/__tests__/adapter-factory-opencode.test.ts +92 -0
- package/src/providers/inference/__tests__/base-url-route-validation.test.ts +16 -0
- package/src/providers/inference/__tests__/base-url-security.test.ts +12 -0
- package/src/providers/inference/adapter-factory.ts +246 -3
- package/src/providers/inference/auth.ts +15 -14
- package/src/providers/inference/chatgpt-subscription-auth.ts +70 -0
- package/src/providers/inference/chatgpt-subscription-credentials.ts +84 -0
- package/src/providers/inference/resolve-auth.ts +1 -1
- package/src/providers/media-resolve.ts +2 -3
- package/src/providers/model-catalog.ts +100 -0
- package/src/providers/openai/__tests__/api-error-normalization.test.ts +22 -0
- package/src/providers/openai/__tests__/chat-template-flatten.test.ts +198 -0
- package/src/providers/openai/api-error-normalization.ts +13 -0
- package/src/providers/openai/chat-completions-provider.ts +84 -0
- package/src/providers/opencode/client.test.ts +150 -0
- package/src/providers/opencode/client.ts +61 -0
- package/src/providers/platform-proxy/constants.ts +5 -0
- package/src/providers/provider-resolvability.ts +20 -0
- package/src/providers/provider-secret-catalog.ts +3 -2
- package/src/providers/retry.ts +956 -68
- package/src/providers/speech-to-text/__tests__/deepgram-flux-frames.test.ts +42 -0
- package/src/providers/speech-to-text/__tests__/deepgram-flux-realtime.test.ts +136 -0
- package/src/providers/speech-to-text/__tests__/provider-catalog.test.ts +120 -0
- package/src/providers/speech-to-text/__tests__/resolve.test.ts +316 -12
- package/src/providers/speech-to-text/deepgram-flux-frames.ts +83 -3
- package/src/providers/speech-to-text/deepgram-flux-realtime.ts +132 -4
- package/src/providers/speech-to-text/provider-catalog.ts +191 -3
- package/src/providers/speech-to-text/resolve.ts +132 -24
- package/src/providers/speech-to-text/vellum-managed-flux-realtime.ts +296 -0
- package/src/providers/types.ts +39 -0
- package/src/providers/usage-tracking.ts +12 -1
- package/src/providers/vellum/client.ts +29 -0
- package/src/providers/vellum-model-routing.test.ts +2 -0
- package/src/providers/vellum-model-routing.ts +3 -3
- package/src/runtime/AGENTS.md +1 -2
- package/src/runtime/__tests__/agent-wake.test.ts +105 -3
- package/src/runtime/__tests__/background-job-runner.test.ts +89 -3
- package/src/runtime/agent-wake.ts +44 -5
- package/src/runtime/background-job-runner.ts +68 -50
- package/src/runtime/channel-readiness-service.ts +0 -39
- package/src/runtime/http-server.ts +12 -4
- package/src/runtime/routes/__tests__/avatar-state-routes.test.ts +18 -18
- package/src/runtime/routes/__tests__/backup-routes.test.ts +9 -5
- package/src/runtime/routes/__tests__/chatgpt-subscription-auth-routes.test.ts +372 -0
- package/src/runtime/routes/__tests__/contact-routes.test.ts +7 -0
- package/src/runtime/routes/__tests__/conversation-query-routes.test.ts +96 -0
- package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +37 -0
- package/src/runtime/routes/__tests__/ingress-status-routes.test.ts +2 -1
- package/src/runtime/routes/__tests__/mcp-add-default-risk.test.ts +87 -0
- package/src/runtime/routes/acp-claude-auth-routes.ts +25 -54
- package/src/runtime/routes/approval-routes.ts +0 -12
- package/src/runtime/routes/avatar-routes.ts +17 -30
- package/src/runtime/routes/backup-routes.ts +2 -2
- package/src/runtime/routes/chatgpt-subscription-auth-routes.ts +296 -113
- package/src/runtime/routes/contact-prompt-routes.ts +65 -6
- package/src/runtime/routes/contact-routes.ts +20 -15
- package/src/runtime/routes/conversation-query-routes.ts +98 -10
- package/src/runtime/routes/conversation-routes.ts +2 -2
- package/src/runtime/routes/credential-routes.ts +2 -1
- package/src/runtime/routes/guardian-approval-interception.ts +12 -9
- package/src/runtime/routes/inbound-message-handler.ts +47 -48
- package/src/runtime/routes/inbound-stages/acl-enforcement.ts +18 -13
- package/src/runtime/routes/inbound-stages/admission-policy.ts +2 -2
- package/src/runtime/routes/inbound-stages/edit-intercept.ts +20 -2
- package/src/runtime/routes/inbound-stages/reaction-intercept.ts +9 -8
- package/src/runtime/routes/inbound-stages/transcribe-audio.ts +1 -1
- package/src/runtime/routes/index.ts +2 -0
- package/src/runtime/routes/inference-profile-session-handler.ts +2 -2
- package/src/runtime/routes/inference-profiles-routes.ts +12 -7
- package/src/runtime/routes/inference-provider-connection-routes.ts +2 -2
- package/src/runtime/routes/inference-send-routes.ts +2 -2
- package/src/runtime/routes/ingress-status-routes.ts +2 -2
- package/src/runtime/routes/integrations/discord.ts +119 -0
- package/src/runtime/routes/llm-call-sites-routes.ts +2 -2
- package/src/runtime/routes/mcp-auth-routes.ts +12 -6
- package/src/runtime/routes/migration-routes.ts +3 -1
- package/src/runtime/routes/oauth-pending-flows.ts +136 -0
- package/src/runtime/routes/secret-routes.ts +7 -0
- package/src/runtime/routes/stt-routes.ts +38 -6
- package/src/runtime/routes/wake-conversation-routes.ts +8 -1
- package/src/runtime/routes/watch-routes.ts +9 -5
- package/src/runtime/sync/resource-sync-events.ts +2 -0
- package/src/runtime/trust-verdict-consumer.ts +3 -3
- package/src/schedule/__tests__/worker-feature-flags.test.ts +58 -0
- package/src/schedule/retry-policy.ts +8 -8
- package/src/schedule/run-script.ts +14 -23
- package/src/schedule/schedule-recovery.ts +0 -1
- package/src/schedule/scheduler.ts +89 -51
- package/src/schedule/worker.ts +57 -0
- package/src/security/__tests__/openai-device-auth.test.ts +355 -0
- package/src/security/ces-rpc-record-backend.ts +123 -0
- package/src/security/oauth2.ts +7 -0
- package/src/security/openai-device-auth.ts +364 -0
- package/src/security/secure-keys.ts +15 -0
- package/src/skills/available-skills.ts +10 -1
- package/src/skills/catalog-install.ts +19 -1
- package/src/skills/inline-command-runner.ts +6 -2
- package/src/skills/platform-compatibility.test.ts +49 -0
- package/src/skills/platform-compatibility.ts +65 -0
- package/src/skills/tool-manifest.ts +20 -0
- package/src/stt/__tests__/roles.test.ts +152 -0
- package/src/stt/daemon-batch-transcriber.ts +4 -1
- package/src/stt/roles.ts +180 -0
- package/src/stt/stt-stream-session.ts +3 -2
- package/src/stt/types.ts +33 -3
- package/src/subagent/manager.ts +12 -0
- package/src/telemetry/__tests__/live-voice-funnel.test.ts +34 -0
- package/src/telemetry/live-voice-funnel.ts +12 -2
- package/src/telemetry/telemetry-wire.generated.ts +6 -0
- package/src/telemetry/tool-audit.ts +83 -30
- package/src/telemetry/tool-executed-events-store.test.ts +0 -1
- package/src/telemetry/turn-outcome.ts +75 -4
- package/src/tools/client-os.ts +115 -0
- package/src/tools/computer-use/definitions.ts +11 -7
- package/src/tools/computer-use/skill-proxy-bridge.ts +2 -2
- package/src/tools/credentials/broker-types.ts +0 -71
- package/src/tools/credentials/broker.ts +6 -201
- package/src/tools/credentials/metadata-store.ts +65 -16
- package/src/tools/credentials/store.ts +2 -0
- package/src/tools/executor.ts +2 -8
- package/src/tools/host-shell.test.ts +75 -0
- package/src/tools/host-shell.ts +24 -0
- package/src/tools/host-terminal/host-shell.ts +15 -44
- package/src/tools/mcp/mcp-tool-factory.ts +44 -9
- package/src/tools/permission-checker.ts +73 -21
- package/src/tools/policy-context.ts +1 -0
- package/src/tools/shared/filesystem/image-read.ts +1 -1
- package/src/tools/skills/find-similar-skills.ts +10 -2
- package/src/tools/skills/load.ts +41 -6
- package/src/tools/skills/sandbox-runner.ts +8 -16
- package/src/tools/skills/skill-tool-factory.ts +1 -0
- package/src/tools/terminal/__tests__/safe-env.test.ts +64 -1
- package/src/tools/terminal/safe-env.ts +39 -9
- package/src/tools/terminal/shell.ts +12 -31
- package/src/tools/tool-approval-handler.ts +64 -10
- package/src/tools/tool-types.ts +10 -0
- package/src/tools/types.ts +19 -3
- package/src/tools/ui-surface/surface-shape-docs.ts +4 -3
- package/src/tools/workflows/manage-workflows.ts +2 -2
- package/src/tools/workflows/run-workflow.test.ts +2 -1
- package/src/util/__tests__/file-use.test.ts +61 -0
- package/src/util/__tests__/process-table.test.ts +66 -0
- package/src/util/__tests__/process-tree.test.ts +46 -0
- package/src/util/errors.ts +1 -0
- package/src/util/file-use.ts +91 -0
- package/src/util/host-process.test.ts +17 -0
- package/src/util/host-process.ts +75 -0
- package/src/util/image-conversion.ts +39 -7
- package/src/util/platform.ts +32 -13
- package/src/util/process-table.ts +249 -0
- package/src/util/process-tree.ts +85 -129
- package/src/util/provider-error-patterns.ts +20 -0
- package/src/util/spawn.ts +9 -2
- package/src/watch/__tests__/watch-retro.test.ts +48 -1
- package/src/watch/watch-retro.ts +46 -4
- package/src/workspace/git-service.ts +4 -14
- package/src/workspace/migrations/147-rename-colliding-backup-profile-names.ts +360 -0
- package/src/workspace/migrations/148-strip-unsupported-fallback-profiles.ts +117 -0
- package/src/workspace/migrations/149-repoint-backup-profile-selections.ts +204 -0
- package/src/workspace/migrations/150-stt-flux-provider-to-model-family.ts +139 -0
- package/src/workspace/migrations/__tests__/150-stt-flux-provider-to-model-family.test.ts +160 -0
- package/src/workspace/migrations/registry.ts +8 -0
- package/src/__tests__/permission-types.test.ts +0 -17
- package/src/channels/gateway-discord-admission.ts +0 -32
- package/src/persistence/fork-message-copy.ts +0 -239
- package/src/plugins/defaults/memory/__tests__/fork-message-copy.test.ts +0 -221
- package/src/security/__tests__/oauth2-device-code.test.ts +0 -431
- package/src/security/oauth2-device-code.ts +0 -312
package/src/cli/commands/mcp.ts
CHANGED
|
@@ -169,6 +169,7 @@ function renderOutcome(outcome: MemoryRetrospectiveOutcome): void {
|
|
|
169
169
|
case "invoked":
|
|
170
170
|
log.info(
|
|
171
171
|
`Retrospective invoked.\n` +
|
|
172
|
+
` outcome: ${outcome.noFindings ? "reviewed, nothing new to save" : "saved to memory"}\n` +
|
|
172
173
|
` fork conversation: ${outcome.backgroundConversationId}\n` +
|
|
173
174
|
` cutoff message: ${outcome.cutoffMessageId}\n` +
|
|
174
175
|
` new messages: ${outcome.newMessageCount}` +
|
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
// `import()` has nothing to defer.
|
|
13
13
|
// eslint-disable-next-line cli/no-daemon-internals
|
|
14
14
|
import {
|
|
15
|
-
|
|
15
|
+
listSelectableProviderIds,
|
|
16
16
|
supportsBoundary,
|
|
17
17
|
} from "../../providers/speech-to-text/provider-catalog.js";
|
|
18
18
|
import type { CliCommandHelp } from "../lib/cli-command-help.js";
|
|
@@ -22,7 +22,7 @@ import type { CliCommandHelp } from "../lib/cli-command-help.js";
|
|
|
22
22
|
* providers are omitted: naming one would advertise a configuration that
|
|
23
23
|
* fails on every file.
|
|
24
24
|
*/
|
|
25
|
-
const batchProviders =
|
|
25
|
+
const batchProviders = listSelectableProviderIds()
|
|
26
26
|
.filter((id) => supportsBoundary(id, "daemon-batch"))
|
|
27
27
|
.join(", ");
|
|
28
28
|
|
|
@@ -7,7 +7,7 @@ import {
|
|
|
7
7
|
} from "../schemas/backup.js";
|
|
8
8
|
|
|
9
9
|
describe("BackupConfigSchema", () => {
|
|
10
|
-
test("empty object parses to full defaults
|
|
10
|
+
test("empty object parses to full platform-neutral defaults", () => {
|
|
11
11
|
const parsed = BackupConfigSchema.parse({});
|
|
12
12
|
expect(parsed).toEqual({
|
|
13
13
|
enabled: false,
|
|
@@ -8,10 +8,12 @@ import {
|
|
|
8
8
|
getEffectiveProfile,
|
|
9
9
|
getEffectiveProfiles,
|
|
10
10
|
getEffectiveProfilesForProvider,
|
|
11
|
+
getUserSelectableProfilesForProvider,
|
|
11
12
|
PROFILE_IMPLS,
|
|
12
13
|
resolveDefaultProfileForProvider,
|
|
13
14
|
} from "../default-profile-catalog.js";
|
|
14
15
|
import {
|
|
16
|
+
BACKUP_PROFILE_KEYS,
|
|
15
17
|
DEFAULT_PROFILE_KEYS,
|
|
16
18
|
DEFAULT_PROFILE_PROVIDERS,
|
|
17
19
|
OS_BETA_PROFILE_KEY,
|
|
@@ -78,7 +80,7 @@ describe("getEffectiveProfiles", () => {
|
|
|
78
80
|
test("defaults absent from the workspace resolve from the catalog; os-beta stays flag-gated", () => {
|
|
79
81
|
const effective = getEffectiveProfiles(undefined);
|
|
80
82
|
expect(Object.keys(effective).sort()).toEqual(
|
|
81
|
-
[...DEFAULT_PROFILE_KEYS].sort(),
|
|
83
|
+
[...DEFAULT_PROFILE_KEYS, ...BACKUP_PROFILE_KEYS].sort(),
|
|
82
84
|
);
|
|
83
85
|
expect(getEffectiveProfile({}, "balanced")?.model).toBe(
|
|
84
86
|
CODE_DEFAULT_PROFILE_ENTRIES.balanced.model as string,
|
|
@@ -155,6 +157,25 @@ describe("getEffectiveProfiles", () => {
|
|
|
155
157
|
});
|
|
156
158
|
});
|
|
157
159
|
|
|
160
|
+
describe("user-selectable profile view", () => {
|
|
161
|
+
test("hides managed backups while retaining them in the fallback catalog", () => {
|
|
162
|
+
const effective = getEffectiveProfilesForProvider(undefined, {
|
|
163
|
+
provider: "vellum",
|
|
164
|
+
});
|
|
165
|
+
const selectable = getUserSelectableProfilesForProvider(undefined, {
|
|
166
|
+
provider: "vellum",
|
|
167
|
+
});
|
|
168
|
+
|
|
169
|
+
for (const key of DEFAULT_PROFILE_KEYS) {
|
|
170
|
+
expect(selectable[key]).toBeDefined();
|
|
171
|
+
}
|
|
172
|
+
for (const key of BACKUP_PROFILE_KEYS) {
|
|
173
|
+
expect(effective[key]).toBeDefined();
|
|
174
|
+
expect(selectable[key]).toBeUndefined();
|
|
175
|
+
}
|
|
176
|
+
});
|
|
177
|
+
});
|
|
178
|
+
|
|
158
179
|
describe("resolver integration", () => {
|
|
159
180
|
test("resolution serves default-profile content from the code catalog, not the workspace body", () => {
|
|
160
181
|
// The headline milestone test: with only thin managed stubs on disk,
|
|
@@ -267,6 +288,40 @@ describe("resolver integration", () => {
|
|
|
267
288
|
}),
|
|
268
289
|
).not.toThrow();
|
|
269
290
|
});
|
|
291
|
+
|
|
292
|
+
test("a user profile cannot shadow a managed backup profile", () => {
|
|
293
|
+
// Every backup is code-owned for the same reason `latency-optimized` is:
|
|
294
|
+
// a backup earns its place by pinning a different upstream than the
|
|
295
|
+
// primary it backs, and a user shadow can repoint it at the primary's
|
|
296
|
+
// own provider, so the fallback re-sends into the outage it exists to
|
|
297
|
+
// route around. `latency-optimized-backup` additionally serves the
|
|
298
|
+
// live-voice path whenever that primary falls back.
|
|
299
|
+
for (const key of BACKUP_PROFILE_KEYS) {
|
|
300
|
+
const llm = LLMSchema.parse({
|
|
301
|
+
profiles: {
|
|
302
|
+
[key]: {
|
|
303
|
+
source: "user",
|
|
304
|
+
provider: "anthropic",
|
|
305
|
+
model: "claude-opus-4-6",
|
|
306
|
+
provider_connection: "anthropic-personal",
|
|
307
|
+
maxTokens: 32000,
|
|
308
|
+
effort: "high",
|
|
309
|
+
},
|
|
310
|
+
},
|
|
311
|
+
});
|
|
312
|
+
const body = CODE_DEFAULT_PROFILE_ENTRIES[key]!;
|
|
313
|
+
const effective = getEffectiveProfiles(llm.profiles)[key];
|
|
314
|
+
expect(effective?.model).toBe(body.model);
|
|
315
|
+
expect(effective?.model).not.toBe("claude-opus-4-6");
|
|
316
|
+
expect(String(effective?.provider)).toBe("vellum");
|
|
317
|
+
expect(effective?.provider_connection).toBeUndefined();
|
|
318
|
+
// Resolution through the managed column agrees with the listing.
|
|
319
|
+
const resolved = resolveDefaultProfileForProvider(llm.profiles, key, {
|
|
320
|
+
provider: "vellum",
|
|
321
|
+
});
|
|
322
|
+
expect(resolved?.model).toBe(body.model);
|
|
323
|
+
}
|
|
324
|
+
});
|
|
270
325
|
});
|
|
271
326
|
|
|
272
327
|
describe("schema validation", () => {
|
|
@@ -301,6 +356,27 @@ describe("schema validation", () => {
|
|
|
301
356
|
).toThrow();
|
|
302
357
|
});
|
|
303
358
|
|
|
359
|
+
test("managed backup names are valid references without being materialized", () => {
|
|
360
|
+
// Backups are listed in the effective catalog and are selectable, but
|
|
361
|
+
// selecting one persists only the reference: nothing lands in
|
|
362
|
+
// `llm.profiles`. Unless the schema treats the keys as always-available,
|
|
363
|
+
// the next load strips the selection and silently reverts it.
|
|
364
|
+
for (const key of BACKUP_PROFILE_KEYS) {
|
|
365
|
+
expect(() => LLMSchema.parse({ activeProfile: key })).not.toThrow();
|
|
366
|
+
expect(() => LLMSchema.parse({ advisorProfile: key })).not.toThrow();
|
|
367
|
+
expect(() =>
|
|
368
|
+
LLMSchema.parse({ callSites: { mainAgent: { profile: key } } }),
|
|
369
|
+
).not.toThrow();
|
|
370
|
+
}
|
|
371
|
+
// The keys stay out of `DEFAULT_PROFILE_KEYS`: that array drives picker
|
|
372
|
+
// order and the intent x provider matrix.
|
|
373
|
+
expect(
|
|
374
|
+
BACKUP_PROFILE_KEYS.some((key) =>
|
|
375
|
+
(DEFAULT_PROFILE_KEYS as readonly string[]).includes(key),
|
|
376
|
+
),
|
|
377
|
+
).toBe(false);
|
|
378
|
+
});
|
|
379
|
+
|
|
304
380
|
test("defaultProvider accepts any default-capable API-key provider and drops the rest", () => {
|
|
305
381
|
expect(
|
|
306
382
|
LLMSchema.parse({ defaultProvider: { provider: "together" } })
|
|
@@ -308,7 +384,12 @@ describe("schema validation", () => {
|
|
|
308
384
|
).toEqual({ provider: "together" });
|
|
309
385
|
// Endpoint-supplied and keyless providers have no code-resolvable
|
|
310
386
|
// default profile implementation; the catch drops them atomically.
|
|
311
|
-
for (const provider of [
|
|
387
|
+
for (const provider of [
|
|
388
|
+
"litellm",
|
|
389
|
+
"opencode",
|
|
390
|
+
"openai-compatible",
|
|
391
|
+
"ollama",
|
|
392
|
+
]) {
|
|
312
393
|
expect(
|
|
313
394
|
LLMSchema.parse({ defaultProvider: { provider } }).defaultProvider,
|
|
314
395
|
).toBeUndefined();
|
|
@@ -99,7 +99,7 @@ describe("config recovery compacts arrays after stripping invalid elements", ()
|
|
|
99
99
|
|
|
100
100
|
test("emptied-by-compaction array reverts to a non-empty schema default (backup.offsite.destinations)", () => {
|
|
101
101
|
// `backup.offsite.destinations` defaults to `null`, where `null` means "use
|
|
102
|
-
// the
|
|
102
|
+
// the platform default" and `[]` means "no offsite destinations".
|
|
103
103
|
// Stripping the sole invalid element must NOT leave an explicit `[]` that
|
|
104
104
|
// overrides the default — it must fall back to the schema default `null`.
|
|
105
105
|
writeConfig({
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Guards over the `memory.retrospective` switches that decide whether the
|
|
3
|
-
* retrospective pass runs at all
|
|
3
|
+
* retrospective pass runs at all.
|
|
4
4
|
*
|
|
5
|
-
* The defaults are the contract: a workspace that says nothing must
|
|
6
|
-
*
|
|
7
|
-
*
|
|
5
|
+
* The defaults are the contract: a workspace that says nothing must leave
|
|
6
|
+
* retrospectives on. A leftover `forkStrategy` key is ignored; retrospective
|
|
7
|
+
* forks are always referential.
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
10
|
import { describe, expect, test } from "bun:test";
|
|
@@ -12,10 +12,10 @@ import { describe, expect, test } from "bun:test";
|
|
|
12
12
|
import { MemoryRetrospectiveConfigSchema } from "../schemas/memory-retrospective.js";
|
|
13
13
|
|
|
14
14
|
describe("memory.retrospective config schema", () => {
|
|
15
|
-
test("an empty block leaves retrospectives on
|
|
15
|
+
test("an empty block leaves retrospectives on", () => {
|
|
16
16
|
const parsed = MemoryRetrospectiveConfigSchema.parse({});
|
|
17
17
|
expect(parsed.enabled).toBe(true);
|
|
18
|
-
expect(parsed
|
|
18
|
+
expect(parsed).not.toHaveProperty("forkStrategy");
|
|
19
19
|
});
|
|
20
20
|
|
|
21
21
|
test("enabled is a boolean-only kill switch", () => {
|
|
@@ -27,14 +27,11 @@ describe("memory.retrospective config schema", () => {
|
|
|
27
27
|
).toBe(false);
|
|
28
28
|
});
|
|
29
29
|
|
|
30
|
-
test("
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
).
|
|
35
|
-
expect(
|
|
36
|
-
MemoryRetrospectiveConfigSchema.safeParse({ forkStrategy: "referential" })
|
|
37
|
-
.success,
|
|
38
|
-
).toBe(false);
|
|
30
|
+
test("a leftover forkStrategy key is ignored", () => {
|
|
31
|
+
const parsed = MemoryRetrospectiveConfigSchema.parse({
|
|
32
|
+
forkStrategy: "cloning",
|
|
33
|
+
});
|
|
34
|
+
expect(parsed).not.toHaveProperty("forkStrategy");
|
|
35
|
+
expect(parsed.enabled).toBe(true);
|
|
39
36
|
});
|
|
40
37
|
});
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: computer-use
|
|
3
|
-
description: Control
|
|
3
|
+
description: Control a connected desktop
|
|
4
4
|
compatibility: "Designed for Vellum personal assistants"
|
|
5
5
|
metadata:
|
|
6
6
|
emoji: "🖥️"
|
|
@@ -8,17 +8,17 @@ metadata:
|
|
|
8
8
|
display-name: "Computer Use"
|
|
9
9
|
category: "system"
|
|
10
10
|
activation-hints:
|
|
11
|
-
- "User asks the assistant to click, type,
|
|
11
|
+
- "User asks the assistant to click, type, or interact with the desktop GUI directly"
|
|
12
12
|
- "User wants control of a desktop app with no CLI or API alternative (games, design tools, visual workflows)"
|
|
13
13
|
- "User wants screenshots or visual inspection of what is currently on screen"
|
|
14
14
|
avoid-when:
|
|
15
15
|
- "Task can be done via a more specific skill (gmail, calendar, contacts, terminal-sessions) or a CLI / API call"
|
|
16
16
|
---
|
|
17
17
|
|
|
18
|
-
This skill provides the computer_use_* action tools for controlling
|
|
19
|
-
|
|
18
|
+
This skill provides the computer_use_* action tools for controlling a
|
|
19
|
+
connected desktop. CU tools run through the main agent loop via HostCuProxy.
|
|
20
20
|
|
|
21
21
|
The skill is internally preactivated for conversations with a connected desktop client.
|
|
22
22
|
|
|
23
|
-
Tools in this skill are proxy tools
|
|
24
|
-
|
|
23
|
+
Tools in this skill are proxy tools. Execution is forwarded to a connected
|
|
24
|
+
desktop client and is never handled locally by the assistant.
|
|
@@ -86,7 +86,7 @@
|
|
|
86
86
|
},
|
|
87
87
|
{
|
|
88
88
|
"name": "computer_use_key",
|
|
89
|
-
"description": "Press a key or keyboard shortcut. Supported: enter, tab, escape, backspace, delete, up, down, left, right, space, cmd+a, cmd+c, cmd+v, cmd+z, cmd+tab, cmd+w, shift+tab, option+tab",
|
|
89
|
+
"description": "Press a key or keyboard shortcut. Supported: enter, tab, escape, backspace, delete, up, down, left, right, space, cmd+a, cmd+c, cmd+v, cmd+z, cmd+tab, cmd+w, shift+tab, option+tab. On Windows use ctrl/alt in place of cmd/option.",
|
|
90
90
|
"category": "computer-use",
|
|
91
91
|
"risk": "medium",
|
|
92
92
|
"input_schema": {
|
|
@@ -94,7 +94,7 @@
|
|
|
94
94
|
"properties": {
|
|
95
95
|
"key": {
|
|
96
96
|
"type": "string",
|
|
97
|
-
"description": "Key or shortcut to press (e.g. enter, tab,
|
|
97
|
+
"description": "Key or shortcut to press (e.g. enter, tab, ctrl+c, cmd+c)"
|
|
98
98
|
},
|
|
99
99
|
"reasoning": {
|
|
100
100
|
"type": "string",
|
|
@@ -197,7 +197,8 @@
|
|
|
197
197
|
"required": ["reasoning"]
|
|
198
198
|
},
|
|
199
199
|
"executor": "tools/computer-use-drag.ts",
|
|
200
|
-
"execution_target": "host"
|
|
200
|
+
"execution_target": "host",
|
|
201
|
+
"supported_client_os": ["macos", "windows"]
|
|
201
202
|
},
|
|
202
203
|
{
|
|
203
204
|
"name": "computer_use_wait",
|
|
@@ -227,7 +228,7 @@
|
|
|
227
228
|
},
|
|
228
229
|
{
|
|
229
230
|
"name": "computer_use_open_app",
|
|
230
|
-
"description": "Open or switch to a
|
|
231
|
+
"description": "Open or switch to a desktop application by name. Preferred over cmd+tab / alt+tab for switching apps - more reliable and explicit.",
|
|
231
232
|
"category": "computer-use",
|
|
232
233
|
"risk": "medium",
|
|
233
234
|
"input_schema": {
|
|
@@ -249,7 +250,8 @@
|
|
|
249
250
|
"required": ["app_name", "reasoning"]
|
|
250
251
|
},
|
|
251
252
|
"executor": "tools/computer-use-open-app.ts",
|
|
252
|
-
"execution_target": "host"
|
|
253
|
+
"execution_target": "host",
|
|
254
|
+
"supported_client_os": ["macos", "windows"]
|
|
253
255
|
},
|
|
254
256
|
{
|
|
255
257
|
"name": "computer_use_run_applescript",
|
|
@@ -275,7 +277,8 @@
|
|
|
275
277
|
"required": ["script", "reasoning"]
|
|
276
278
|
},
|
|
277
279
|
"executor": "tools/computer-use-run-applescript.ts",
|
|
278
|
-
"execution_target": "host"
|
|
280
|
+
"execution_target": "host",
|
|
281
|
+
"supported_client_os": ["macos"]
|
|
279
282
|
},
|
|
280
283
|
{
|
|
281
284
|
"name": "computer_use_done",
|
|
@@ -40,7 +40,9 @@ export async function transcribeSegmentAudio(
|
|
|
40
40
|
// Use the provided transcriber or resolve on demand.
|
|
41
41
|
// null = "already resolved, no provider"; only re-resolve when undefined (not passed).
|
|
42
42
|
const resolved =
|
|
43
|
-
transcriber === undefined
|
|
43
|
+
transcriber === undefined
|
|
44
|
+
? await resolveBatchTranscriber({ role: "batch" })
|
|
45
|
+
: transcriber;
|
|
44
46
|
if (!resolved) {
|
|
45
47
|
return "";
|
|
46
48
|
}
|
|
@@ -466,7 +466,7 @@ export async function preprocessForAsset(
|
|
|
466
466
|
let transcriber: BatchTranscriber | null = null;
|
|
467
467
|
if (options.includeAudio) {
|
|
468
468
|
try {
|
|
469
|
-
transcriber = await resolveBatchTranscriber();
|
|
469
|
+
transcriber = await resolveBatchTranscriber({ role: "batch" });
|
|
470
470
|
} catch (err) {
|
|
471
471
|
onProgress?.(
|
|
472
472
|
`Audio transcription unavailable: ${(err as Error).message}\n`,
|
|
@@ -26,7 +26,7 @@ When speaking on behalf of your user during calls, refer to yourself as an "assi
|
|
|
26
26
|
|
|
27
27
|
# Overview
|
|
28
28
|
|
|
29
|
-
The calling system connects both **outbound** and **inbound** voice calls over Twilio Media Streams: the assistant transcribes and synthesizes call audio itself using the
|
|
29
|
+
The calling system connects both **outbound** and **inbound** voice calls over Twilio Media Streams: the assistant transcribes and synthesizes call audio itself using the STT provider telephony selects (`services.stt.roles.telephony` when set, otherwise `services.stt.provider`) and the TTS provider (`services.tts.provider`, default: ElevenLabs), so working API keys for both are required before calls can connect. After Twilio setup, the assistant prompts the user to choose a voice from a curated list of supported options.
|
|
30
30
|
|
|
31
31
|
# Initial Setup
|
|
32
32
|
|
|
@@ -2,21 +2,22 @@
|
|
|
2
2
|
|
|
3
3
|
All call-related settings can be managed via `assistant config`:
|
|
4
4
|
|
|
5
|
-
| Setting | Description
|
|
6
|
-
| ------------------------------------------- |
|
|
7
|
-
| `calls.enabled` | Master switch for the calling feature
|
|
8
|
-
| `calls.provider` | Voice provider (currently only `twilio`)
|
|
9
|
-
| `calls.maxDurationSeconds` | Maximum call length in seconds
|
|
10
|
-
| `calls.userConsultTimeoutSeconds` | How long to wait for user answers
|
|
11
|
-
| `calls.disclosure.enabled` | Whether the AI announces itself at call start
|
|
12
|
-
| `calls.disclosure.text` | The disclosure message spoken at call start
|
|
13
|
-
| `llm.callSites.callAgent.model` | Override LLM model for call orchestration
|
|
14
|
-
| `calls.callerIdentity.allowPerCallOverride` | Allow per-call caller identity selection
|
|
15
|
-
| `calls.callerIdentity.userNumber` | E.164 phone number for user-number mode
|
|
16
|
-
| `services.stt.language` | Spoken language for transcription (BCP-47 code, or unset for the provider's multilingual default). Per-language TTS voices are configured via `services.tts.providers.<id>.languageVoices` (supported on the elevenlabs, deepgram, and vellum provider blocks).
|
|
17
|
-
| `services.stt.provider` | STT provider
|
|
18
|
-
| `services.
|
|
19
|
-
| `services.tts.
|
|
5
|
+
| Setting | Description | Default |
|
|
6
|
+
| ------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------- |
|
|
7
|
+
| `calls.enabled` | Master switch for the calling feature | `true` |
|
|
8
|
+
| `calls.provider` | Voice provider (currently only `twilio`) | `twilio` |
|
|
9
|
+
| `calls.maxDurationSeconds` | Maximum call length in seconds | `3600` (1 hour) |
|
|
10
|
+
| `calls.userConsultTimeoutSeconds` | How long to wait for user answers | `120` (2 min) |
|
|
11
|
+
| `calls.disclosure.enabled` | Whether the AI announces itself at call start | `true` |
|
|
12
|
+
| `calls.disclosure.text` | The disclosure message spoken at call start | `"At the very beginning of the call, introduce yourself as an assistant calling on behalf of my human."` |
|
|
13
|
+
| `llm.callSites.callAgent.model` | Override LLM model for call orchestration | _(unset, falls back to the resolved call-site default)_ |
|
|
14
|
+
| `calls.callerIdentity.allowPerCallOverride` | Allow per-call caller identity selection | `true` |
|
|
15
|
+
| `calls.callerIdentity.userNumber` | E.164 phone number for user-number mode | _(empty)_ |
|
|
16
|
+
| `services.stt.language` | Spoken language for transcription (BCP-47 code, or unset for the provider's multilingual default). Per-language TTS voices are configured via `services.tts.providers.<id>.languageVoices` (supported on the elevenlabs, deepgram, and vellum provider blocks). | _(unset)_ |
|
|
17
|
+
| `services.stt.provider` | Global STT provider, used by telephony when `services.stt.roles.telephony` is unset. The assistant transcribes call audio itself over the Twilio media stream (streaming when the provider supports it, batch otherwise), so calls require a working API key for whichever provider telephony resolves to. | `deepgram` |
|
|
18
|
+
| `services.stt.roles.telephony` | STT provider (and optional `model` family) used for calls, overriding `services.stt.provider` for this consumer only. Calls require a working API key for this provider when it is set. A provider that cannot transcribe calls is rejected when the config is read, not at dial time. | _(unset, falls back to `services.stt.provider`)_ |
|
|
19
|
+
| `services.tts.provider` | Active TTS provider for speech synthesis. Must be a provider ID from the catalog (`elevenlabs`, `fish-audio`, `deepgram`, `xai`). New providers can be added via the catalog without code changes to call routing. | `elevenlabs` |
|
|
20
|
+
| `services.tts.providers.<id>.*` | Provider-specific settings block. Each catalog provider has its own settings namespace under `services.tts.providers.<id>`. See voice settings in the desktop/iOS app or run `assistant config list` for available settings per provider. | _(per-provider defaults)_ |
|
|
20
21
|
|
|
21
22
|
## TTS provider call-path behavior
|
|
22
23
|
|
|
@@ -94,4 +94,5 @@ Load `twilio-setup` and follow its Channel Trust Floors section. Change **Who ca
|
|
|
94
94
|
## Call answers with a spoken "setup required" message and hangs up
|
|
95
95
|
|
|
96
96
|
- The credential preflight found a missing STT or TTS provider credential — the spoken message names what is missing
|
|
97
|
-
- Store the required API key for the
|
|
97
|
+
- Store the required API key for the provider telephony actually resolves to (`services.stt.roles.telephony` when set, otherwise `services.stt.provider`) and for `services.tts.provider`, then retry
|
|
98
|
+
- A key stored for the global provider does not satisfy a `services.stt.roles.telephony` override: the preflight judges the provider the call will dial
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
"tools": [
|
|
4
4
|
{
|
|
5
5
|
"name": "voice_config_update",
|
|
6
|
-
"description": "Update a voice configuration setting. Use tts_provider / stt_provider to switch the active TTS / STT provider. Provider \"vellum\" is Vellum-managed speech (billed to your organization; requires a Vellum platform connection via 'assistant platform connect'); any other provider uses the user's own API key. Valid TTS providers come from the provider catalog (vellum, elevenlabs, fish-audio, deepgram, xai); valid STT providers: vellum, deepgram,
|
|
6
|
+
"description": "Update a voice configuration setting. Use tts_provider / stt_provider to switch the active TTS / STT provider. Provider \"vellum\" is Vellum-managed speech (billed to your organization; requires a Vellum platform connection via 'assistant platform connect'); any other provider uses the user's own API key. Valid TTS providers come from the provider catalog (vellum, elevenlabs, fish-audio, deepgram, xai); valid STT providers: vellum, deepgram, google-gemini, openai-whisper, xai. Use stt_model to pick a provider's model family where it has more than one: deepgram and vellum offer nova-3 (the default, and the only family that transcribes audio files or voice messages) and flux (conversational, streaming-only, with model-native turn detection). Do not select flux when the user wants file transcription. Use tts_voice_id to change the voice; it targets whichever TTS provider is currently active. For elevenlabs, pass an ElevenLabs voice ID. For vellum (managed), pass a managed voice model ID: this may be an ElevenLabs voice ID (managed speech serves the same ElevenLabs voices) or a Deepgram Aura model ID (e.g. aura-2-thalia-en); only rate-carded voices synthesize, so prefer models from the managed catalog (assistant route GET tts/managed-voices, also used by the web voice picker). For deepgram, pass a Deepgram Aura model ID. Use fish_audio_reference_id for Fish Audio voice reference. Use stt_language to set the spoken language for speech recognition: one of the 50 base language codes on the verified Deepgram nova-3 monolingual roster (e.g. en, es, hi, ta, zh, ko), or \"multi\" for code-switching mid-sentence across its 10-language roster (English, Spanish, French, German, Hindi, Russian, Portuguese, Japanese, Italian, Dutch; e.g. Hinglish); plain language names like \"tamil\" or \"multilingual\" are accepted and normalized. Accepted values follow the configured STT provider: vellum-managed and deepgram accept the full roster plus \"multi\"; xai accepts only the 10 multilingual-roster codes (en, es, fr, de, hi, ru, pt, ja, it, nl) and rejects \"multi\" and the extended codes (both are verified for Deepgram nova-3 only); google-gemini and openai-whisper auto-detect natively, so the value persists but is ignored while they are active. Deepgram uses the same API key for TTS and STT, whichever model family is selected. Changes persist to services.stt / services.tts config and take effect immediately.",
|
|
7
7
|
"category": "system",
|
|
8
8
|
"risk": "low",
|
|
9
9
|
"input_schema": {
|
|
@@ -16,14 +16,15 @@
|
|
|
16
16
|
"conversation_timeout",
|
|
17
17
|
"fish_audio_reference_id",
|
|
18
18
|
"stt_language",
|
|
19
|
+
"stt_model",
|
|
19
20
|
"stt_provider",
|
|
20
21
|
"tts_provider",
|
|
21
22
|
"tts_voice_id"
|
|
22
23
|
],
|
|
23
|
-
"description": "The voice setting to change. tts_provider / stt_provider select the active provider for each service (\"vellum\" = Vellum-managed speech; anything else = the user's own API key). tts_voice_id sets the voice for the currently active TTS provider (ElevenLabs voice ID for elevenlabs; a managed voice model ID for vellum, meaning an ElevenLabs voice ID or a Deepgram Aura model ID like aura-2-thalia-en; a Deepgram Aura model ID for deepgram; voice ID for xai). fish_audio_reference_id sets the Fish Audio voice reference. stt_language sets the spoken language for speech recognition (vellum-managed/deepgram: the full roster plus \"multi\"; xai: only the 10 multilingual-roster codes, with \"multi\" and the extended codes rejected; google-gemini and openai-whisper auto-detect and ignore it
|
|
24
|
+
"description": "The voice setting to change. tts_provider / stt_provider select the active provider for each service (\"vellum\" = Vellum-managed speech; anything else = the user's own API key). tts_voice_id sets the voice for the currently active TTS provider (ElevenLabs voice ID for elevenlabs; a managed voice model ID for vellum, meaning an ElevenLabs voice ID or a Deepgram Aura model ID like aura-2-thalia-en; a Deepgram Aura model ID for deepgram; voice ID for xai). fish_audio_reference_id sets the Fish Audio voice reference. stt_model selects a provider's model family where it has more than one (deepgram and vellum: nova-3 or flux). stt_language sets the spoken language for speech recognition (vellum-managed/deepgram: the full roster plus \"multi\"; xai: only the 10 multilingual-roster codes, with \"multi\" and the extended codes rejected; google-gemini and openai-whisper auto-detect and ignore it). Deepgram shares one API key across TTS and STT."
|
|
24
25
|
},
|
|
25
26
|
"value": {
|
|
26
|
-
"description": "The new value for the setting. For tts_provider: one of vellum, elevenlabs, fish-audio, deepgram, xai. For stt_provider: one of vellum, deepgram,
|
|
27
|
+
"description": "The new value for the setting. For tts_provider: one of vellum, elevenlabs, fish-audio, deepgram, xai. For stt_provider: one of vellum, deepgram, google-gemini, openai-whisper, xai. For stt_model: nova-3 or flux, applied to whichever STT provider is currently active; flux serves live speech only and cannot transcribe files. For tts_voice_id: a voice ID for the active TTS provider: an alphanumeric ElevenLabs voice ID (elevenlabs), a managed voice model ID which may be an ElevenLabs voice ID or a Deepgram Aura model ID like aura-2-thalia-en (vellum), or a Deepgram Aura model ID (deepgram). For fish_audio_reference_id: a Fish Audio voice reference ID. For stt_language: one of the 50 base codes on the nova-3 monolingual roster (e.g. en, es, hi, ta, zh, ko), or multi (code-switching across its 10-language roster); language names like \"hindi\", \"tamil\", or \"multilingual\" are also accepted and normalized; when the configured STT provider is xai, only the 10 multilingual-roster codes (en, es, fr, de, hi, ru, pt, ja, it, nl) are accepted. For conversation_timeout: seconds (5, 10, 15, 30, or 60). For activation_key: key identifier string."
|
|
27
28
|
}
|
|
28
29
|
}
|
|
29
30
|
},
|
|
@@ -32,7 +33,7 @@
|
|
|
32
33
|
},
|
|
33
34
|
{
|
|
34
35
|
"name": "open_system_settings",
|
|
35
|
-
"description": "Open a specific macOS
|
|
36
|
+
"description": "Open a specific macOS or Windows system settings pane (e.g. Microphone or Speech Recognition privacy). Use this to guide the user through granting permissions that can only be toggled in system settings. Set platform from the current client_os context.",
|
|
36
37
|
"category": "system",
|
|
37
38
|
"risk": "low",
|
|
38
39
|
"input_schema": {
|
|
@@ -42,9 +43,14 @@
|
|
|
42
43
|
"type": "string",
|
|
43
44
|
"enum": ["microphone", "speech_recognition"],
|
|
44
45
|
"description": "The System Settings pane to open"
|
|
46
|
+
},
|
|
47
|
+
"platform": {
|
|
48
|
+
"type": "string",
|
|
49
|
+
"enum": ["macos", "windows"],
|
|
50
|
+
"description": "The desktop platform from the current client_os context"
|
|
45
51
|
}
|
|
46
52
|
},
|
|
47
|
-
"required": ["pane"]
|
|
53
|
+
"required": ["pane", "platform"]
|
|
48
54
|
},
|
|
49
55
|
"executor": "tools/open-system-settings.ts",
|
|
50
56
|
"execution_target": "host"
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import { beforeEach, describe, expect, mock, test } from "bun:test";
|
|
2
|
+
|
|
3
|
+
import type { ToolContext } from "../../../../tools/types.js";
|
|
4
|
+
|
|
5
|
+
let windowsHost = true;
|
|
6
|
+
|
|
7
|
+
mock.module("../../../../util/platform.js", () => ({
|
|
8
|
+
isWindows: () => windowsHost,
|
|
9
|
+
}));
|
|
10
|
+
|
|
11
|
+
const { run } = await import("./open-system-settings.js");
|
|
12
|
+
|
|
13
|
+
let sentMessages: Array<{ type: string; [key: string]: unknown }> = [];
|
|
14
|
+
|
|
15
|
+
function makeContext(): ToolContext {
|
|
16
|
+
return {
|
|
17
|
+
workingDir: "/tmp",
|
|
18
|
+
conversationId: "conv-xyz",
|
|
19
|
+
trustClass: "guardian",
|
|
20
|
+
sendToClient: (message) => {
|
|
21
|
+
sentMessages.push(message);
|
|
22
|
+
},
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
describe("open_system_settings tool", () => {
|
|
27
|
+
beforeEach(() => {
|
|
28
|
+
sentMessages = [];
|
|
29
|
+
windowsHost = true;
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
test("opens Windows microphone privacy settings", async () => {
|
|
33
|
+
const result = await run(
|
|
34
|
+
{ pane: "microphone", platform: "windows" },
|
|
35
|
+
makeContext(),
|
|
36
|
+
);
|
|
37
|
+
|
|
38
|
+
expect(result).toEqual({
|
|
39
|
+
content:
|
|
40
|
+
"Opened Windows Settings to Microphone privacy. Please enable Vellum Assistant.",
|
|
41
|
+
isError: false,
|
|
42
|
+
});
|
|
43
|
+
expect(sentMessages).toEqual([
|
|
44
|
+
{
|
|
45
|
+
type: "open_url",
|
|
46
|
+
url: "ms-settings:privacy-microphone",
|
|
47
|
+
conversationId: "conv-xyz",
|
|
48
|
+
},
|
|
49
|
+
]);
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
test("opens macOS speech recognition settings", async () => {
|
|
53
|
+
const result = await run(
|
|
54
|
+
{ pane: "speech_recognition", platform: "macos" },
|
|
55
|
+
makeContext(),
|
|
56
|
+
);
|
|
57
|
+
|
|
58
|
+
expect(result.isError).toBe(false);
|
|
59
|
+
expect(sentMessages).toEqual([
|
|
60
|
+
{
|
|
61
|
+
type: "open_url",
|
|
62
|
+
url: "x-apple.systempreferences:com.apple.preference.security?Privacy_SpeechRecognition",
|
|
63
|
+
conversationId: "conv-xyz",
|
|
64
|
+
},
|
|
65
|
+
]);
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
test("uses the host platform when the client platform is absent", async () => {
|
|
69
|
+
const result = await run({ pane: "microphone" }, makeContext());
|
|
70
|
+
|
|
71
|
+
expect(result.isError).toBe(false);
|
|
72
|
+
expect(sentMessages[0]?.url).toBe("ms-settings:privacy-microphone");
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
test("rejects an unsupported platform", async () => {
|
|
76
|
+
const result = await run(
|
|
77
|
+
{ pane: "microphone", platform: "linux" },
|
|
78
|
+
makeContext(),
|
|
79
|
+
);
|
|
80
|
+
|
|
81
|
+
expect(result.isError).toBe(true);
|
|
82
|
+
expect(result.content).toContain('unknown platform "linux"');
|
|
83
|
+
expect(sentMessages).toEqual([]);
|
|
84
|
+
});
|
|
85
|
+
});
|