@vellumai/assistant 0.3.4 → 0.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Dockerfile +2 -0
- package/README.md +88 -2
- package/eslint.config.mjs +31 -0
- package/package.json +1 -1
- package/scripts/ipc/check-swift-decoder-drift.ts +4 -1
- package/scripts/ipc/generate-swift.ts +31 -2
- package/src/__tests__/__snapshots__/ipc-snapshot.test.ts.snap +438 -1
- package/src/__tests__/approval-conversation-turn.test.ts +214 -0
- package/src/__tests__/approval-hardcoded-copy-guard.test.ts +41 -0
- package/src/__tests__/approval-message-composer.test.ts +253 -0
- package/src/__tests__/browser-manager.test.ts +1 -0
- package/src/__tests__/call-conversation-messages.test.ts +130 -0
- package/src/__tests__/call-domain.test.ts +12 -2
- package/src/__tests__/call-orchestrator.test.ts +799 -249
- package/src/__tests__/call-pointer-messages.test.ts +148 -0
- package/src/__tests__/call-recovery.test.ts +3 -0
- package/src/__tests__/call-routes-http.test.ts +32 -2
- package/src/__tests__/call-store.test.ts +3 -0
- package/src/__tests__/channel-approval-routes.test.ts +1277 -98
- package/src/__tests__/channel-approval.test.ts +37 -0
- package/src/__tests__/channel-approvals.test.ts +36 -50
- package/src/__tests__/channel-guardian.test.ts +630 -22
- package/src/__tests__/channel-readiness-service.test.ts +324 -0
- package/src/__tests__/checker.test.ts +14 -7
- package/src/__tests__/clarification-resolver.test.ts +44 -24
- package/src/__tests__/commit-message-enrichment-service.test.ts +9 -4
- package/src/__tests__/computer-use-session-working-dir.test.ts +8 -0
- package/src/__tests__/config-schema.test.ts +14 -8
- package/src/__tests__/context-window-manager.test.ts +30 -2
- package/src/__tests__/contradiction-checker.test.ts +20 -5
- package/src/__tests__/credential-security-invariants.test.ts +7 -2
- package/src/__tests__/daemon-lifecycle.test.ts +13 -12
- package/src/__tests__/db-migration-rollback.test.ts +752 -0
- package/src/__tests__/dictation-mode-detection.test.ts +63 -0
- package/src/__tests__/dynamic-skill-workflow-prompt.test.ts +2 -0
- package/src/__tests__/entity-search.test.ts +615 -0
- package/src/__tests__/fuzzy-match-property.test.ts +5 -5
- package/src/__tests__/guardian-action-store.test.ts +123 -0
- package/src/__tests__/guardian-action-sweep.test.ts +277 -0
- package/src/__tests__/guardian-dispatch.test.ts +389 -0
- package/src/__tests__/guardian-question-copy.test.ts +47 -0
- package/src/__tests__/handlers-telegram-config.test.ts +4 -2
- package/src/__tests__/handlers-twilio-config.test.ts +533 -0
- package/src/__tests__/intent-routing.test.ts +2 -0
- package/src/__tests__/ipc-snapshot.test.ts +291 -1
- package/src/__tests__/memory-upsert-concurrency.test.ts +828 -0
- package/src/__tests__/messaging-send-tool.test.ts +65 -0
- package/src/__tests__/model-intents.test.ts +96 -0
- package/src/__tests__/no-direct-anthropic-sdk-imports.test.ts +42 -0
- package/src/__tests__/oauth2-gateway-transport.test.ts +130 -0
- package/src/__tests__/onboarding-starter-tasks.test.ts +2 -0
- package/src/__tests__/provider-commit-message-generator.test.ts +89 -13
- package/src/__tests__/provider-error-scenarios.test.ts +621 -0
- package/src/__tests__/provider-fail-open-selection.test.ts +119 -0
- package/src/__tests__/qdrant-manager.test.ts +27 -20
- package/src/__tests__/relay-server.test.ts +779 -40
- package/src/__tests__/run-orchestrator-assistant-events.test.ts +6 -0
- package/src/__tests__/run-orchestrator.test.ts +42 -4
- package/src/__tests__/runtime-runs-http.test.ts +17 -1
- package/src/__tests__/runtime-runs.test.ts +16 -0
- package/src/__tests__/schedule-store.test.ts +18 -4
- package/src/__tests__/scheduler-recurrence.test.ts +13 -4
- package/src/__tests__/session-abort-tool-results.test.ts +6 -0
- package/src/__tests__/session-agent-loop.test.ts +857 -0
- package/src/__tests__/session-conflict-gate.test.ts +6 -0
- package/src/__tests__/session-pre-run-repair.test.ts +6 -0
- package/src/__tests__/session-profile-injection.test.ts +6 -0
- package/src/__tests__/session-provider-retry-repair.test.ts +6 -0
- package/src/__tests__/session-queue.test.ts +6 -0
- package/src/__tests__/session-runtime-assembly.test.ts +321 -13
- package/src/__tests__/session-slash-known.test.ts +6 -0
- package/src/__tests__/session-slash-queue.test.ts +6 -0
- package/src/__tests__/session-slash-unknown.test.ts +6 -0
- package/src/__tests__/session-surfaces-task-progress.test.ts +2 -0
- package/src/__tests__/session-tool-setup-app-refresh.test.ts +1 -0
- package/src/__tests__/session-tool-setup-memory-scope.test.ts +1 -0
- package/src/__tests__/session-tool-setup-side-effect-flag.test.ts +1 -0
- package/src/__tests__/session-workspace-injection.test.ts +6 -0
- package/src/__tests__/session-workspace-tool-tracking.test.ts +6 -0
- package/src/__tests__/skills.test.ts +2 -0
- package/src/__tests__/sms-messaging-provider.test.ts +126 -0
- package/src/__tests__/starter-task-flow.test.ts +2 -0
- package/src/__tests__/swarm-dag-pathological.test.ts +535 -0
- package/src/__tests__/system-prompt.test.ts +2 -0
- package/src/__tests__/task-management-tools.test.ts +2 -2
- package/src/__tests__/task-runner.test.ts +14 -4
- package/src/__tests__/terminal-tools.test.ts +25 -19
- package/src/__tests__/tool-execution-abort-cleanup.test.ts +545 -0
- package/src/__tests__/tool-executor-shell-integration.test.ts +11 -11
- package/src/__tests__/tool-executor.test.ts +23 -24
- package/src/__tests__/trust-store.test.ts +3 -3
- package/src/__tests__/twilio-rest.test.ts +29 -0
- package/src/__tests__/twilio-routes-elevenlabs.test.ts +3 -0
- package/src/__tests__/twilio-routes-twiml.test.ts +11 -0
- package/src/__tests__/twilio-routes.test.ts +167 -11
- package/src/__tests__/twitter-cli-error-shaping.test.ts +2 -2
- package/src/__tests__/user-reference.test.ts +2 -0
- package/src/__tests__/voice-quality.test.ts +222 -0
- package/src/__tests__/web-search.test.ts +46 -30
- package/src/__tests__/work-item-output.test.ts +110 -0
- package/src/agent/loop.ts +1 -1
- package/src/agent-heartbeat/agent-heartbeat-service.ts +2 -10
- package/src/amazon/client.ts +1418 -0
- package/src/amazon/request-extractor.ts +135 -0
- package/src/amazon/session.ts +109 -0
- package/src/autonomy/autonomy-store.ts +5 -5
- package/src/browser-extension-relay/client.ts +124 -0
- package/src/browser-extension-relay/protocol.ts +63 -0
- package/src/browser-extension-relay/server.ts +177 -0
- package/src/bundler/app-bundler.ts +3 -3
- package/src/bundler/bundle-signer.ts +1 -1
- package/src/bundler/signature-verifier.ts +1 -1
- package/src/calls/call-conversation-messages.ts +33 -0
- package/src/calls/call-domain.ts +114 -10
- package/src/calls/call-orchestrator.ts +268 -59
- package/src/calls/call-pointer-messages.ts +53 -0
- package/src/calls/call-recovery.ts +3 -8
- package/src/calls/call-store.ts +69 -87
- package/src/calls/elevenlabs-config.ts +3 -2
- package/src/calls/guardian-action-sweep.ts +105 -0
- package/src/calls/guardian-dispatch.ts +203 -0
- package/src/calls/guardian-question-copy.ts +133 -0
- package/src/calls/relay-server.ts +466 -8
- package/src/calls/speaker-identification.ts +1 -1
- package/src/calls/twilio-config.ts +22 -14
- package/src/calls/twilio-provider.ts +6 -4
- package/src/calls/twilio-rest.ts +308 -7
- package/src/calls/twilio-routes.ts +65 -12
- package/src/calls/types.ts +3 -1
- package/src/channels/types.ts +25 -0
- package/src/cli/amazon.ts +815 -0
- package/src/cli/config-commands.ts +2 -2
- package/src/cli/core-commands.ts +4 -3
- package/src/cli/influencer.ts +244 -0
- package/src/cli/map.ts +89 -6
- package/src/cli.ts +1 -1
- package/src/config/agent-schema.ts +171 -0
- package/src/config/bundled-skills/amazon/SKILL.md +127 -0
- package/src/config/bundled-skills/amazon/icon.svg +13 -0
- package/src/config/bundled-skills/api-mapping/SKILL.md +78 -0
- package/src/config/bundled-skills/browser/SKILL.md +1 -0
- package/src/config/bundled-skills/browser/TOOLS.json +17 -0
- package/src/config/bundled-skills/browser/tools/browser-wait-for-download.ts +25 -0
- package/src/config/bundled-skills/doordash/SKILL.md +51 -51
- package/src/config/bundled-skills/email-setup/SKILL.md +14 -5
- package/src/config/bundled-skills/google-oauth-setup/SKILL.md +183 -0
- package/src/config/bundled-skills/influencer/SKILL.md +144 -0
- package/src/config/bundled-skills/knowledge-graph/SKILL.md +15 -0
- package/src/config/bundled-skills/knowledge-graph/TOOLS.json +56 -0
- package/src/config/bundled-skills/knowledge-graph/tools/graph-query.ts +185 -0
- package/src/config/bundled-skills/macos-automation/icon.svg +12 -0
- package/src/config/bundled-skills/media-processing/SKILL.md +176 -0
- package/src/config/bundled-skills/media-processing/TOOLS.json +230 -0
- package/src/config/bundled-skills/media-processing/__tests__/concurrency-pool.test.ts +77 -0
- package/src/config/bundled-skills/media-processing/__tests__/cost-tracker.test.ts +69 -0
- package/src/config/bundled-skills/media-processing/__tests__/preprocess.test.ts +303 -0
- package/src/config/bundled-skills/media-processing/services/concurrency-pool.ts +55 -0
- package/src/config/bundled-skills/media-processing/services/cost-tracker.ts +86 -0
- package/src/config/bundled-skills/media-processing/services/gemini-map.ts +339 -0
- package/src/config/bundled-skills/media-processing/services/preprocess.ts +551 -0
- package/src/config/bundled-skills/media-processing/services/processing-pipeline.ts +259 -0
- package/src/config/bundled-skills/media-processing/services/reduce.ts +197 -0
- package/src/config/bundled-skills/media-processing/tools/analyze-keyframes.ts +136 -0
- package/src/config/bundled-skills/media-processing/tools/extract-keyframes.ts +59 -0
- package/src/config/bundled-skills/media-processing/tools/generate-clip.ts +195 -0
- package/src/config/bundled-skills/media-processing/tools/ingest-media.ts +197 -0
- package/src/config/bundled-skills/media-processing/tools/media-diagnostics.ts +143 -0
- package/src/config/bundled-skills/media-processing/tools/media-status.ts +75 -0
- package/src/config/bundled-skills/media-processing/tools/query-media-events.ts +65 -0
- package/src/config/bundled-skills/messaging/SKILL.md +33 -8
- package/src/config/bundled-skills/messaging/tools/messaging-analyze-style.ts +4 -7
- package/src/config/bundled-skills/messaging/tools/messaging-reply.ts +2 -1
- package/src/config/bundled-skills/messaging/tools/messaging-send.ts +5 -1
- package/src/config/bundled-skills/phone-calls/SKILL.md +88 -23
- package/src/config/bundled-skills/twitter/SKILL.md +19 -3
- package/src/config/bundled-skills/twitter/icon.svg +14 -0
- package/src/config/bundled-tool-registry.ts +310 -0
- package/src/config/calls-schema.ts +181 -0
- package/src/config/core-schema.ts +309 -0
- package/src/config/defaults.ts +28 -3
- package/src/config/env-registry.ts +162 -0
- package/src/config/env.ts +175 -0
- package/src/config/loader.ts +6 -6
- package/src/config/memory-schema.ts +528 -0
- package/src/config/sandbox-schema.ts +55 -0
- package/src/config/schema.ts +158 -1133
- package/src/config/skill-state.ts +1 -1
- package/src/config/skills-schema.ts +32 -0
- package/src/config/skills.ts +35 -24
- package/src/config/system-prompt.ts +131 -56
- package/src/config/templates/IDENTITY.md +2 -2
- package/src/config/templates/SOUL.md +1 -1
- package/src/config/types.ts +1 -0
- package/src/config/user-reference.ts +4 -9
- package/src/config/vellum-skills/catalog.json +6 -7
- package/src/config/vellum-skills/chatgpt-import/tools/chatgpt-import.ts +5 -1
- package/src/config/vellum-skills/slack-oauth-setup/SKILL.md +4 -3
- package/src/config/vellum-skills/sms-setup/SKILL.md +216 -0
- package/src/config/vellum-skills/twilio-setup/SKILL.md +40 -8
- package/src/context/window-manager.ts +27 -7
- package/src/daemon/approval-generators.ts +186 -0
- package/src/daemon/approved-devices-store.ts +140 -0
- package/src/daemon/assistant-attachments.ts +1 -1
- package/src/daemon/classifier.ts +35 -32
- package/src/daemon/config-watcher.ts +1 -1
- package/src/daemon/daemon-control.ts +217 -0
- package/src/daemon/handlers/apps.ts +2 -3
- package/src/daemon/handlers/config-channels.ts +158 -0
- package/src/daemon/handlers/config-inbox.ts +540 -0
- package/src/daemon/handlers/config-ingress.ts +231 -0
- package/src/daemon/handlers/config-integrations.ts +258 -0
- package/src/daemon/handlers/config-model.ts +143 -0
- package/src/daemon/handlers/config-parental.ts +163 -0
- package/src/daemon/handlers/config-scheduling.ts +172 -0
- package/src/daemon/handlers/config-slack.ts +92 -0
- package/src/daemon/handlers/config-telegram.ts +301 -0
- package/src/daemon/handlers/config-tools.ts +177 -0
- package/src/daemon/handlers/config-trust.ts +104 -0
- package/src/daemon/handlers/config-twilio.ts +1080 -0
- package/src/daemon/handlers/config.ts +53 -1689
- package/src/daemon/handlers/diagnostics.ts +1 -1
- package/src/daemon/handlers/dictation.ts +180 -0
- package/src/daemon/handlers/documents.ts +18 -32
- package/src/daemon/handlers/identity.ts +14 -23
- package/src/daemon/handlers/index.ts +11 -0
- package/src/daemon/handlers/misc.ts +3 -5
- package/src/daemon/handlers/pairing.ts +98 -0
- package/src/daemon/handlers/sessions.ts +56 -5
- package/src/daemon/handlers/shared.ts +6 -1
- package/src/daemon/handlers/skills.ts +1 -1
- package/src/daemon/handlers/twitter-auth.ts +2 -0
- package/src/daemon/handlers/work-items.ts +17 -9
- package/src/daemon/handlers/workspace-files.ts +4 -3
- package/src/daemon/install-cli-launchers.ts +113 -0
- package/src/daemon/ipc-contract/apps.ts +356 -0
- package/src/daemon/ipc-contract/browser.ts +74 -0
- package/src/daemon/ipc-contract/computer-use.ts +151 -0
- package/src/daemon/ipc-contract/diagnostics.ts +56 -0
- package/src/daemon/ipc-contract/documents.ts +74 -0
- package/src/daemon/ipc-contract/inbox.ts +209 -0
- package/src/daemon/ipc-contract/integrations.ts +284 -0
- package/src/daemon/ipc-contract/memory.ts +48 -0
- package/src/daemon/ipc-contract/messages.ts +211 -0
- package/src/daemon/ipc-contract/pairing.ts +45 -0
- package/src/daemon/ipc-contract/parental-control.ts +95 -0
- package/src/daemon/ipc-contract/schedules.ts +97 -0
- package/src/daemon/ipc-contract/sessions.ts +315 -0
- package/src/daemon/ipc-contract/shared.ts +42 -0
- package/src/daemon/ipc-contract/skills.ts +120 -0
- package/src/daemon/ipc-contract/subagents.ts +58 -0
- package/src/daemon/ipc-contract/surfaces.ts +250 -0
- package/src/daemon/ipc-contract/trust.ts +60 -0
- package/src/daemon/ipc-contract/work-items.ts +225 -0
- package/src/daemon/ipc-contract/workspace.ts +113 -0
- package/src/daemon/ipc-contract-inventory.json +70 -0
- package/src/daemon/ipc-contract-inventory.ts +55 -29
- package/src/daemon/ipc-contract.ts +229 -2426
- package/src/daemon/ipc-protocol.ts +1 -1
- package/src/daemon/ipc-validate.ts +7 -0
- package/src/daemon/lifecycle.ts +97 -377
- package/src/daemon/pairing-store.ts +177 -0
- package/src/daemon/providers-setup.ts +43 -0
- package/src/daemon/ride-shotgun-handler.ts +68 -3
- package/src/daemon/server.ts +66 -46
- package/src/daemon/session-agent-loop-handlers.ts +421 -0
- package/src/daemon/session-agent-loop.ts +117 -275
- package/src/daemon/session-dynamic-profile.ts +1 -1
- package/src/daemon/session-history.ts +1 -1
- package/src/daemon/session-media-retry.ts +1 -1
- package/src/daemon/session-messaging.ts +37 -2
- package/src/daemon/session-notifiers.ts +5 -25
- package/src/daemon/session-process.ts +99 -59
- package/src/daemon/session-queue-manager.ts +96 -4
- package/src/daemon/session-runtime-assembly.ts +199 -10
- package/src/daemon/session-surfaces.ts +19 -4
- package/src/daemon/session-tool-setup.ts +30 -30
- package/src/daemon/session-workspace.ts +1 -1
- package/src/daemon/session.ts +35 -2
- package/src/daemon/shutdown-handlers.ts +122 -0
- package/src/daemon/trace-emitter.ts +1 -1
- package/src/daemon/watch-handler.ts +36 -33
- package/src/doordash/cart-queries.ts +787 -0
- package/src/doordash/client.ts +144 -127
- package/src/doordash/order-queries.ts +85 -0
- package/src/doordash/queries.ts +10 -1308
- package/src/doordash/search-queries.ts +203 -0
- package/src/doordash/session.ts +3 -2
- package/src/doordash/store-queries.ts +246 -0
- package/src/doordash/types.ts +367 -0
- package/src/email/providers/agentmail.ts +2 -1
- package/src/email/providers/index.ts +3 -2
- package/src/email/service.ts +3 -2
- package/src/errors.ts +43 -0
- package/src/home-base/prebuilt/seed.ts +1 -1
- package/src/hooks/cli.ts +6 -5
- package/src/hooks/config.ts +6 -8
- package/src/hooks/discovery.ts +6 -5
- package/src/hooks/manager.ts +4 -3
- package/src/hooks/runner.ts +2 -2
- package/src/hooks/templates.ts +5 -5
- package/src/inbound/public-ingress-urls.ts +6 -4
- package/src/index.ts +4 -2
- package/src/influencer/client.ts +1104 -0
- package/src/instrument.ts +4 -3
- package/src/logfire.ts +4 -3
- package/src/memory/admin.ts +25 -35
- package/src/memory/attachments-store.ts +4 -7
- package/src/memory/channel-delivery-store.ts +30 -1
- package/src/memory/channel-guardian-store.ts +202 -2
- package/src/memory/clarification-resolver.ts +37 -33
- package/src/memory/conflict-store.ts +67 -61
- package/src/memory/contradiction-checker.ts +141 -117
- package/src/memory/conversation-store.ts +335 -51
- package/src/memory/db-connection.ts +27 -4
- package/src/memory/db-init.ts +265 -4
- package/src/memory/db.ts +14 -1
- package/src/memory/embedding-backend.ts +27 -5
- package/src/memory/embedding-ollama.ts +2 -1
- package/src/memory/entity-extractor.ts +38 -35
- package/src/memory/guardian-action-store.ts +430 -0
- package/src/memory/inbox-escalation-projection.ts +59 -0
- package/src/memory/inbox-thread-store.ts +218 -0
- package/src/memory/ingress-invite-store.ts +338 -0
- package/src/memory/ingress-member-store.ts +350 -0
- package/src/memory/items-extractor.ts +91 -97
- package/src/memory/job-handlers/index-maintenance.ts +3 -3
- package/src/memory/job-handlers/media-processing.ts +69 -0
- package/src/memory/job-handlers/summarization.ts +32 -26
- package/src/memory/job-utils.ts +3 -10
- package/src/memory/jobs-store.ts +8 -10
- package/src/memory/jobs-worker.ts +55 -36
- package/src/memory/media-store.ts +759 -0
- package/src/memory/migrations/001-job-deferrals.ts +45 -0
- package/src/memory/migrations/002-tool-invocations-fk.ts +43 -0
- package/src/memory/migrations/003-memory-fts-backfill.ts +24 -0
- package/src/memory/migrations/004-entity-relation-dedup.ts +87 -0
- package/src/memory/migrations/005-fingerprint-scope-unique.ts +80 -0
- package/src/memory/migrations/006-scope-salted-fingerprints.ts +62 -0
- package/src/memory/migrations/007-assistant-id-to-self.ts +254 -0
- package/src/memory/migrations/008-remove-assistant-id-columns.ts +208 -0
- package/src/memory/migrations/009-llm-usage-events-drop-assistant-id.ts +83 -0
- package/src/memory/migrations/010-ext-conv-bindings-channel-chat-unique.ts +56 -0
- package/src/memory/migrations/011-call-sessions-provider-sid-dedup.ts +63 -0
- package/src/memory/migrations/012-call-sessions-add-initiated-from.ts +19 -0
- package/src/memory/migrations/013-guardian-action-tables.ts +68 -0
- package/src/memory/migrations/014-backfill-inbox-thread-state.ts +76 -0
- package/src/memory/migrations/015-drop-active-search-index.ts +27 -0
- package/src/memory/migrations/016-memory-segments-indexes.ts +11 -0
- package/src/memory/migrations/017-memory-items-indexes.ts +10 -0
- package/src/memory/migrations/018-remaining-table-indexes.ts +13 -0
- package/src/memory/migrations/index.ts +24 -0
- package/src/memory/migrations/registry.ts +79 -0
- package/src/memory/migrations/validate-migration-state.ts +69 -0
- package/src/memory/qdrant-manager.ts +49 -8
- package/src/memory/query-builder.ts +1 -1
- package/src/memory/raw-query.ts +119 -0
- package/src/memory/recall-cache.ts +4 -1
- package/src/memory/retriever.ts +165 -47
- package/src/memory/schema-migration.ts +25 -984
- package/src/memory/schema.ts +228 -7
- package/src/memory/search/entity.ts +205 -31
- package/src/memory/search/lexical.ts +81 -52
- package/src/memory/search/ranking.ts +27 -23
- package/src/memory/search/semantic.ts +157 -19
- package/src/memory/search/types.ts +24 -0
- package/src/memory/shared-app-links-store.ts +4 -5
- package/src/memory/validation.ts +19 -0
- package/src/messaging/draft-store.ts +5 -6
- package/src/messaging/provider-types.ts +2 -0
- package/src/messaging/providers/sms/adapter.ts +201 -0
- package/src/messaging/providers/sms/client.ts +93 -0
- package/src/messaging/providers/sms/types.ts +7 -0
- package/src/messaging/providers/telegram-bot/adapter.ts +2 -5
- package/src/messaging/providers/whatsapp/adapter.ts +136 -0
- package/src/messaging/providers/whatsapp/client.ts +67 -0
- package/src/messaging/style-analyzer.ts +5 -4
- package/src/messaging/thread-summarizer.ts +61 -69
- package/src/messaging/triage-engine.ts +62 -71
- package/src/migrations/config-merge.ts +53 -0
- package/src/migrations/data-layout.ts +68 -0
- package/src/migrations/data-merge.ts +33 -0
- package/src/migrations/hooks-merge.ts +90 -0
- package/src/migrations/index.ts +6 -0
- package/src/migrations/log.ts +23 -0
- package/src/migrations/skills-merge.ts +33 -0
- package/src/migrations/workspace-layout.ts +79 -0
- package/src/permissions/checker.ts +133 -11
- package/src/permissions/prompter.ts +14 -0
- package/src/permissions/shell-identity.ts +31 -1
- package/src/permissions/trust-store.ts +21 -1
- package/src/providers/anthropic/client.ts +4 -4
- package/src/providers/failover.ts +2 -2
- package/src/providers/model-intents.ts +70 -0
- package/src/providers/ollama/client.ts +2 -1
- package/src/providers/provider-send-message.ts +176 -0
- package/src/providers/registry.ts +71 -30
- package/src/providers/retry.ts +35 -1
- package/src/providers/types.ts +12 -1
- package/src/runtime/approval-conversation-turn.ts +97 -0
- package/src/runtime/approval-message-composer.ts +253 -0
- package/src/runtime/channel-approval-parser.ts +36 -2
- package/src/runtime/channel-approvals.ts +11 -24
- package/src/runtime/channel-guardian-service.ts +88 -21
- package/src/runtime/channel-readiness-service.ts +418 -0
- package/src/runtime/channel-readiness-types.ts +35 -0
- package/src/runtime/channel-retry-sweep.ts +184 -0
- package/src/runtime/guardian-context-resolver.ts +108 -0
- package/src/runtime/http-server.ts +275 -717
- package/src/runtime/http-types.ts +59 -3
- package/src/runtime/middleware/auth.ts +116 -0
- package/src/runtime/middleware/error-handler.ts +33 -0
- package/src/runtime/middleware/twilio-validation.ts +127 -0
- package/src/runtime/routes/app-routes.ts +1 -1
- package/src/runtime/routes/call-routes.ts +51 -7
- package/src/runtime/routes/channel-delivery-routes.ts +170 -0
- package/src/runtime/routes/channel-guardian-routes.ts +1191 -0
- package/src/runtime/routes/channel-inbound-routes.ts +1152 -0
- package/src/runtime/routes/channel-route-shared.ts +144 -0
- package/src/runtime/routes/channel-routes.ts +32 -1588
- package/src/runtime/routes/conversation-routes.ts +50 -7
- package/src/runtime/routes/events-routes.ts +2 -2
- package/src/runtime/routes/identity-routes.ts +126 -0
- package/src/runtime/routes/pairing-routes.ts +143 -0
- package/src/runtime/routes/run-routes.ts +15 -1
- package/src/runtime/run-orchestrator.ts +86 -35
- package/src/schedule/schedule-store.ts +36 -32
- package/src/schedule/scheduler.ts +3 -3
- package/src/security/encrypted-store.ts +5 -7
- package/src/security/oauth2.ts +45 -15
- package/src/security/parental-control-store.ts +183 -0
- package/src/security/secret-allowlist.ts +4 -3
- package/src/security/secret-scanner.ts +5 -5
- package/src/security/secure-keys.ts +1 -1
- package/src/security/token-manager.ts +3 -2
- package/src/services/vercel-deploy.ts +6 -2
- package/src/skills/tool-manifest.ts +3 -3
- package/src/skills/vellum-catalog-remote.ts +75 -16
- package/src/slack/slack-webhook.ts +2 -1
- package/src/swarm/orchestrator.ts +92 -1
- package/src/swarm/router-planner.ts +6 -9
- package/src/swarm/worker-prompts.ts +9 -12
- package/src/tasks/task-compiler.ts +19 -28
- package/src/tasks/task-runner.ts +1 -1
- package/src/tools/assets/materialize.ts +2 -2
- package/src/tools/assets/search.ts +15 -14
- package/src/tools/browser/__tests__/auth-detector.test.ts +1 -0
- package/src/tools/browser/auto-navigate.ts +1 -0
- package/src/tools/browser/browser-execution.ts +10 -1
- package/src/tools/browser/browser-manager.ts +119 -4
- package/src/tools/browser/network-recorder.ts +5 -0
- package/src/tools/calls/call-start.ts +1 -0
- package/src/tools/credentials/broker.ts +11 -2
- package/src/tools/credentials/metadata-store.ts +18 -14
- package/src/tools/credentials/post-connect-hooks.ts +61 -0
- package/src/tools/credentials/vault.ts +49 -23
- package/src/tools/execution-target.ts +11 -1
- package/src/tools/executor.ts +68 -9
- package/src/tools/host-terminal/cli-discover.ts +1 -1
- package/src/tools/network/script-proxy/http-forwarder.ts +1 -1
- package/src/tools/network/script-proxy/mitm-handler.ts +1 -1
- package/src/tools/network/script-proxy/server.ts +1 -1
- package/src/tools/network/script-proxy/session-manager.ts +6 -5
- package/src/tools/network/web-fetch.ts +18 -2
- package/src/tools/network/web-search.ts +8 -4
- package/src/tools/reminder/reminder-store.ts +14 -15
- package/src/tools/schedule/create.ts +1 -0
- package/src/tools/schedule/list.ts +2 -1
- package/src/tools/shared/filesystem/file-ops-service.ts +5 -7
- package/src/tools/skills/skill-script-runner.ts +24 -9
- package/src/tools/skills/skill-tool-factory.ts +1 -0
- package/src/tools/tasks/work-item-enqueue.ts +2 -2
- package/src/tools/terminal/evaluate-typescript.ts +21 -12
- package/src/tools/terminal/parser.ts +50 -0
- package/src/tools/types.ts +2 -0
- package/src/tools/watcher/delete.ts +6 -0
- package/src/tools/weather/service.ts +1 -1
- package/src/twitter/client.ts +190 -24
- package/src/twitter/router.ts +1 -1
- package/src/twitter/session.ts +4 -3
- package/src/util/clipboard.ts +1 -1
- package/src/util/errors.ts +65 -8
- package/src/util/fs.ts +40 -0
- package/src/util/json.ts +10 -0
- package/src/util/log-redact.ts +189 -0
- package/src/util/logger.ts +19 -17
- package/src/util/object.ts +3 -0
- package/src/util/platform.ts +105 -363
- package/src/util/pricing.ts +1 -1
- package/src/util/promise-guard.ts +1 -1
- package/src/util/retry.ts +19 -0
- package/src/util/row-mapper.ts +79 -0
- package/src/util/silently.ts +21 -0
- package/src/watcher/engine.ts +5 -1
- package/src/watcher/provider-types.ts +20 -0
- package/src/watcher/providers/github.ts +156 -0
- package/src/watcher/providers/gmail.ts +1 -0
- package/src/watcher/providers/google-calendar.ts +1 -0
- package/src/watcher/providers/linear.ts +460 -0
- package/src/watcher/providers/slack.ts +1 -0
- package/src/work-items/work-item-runner.ts +1 -1
- package/src/workspace/git-service.ts +1 -1
- package/src/workspace/provider-commit-message-generator.ts +51 -22
- package/src/__tests__/call-bridge.test.ts +0 -517
- package/src/__tests__/session-process-bridge.test.ts +0 -244
- package/src/calls/call-bridge.ts +0 -168
- package/src/config/vellum-skills/google-oauth-setup/SKILL.md +0 -199
|
@@ -20,7 +20,12 @@ Gmail, Slack, and Telegram setup all require a publicly reachable URL for OAuth
|
|
|
20
20
|
2. **If it fails because no client_id is found:** The user needs to create Google Cloud OAuth credentials first. Install and load the **google-oauth-setup** skill (which depends on **public-ingress** for the redirect URI):
|
|
21
21
|
- Call `vellum_skills_catalog` with `action: "install"` and `skill_id: "google-oauth-setup"`.
|
|
22
22
|
- Then call `skill_load` with `skill: "google-oauth-setup"`.
|
|
23
|
-
- Tell the user
|
|
23
|
+
- Tell the user Gmail isn't connected yet and briefly explain what the setup involves, then use `ui_show` with `surface_type: "confirmation"` to ask for permission to start:
|
|
24
|
+
- **message:** "Ready to set up Gmail?"
|
|
25
|
+
- **detail:** "I'll open a browser where you sign in to Google, then automate everything else — creating a project, enabling APIs, and connecting your account. Takes 2-3 minutes and you can watch in the browser preview panel."
|
|
26
|
+
- **confirmLabel:** "Get Started"
|
|
27
|
+
- **cancelLabel:** "Not Now"
|
|
28
|
+
- If the user confirms, briefly acknowledge (e.g., "Setting up Gmail now...") and proceed with the setup guide. If they decline, acknowledge and let them know they can set it up later.
|
|
24
29
|
3. **If the user provides a client_id directly in chat:** Call `credential_store` with `action: "oauth2_connect"`, `service: "gmail"`, and `client_id: "<their value>"`. Include `client_secret` too if they provide one. Everything else is auto-filled.
|
|
25
30
|
|
|
26
31
|
### Slack
|
|
@@ -28,7 +33,12 @@ Gmail, Slack, and Telegram setup all require a publicly reachable URL for OAuth
|
|
|
28
33
|
2. **If it fails because no client_id is found:** The user needs to create a Slack App first. Install and load the **slack-oauth-setup** skill (which depends on **public-ingress** for the redirect URI):
|
|
29
34
|
- Call `vellum_skills_catalog` with `action: "install"` and `skill_id: "slack-oauth-setup"`.
|
|
30
35
|
- Then call `skill_load` with `skill: "slack-oauth-setup"`.
|
|
31
|
-
- Tell the user
|
|
36
|
+
- Tell the user Slack isn't connected yet and briefly explain what the setup involves, then use `ui_show` with `surface_type: "confirmation"` to ask for permission to start:
|
|
37
|
+
- **message:** "Ready to set up Slack?"
|
|
38
|
+
- **detail:** "I'll walk you through creating a Slack App and connecting your workspace. The process takes a few minutes, and I'll ask for your approval before each step."
|
|
39
|
+
- **confirmLabel:** "Get Started"
|
|
40
|
+
- **cancelLabel:** "Not Now"
|
|
41
|
+
- Wait for the user to confirm before proceeding with the setup guide. If they decline, acknowledge and let them know they can set it up later.
|
|
32
42
|
3. **If the user provides client_id and client_secret directly in chat:** Call `credential_store` with `action: "oauth2_connect"`, `service: "slack"`, `client_id`, and `client_secret`. Everything else is auto-filled. Note: Slack always requires a client_secret.
|
|
33
43
|
|
|
34
44
|
### Telegram
|
|
@@ -42,14 +52,14 @@ The telegram-setup skill handles: verifying the bot token from @BotFather, gener
|
|
|
42
52
|
The telegram-setup skill also includes **guardian verification**, which links your Telegram account as the trusted guardian for the bot.
|
|
43
53
|
|
|
44
54
|
### SMS (Twilio)
|
|
45
|
-
SMS messaging uses Twilio as the telephony provider. Twilio credentials and phone number configuration are shared with the **phone-calls** skill. Load the **
|
|
46
|
-
- Call `vellum_skills_catalog` with `action: "install"` and `skill_id: "
|
|
47
|
-
- Then call `skill_load` with `skill: "
|
|
48
|
-
- Tell the user: *"I've loaded
|
|
55
|
+
SMS messaging uses Twilio as the telephony provider. Twilio credentials and phone number configuration are shared with the **phone-calls** skill. Load the **sms-setup** skill for complete SMS configuration including compliance and testing:
|
|
56
|
+
- Call `vellum_skills_catalog` with `action: "install"` and `skill_id: "sms-setup"`.
|
|
57
|
+
- Then call `skill_load` with `skill: "sms-setup"`.
|
|
58
|
+
- Tell the user: *"I've loaded the SMS setup guide. It will walk you through configuring Twilio, handling compliance requirements, and testing SMS delivery."*
|
|
49
59
|
|
|
50
|
-
The
|
|
60
|
+
The sms-setup skill handles: Twilio credential storage (Account SID + Auth Token), phone number provisioning or assignment, public ingress setup, SMS compliance verification, and end-to-end test sending. Once SMS is set up, messaging is available automatically — no additional feature flag is needed.
|
|
51
61
|
|
|
52
|
-
The
|
|
62
|
+
The sms-setup skill also includes optional **guardian verification** for SMS (inherited from twilio-setup), which links your phone number as the trusted guardian.
|
|
53
63
|
|
|
54
64
|
## Platform Selection
|
|
55
65
|
|
|
@@ -83,6 +93,21 @@ Telegram is supported as a messaging provider with limited capabilities compared
|
|
|
83
93
|
- The bot can only message users or groups that have previously interacted with it (sent `/start` or been added to a group). Bots cannot initiate conversations with arbitrary phone numbers.
|
|
84
94
|
- Future support for MTProto user-account sessions may lift some of these restrictions.
|
|
85
95
|
|
|
96
|
+
### SMS (Twilio)
|
|
97
|
+
SMS is supported as a messaging provider with limited capabilities. The conversation ID is the recipient's phone number in E.164 format (e.g. `+14155551234`):
|
|
98
|
+
|
|
99
|
+
- **Send**: Send an SMS to a phone number (high risk — requires user approval)
|
|
100
|
+
- **Auth Test**: Verify Twilio credentials and show the configured phone number
|
|
101
|
+
|
|
102
|
+
**Not available** (SMS limitations):
|
|
103
|
+
- List conversations — SMS is stateless; there is no API to enumerate past conversations
|
|
104
|
+
- Read message history — message history is not available through the gateway
|
|
105
|
+
- Search messages — no search API is available for SMS
|
|
106
|
+
|
|
107
|
+
**SMS limits:**
|
|
108
|
+
- Outbound SMS uses the assistant's configured Twilio phone number as the sender. The phone number must be provisioned and assigned via the twilio-setup skill.
|
|
109
|
+
- SMS messages are subject to Twilio's character limits and carrier filtering. Long messages may be split into multiple segments.
|
|
110
|
+
|
|
86
111
|
### Slack-specific
|
|
87
112
|
- **Add Reaction**: Add an emoji reaction to a message
|
|
88
113
|
- **Leave Channel**: Leave a Slack channel
|
|
@@ -7,12 +7,9 @@ import { memoryItems } from '../../../../memory/schema.js';
|
|
|
7
7
|
import { enqueueMemoryJob } from '../../../../memory/jobs-store.js';
|
|
8
8
|
import { extractStylePatterns } from '../../../../messaging/style-analyzer.js';
|
|
9
9
|
import { truncate } from '../../../../util/truncate.js';
|
|
10
|
+
import { clampUnitInterval } from '../../../../memory/validation.js';
|
|
10
11
|
import { resolveProvider, withProviderToken, ok, err } from './shared.js';
|
|
11
12
|
|
|
12
|
-
function clamp(value: number, min: number, max: number): number {
|
|
13
|
-
return Math.min(max, Math.max(min, value));
|
|
14
|
-
}
|
|
15
|
-
|
|
16
13
|
function upsertMemoryItem(opts: {
|
|
17
14
|
kind: string;
|
|
18
15
|
subject: string;
|
|
@@ -35,7 +32,7 @@ function upsertMemoryItem(opts: {
|
|
|
35
32
|
.set({
|
|
36
33
|
statement: opts.statement,
|
|
37
34
|
status: 'active',
|
|
38
|
-
importance: Math.max(existing.importance ?? 0, opts.importance),
|
|
35
|
+
importance: clampUnitInterval(Math.max(existing.importance ?? 0, opts.importance)),
|
|
39
36
|
lastSeenAt: now,
|
|
40
37
|
verificationState: 'assistant_inferred',
|
|
41
38
|
})
|
|
@@ -51,7 +48,7 @@ function upsertMemoryItem(opts: {
|
|
|
51
48
|
statement: opts.statement,
|
|
52
49
|
status: 'active',
|
|
53
50
|
confidence: 0.8,
|
|
54
|
-
importance: opts.importance,
|
|
51
|
+
importance: clampUnitInterval(opts.importance),
|
|
55
52
|
fingerprint,
|
|
56
53
|
verificationState: 'assistant_inferred',
|
|
57
54
|
scopeId: opts.scopeId,
|
|
@@ -90,7 +87,7 @@ export async function run(input: Record<string, unknown>, context: ToolContext):
|
|
|
90
87
|
|
|
91
88
|
for (const pattern of result.stylePatterns) {
|
|
92
89
|
const subject = `${provider.id} writing style: ${pattern.aspect}`;
|
|
93
|
-
const importance =
|
|
90
|
+
const importance = clampUnitInterval(Math.min(0.85, Math.max(0.55, pattern.importance ?? 0.65)));
|
|
94
91
|
upsertMemoryItem({ kind: 'style', subject, statement: pattern.summary, importance, scopeId });
|
|
95
92
|
savedCount++;
|
|
96
93
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { ToolContext, ToolExecutionResult } from '../../../../tools/types.js';
|
|
2
2
|
import { resolveProvider, withProviderToken, ok, err } from './shared.js';
|
|
3
3
|
|
|
4
|
-
export async function run(input: Record<string, unknown>,
|
|
4
|
+
export async function run(input: Record<string, unknown>, context: ToolContext): Promise<ToolExecutionResult> {
|
|
5
5
|
const platform = input.platform as string | undefined;
|
|
6
6
|
const conversationId = input.conversation_id as string;
|
|
7
7
|
const threadId = input.thread_id as string;
|
|
@@ -22,6 +22,7 @@ export async function run(input: Record<string, unknown>, _context: ToolContext)
|
|
|
22
22
|
return withProviderToken(provider, async (token) => {
|
|
23
23
|
const result = await provider.sendMessage(token, conversationId, text, {
|
|
24
24
|
threadId,
|
|
25
|
+
assistantId: context.assistantId,
|
|
25
26
|
});
|
|
26
27
|
|
|
27
28
|
return ok(`Reply sent (ID: ${result.id}).`);
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { ToolContext, ToolExecutionResult } from '../../../../tools/types.js';
|
|
2
2
|
import { resolveProvider, withProviderToken, ok, err } from './shared.js';
|
|
3
3
|
|
|
4
|
-
export async function run(input: Record<string, unknown>,
|
|
4
|
+
export async function run(input: Record<string, unknown>, context: ToolContext): Promise<ToolExecutionResult> {
|
|
5
5
|
const platform = input.platform as string | undefined;
|
|
6
6
|
const conversationId = input.conversation_id as string;
|
|
7
7
|
const text = input.text as string;
|
|
@@ -21,8 +21,12 @@ export async function run(input: Record<string, unknown>, _context: ToolContext)
|
|
|
21
21
|
const result = await provider.sendMessage(token, conversationId, text, {
|
|
22
22
|
subject,
|
|
23
23
|
inReplyTo,
|
|
24
|
+
assistantId: context.assistantId,
|
|
24
25
|
});
|
|
25
26
|
|
|
27
|
+
if (provider.id === 'sms') {
|
|
28
|
+
return ok(`SMS accepted by Twilio (ID: ${result.id}). Note: "accepted" means Twilio received it for delivery — it has not yet been confirmed as delivered to the handset.`);
|
|
29
|
+
}
|
|
26
30
|
return ok(`Message sent (ID: ${result.id}).`);
|
|
27
31
|
});
|
|
28
32
|
} catch (e) {
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: "Phone Calls"
|
|
3
|
-
description: "Set up Twilio for
|
|
3
|
+
description: "Set up Twilio for AI-powered voice calls — both outgoing calls on behalf of the user and incoming calls where the assistant answers as a receptionist"
|
|
4
4
|
user-invocable: true
|
|
5
5
|
metadata: {"vellum": {"emoji": "📞", "requires": {"config": ["calls.enabled"]}}}
|
|
6
6
|
includes: ["public-ingress"]
|
|
7
7
|
---
|
|
8
8
|
|
|
9
|
-
You are helping the user set up and
|
|
9
|
+
You are helping the user set up and manage phone calls via Twilio. This skill covers the full lifecycle: Twilio account setup, credential storage, public ingress configuration, enabling the calls feature, placing outbound calls, receiving inbound calls, and monitoring live transcripts.
|
|
10
10
|
|
|
11
11
|
## Prerequisites — Shared Twilio Setup
|
|
12
12
|
|
|
@@ -21,7 +21,9 @@ If Twilio is already configured (check `twilio_config` with `action: "get"`), sk
|
|
|
21
21
|
|
|
22
22
|
## Overview
|
|
23
23
|
|
|
24
|
-
The calling system uses Twilio's ConversationRelay
|
|
24
|
+
The calling system uses Twilio's ConversationRelay for both **outbound** and **inbound** voice calls. Twilio works out of the box as the default voice provider. Optionally, you can enable ElevenLabs integration for higher-quality, more natural-sounding voices — but this is entirely optional.
|
|
25
|
+
|
|
26
|
+
### Outbound calls
|
|
25
27
|
|
|
26
28
|
When a call is placed:
|
|
27
29
|
|
|
@@ -31,6 +33,17 @@ When a call is placed:
|
|
|
31
33
|
4. An LLM-driven orchestrator manages the conversation — receiving caller speech (transcribed by Deepgram), generating responses via Claude, and streaming text back for TTS playback
|
|
32
34
|
5. The transcript is relayed live to the user's conversation thread
|
|
33
35
|
|
|
36
|
+
### Inbound calls
|
|
37
|
+
|
|
38
|
+
When someone dials the assistant's Twilio phone number:
|
|
39
|
+
|
|
40
|
+
1. Twilio sends a voice webhook to the gateway at `/webhooks/twilio/voice` (no `callSessionId` in the URL)
|
|
41
|
+
2. The gateway resolves which assistant owns the dialed number via `resolveAssistantByPhoneNumber`, falling back to the standard routing chain (chat_id, user_id, default/reject). Unmapped numbers are rejected with TwiML `<Reject>`.
|
|
42
|
+
3. The runtime creates a new session keyed by the Twilio CallSid (`createInboundVoiceSession`)
|
|
43
|
+
4. Twilio opens a ConversationRelay WebSocket. The relay detects the call is inbound when `initiatedFromConversationId == null` and optionally gates the call behind **guardian voice verification** if a pending challenge exists.
|
|
44
|
+
5. Once verified (or if no challenge is pending), the LLM orchestrator greets the caller in a receptionist style: "Hello, this is [user]'s assistant. How can I help you today?"
|
|
45
|
+
6. The assistant converses naturally, using ASK_GUARDIAN to consult the user when needed, just like outbound calls.
|
|
46
|
+
|
|
34
47
|
Three voice quality modes are available:
|
|
35
48
|
- **`twilio_standard`** (default) — Fully supported. Standard Twilio TTS with Google voices. No extra setup required.
|
|
36
49
|
- **`twilio_elevenlabs_tts`** — Fully supported. Uses ElevenLabs voices through Twilio ConversationRelay for more natural speech.
|
|
@@ -181,6 +194,25 @@ credential_store action=store service=twilio field=user_phone_number value=+1415
|
|
|
181
194
|
| `calls.callerIdentity.allowPerCallOverride` | Whether per-call mode selection is allowed | `true` |
|
|
182
195
|
| `calls.callerIdentity.userNumber` | Optional E.164 phone number for user-number mode (alternative to storing via `credential_store`) | *(empty)* |
|
|
183
196
|
|
|
197
|
+
## DTMF Callee Verification
|
|
198
|
+
|
|
199
|
+
An optional verification step where the callee must enter a numeric code via their phone's keypad (DTMF tones) before the call proceeds. This ensures the intended person has answered the phone.
|
|
200
|
+
|
|
201
|
+
### How it works
|
|
202
|
+
|
|
203
|
+
1. When the call connects and DTMF verification is enabled, a random numeric code is generated (length configured by `calls.verification.codeLength`).
|
|
204
|
+
2. The verification code is shared with the guardian in the initiating conversation so they know what code was issued.
|
|
205
|
+
3. The AI voice agent speaks the code digit-by-digit to the callee and asks them to enter it on their keypad.
|
|
206
|
+
4. The callee enters the code via DTMF (phone keypad tones).
|
|
207
|
+
5. If the code matches, the call proceeds normally. If the code is incorrect, the agent may re-prompt or end the call depending on configuration.
|
|
208
|
+
|
|
209
|
+
### Configuration
|
|
210
|
+
|
|
211
|
+
| Setting | Description | Default |
|
|
212
|
+
|---|---|---|
|
|
213
|
+
| `calls.verification.enabled` | Enable DTMF callee verification | `false` |
|
|
214
|
+
| `calls.verification.codeLength` | Number of digits in the verification code | `6` |
|
|
215
|
+
|
|
184
216
|
## Optional: Higher Quality Voice with ElevenLabs
|
|
185
217
|
|
|
186
218
|
ElevenLabs integration is entirely optional. The standard Twilio-only setup works unchanged — this section is only relevant if you want to improve voice quality.
|
|
@@ -264,7 +296,7 @@ To go back to the default voice at any time:
|
|
|
264
296
|
vellum config set calls.voice.mode twilio_standard
|
|
265
297
|
```
|
|
266
298
|
|
|
267
|
-
## Making Calls
|
|
299
|
+
## Making Outbound Calls
|
|
268
300
|
|
|
269
301
|
Use the `call_start` tool to place outbound calls. Every call requires:
|
|
270
302
|
- **phone_number**: The number to call in E.164 format (e.g. `+14155551234`)
|
|
@@ -319,6 +351,30 @@ On Twilio trial accounts, outbound calls can ONLY be made to **verified numbers*
|
|
|
319
351
|
1. Tell the user they need to verify the number at https://console.twilio.com/us1/develop/phone-numbers/manage/verified
|
|
320
352
|
2. Or upgrade to a paid Twilio account to call any number
|
|
321
353
|
|
|
354
|
+
## Receiving Inbound Calls
|
|
355
|
+
|
|
356
|
+
Once Twilio is configured and the assistant has a phone number, inbound calls work automatically. When someone dials the assistant's number:
|
|
357
|
+
|
|
358
|
+
1. The gateway resolves the assistant by phone number and forwards to the runtime
|
|
359
|
+
2. A new voice session is created, keyed by the Twilio CallSid
|
|
360
|
+
3. The LLM-driven orchestrator answers in receptionist mode — greeting the caller warmly and asking how it can help
|
|
361
|
+
4. The conversation proceeds naturally, with ASK_GUARDIAN dispatches to consult the user when needed
|
|
362
|
+
|
|
363
|
+
No additional configuration is needed beyond the standard Twilio setup (Steps 1-5 above). As long as `calls.enabled` is `true` and the phone number has been provisioned/assigned, inbound calls are handled automatically.
|
|
364
|
+
|
|
365
|
+
### Guardian voice verification for inbound calls
|
|
366
|
+
|
|
367
|
+
Optionally, the user can require callers to verify themselves by entering a six-digit code before the call proceeds. This is managed through the **channel guardian verification** system:
|
|
368
|
+
|
|
369
|
+
1. The user initiates a verification challenge from the desktop UI for the `voice` channel
|
|
370
|
+
2. A six-digit code is generated and shown to the user
|
|
371
|
+
3. When the next inbound call arrives, the relay server detects the pending challenge and prompts the caller: "Please enter your six-digit verification code using your keypad, or speak the digits now."
|
|
372
|
+
4. The caller enters the code via DTMF (keypad) or by speaking the digits
|
|
373
|
+
5. If the code matches, a guardian binding is created and the call proceeds normally
|
|
374
|
+
6. If verification fails after 3 attempts, the call ends with "Verification failed. Goodbye."
|
|
375
|
+
|
|
376
|
+
This feature is separate from the outbound DTMF callee verification. It uses the `ChannelGuardianService` challenge system rather than the per-call verification config.
|
|
377
|
+
|
|
322
378
|
## Live Call Monitoring
|
|
323
379
|
|
|
324
380
|
### Showing the live transcript
|
|
@@ -343,37 +399,31 @@ By default, always show the live transcript of the call as it happens. When a ca
|
|
|
343
399
|
|
|
344
400
|
### Interacting with a live call
|
|
345
401
|
|
|
346
|
-
During an active call, the user can
|
|
347
|
-
|
|
348
|
-
#### Mode 1: Answering questions
|
|
402
|
+
During an active call, the user can interact with the AI voice agent via the HTTP API endpoints:
|
|
349
403
|
|
|
350
|
-
|
|
404
|
+
#### Answering questions
|
|
351
405
|
|
|
352
|
-
|
|
353
|
-
2. Present the question prominently to the user:
|
|
406
|
+
When the AI voice agent encounters something it needs user input for, it dispatches an **ASK_GUARDIAN** request to all configured guardian channels (mac desktop, Telegram, SMS). The call status changes to `waiting_on_user`.
|
|
354
407
|
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
3. The user replies directly in the chat — since there is a pending question, the reply is automatically routed as an **answer** to the AI voice agent
|
|
361
|
-
4. The AI voice agent receives the answer and continues the conversation naturally
|
|
408
|
+
1. The question is delivered simultaneously to every configured channel. The first channel to respond wins (first-response-wins semantics) -- once one channel provides an answer, the other channels receive a "already answered" notice.
|
|
409
|
+
2. On the mac desktop, a guardian request thread is created with the question. On Telegram/SMS, the question text and a request code are delivered via the gateway.
|
|
410
|
+
3. If DTMF callee verification is enabled, the callee must enter a verification code before the call proceeds (see the **DTMF Callee Verification** section above).
|
|
411
|
+
4. The guardian provides an answer through whichever channel they prefer. The answer is routed to the AI voice agent, which continues the conversation naturally.
|
|
362
412
|
|
|
363
413
|
**Important:** Respond to pending questions quickly. There is a consultation timeout (default: 2 minutes). If no answer is provided in time, the AI voice agent will move on.
|
|
364
414
|
|
|
365
|
-
####
|
|
415
|
+
#### Steering with instructions
|
|
366
416
|
|
|
367
|
-
When there is **no pending question** but the call is still active,
|
|
417
|
+
When there is **no pending question** but the call is still active, the user can send steering instructions via the HTTP API (`POST /v1/calls/:id/instruction`) to proactively guide the call in real time — for example:
|
|
368
418
|
|
|
369
419
|
- "Ask them about their cancellation policy too"
|
|
370
420
|
- "Wrap up the call, we have what we need"
|
|
371
421
|
- "Switch to asking about weekend availability instead"
|
|
372
422
|
- "Be more assertive about getting a discount"
|
|
373
423
|
|
|
374
|
-
The instruction is injected into the AI voice agent's conversation context as high-priority input, and the agent adjusts its behavior accordingly.
|
|
424
|
+
The instruction is injected into the AI voice agent's conversation context as high-priority input, and the agent adjusts its behavior accordingly.
|
|
375
425
|
|
|
376
|
-
**
|
|
426
|
+
**Note:** Mid-call steering via the desktop chat thread is no longer supported. The desktop thread only receives pointer/status messages about the call. To steer a call, use the HTTP API endpoints directly.
|
|
377
427
|
|
|
378
428
|
### Call status values
|
|
379
429
|
|
|
@@ -414,6 +464,7 @@ The `context` field is powerful — use it to give the agent background that hel
|
|
|
414
464
|
|
|
415
465
|
### Things the AI voice agent handles well
|
|
416
466
|
|
|
467
|
+
**Outbound calls:**
|
|
417
468
|
- Making reservations and appointments
|
|
418
469
|
- Checking business hours, availability, or pricing
|
|
419
470
|
- Confirming or rescheduling existing appointments
|
|
@@ -421,6 +472,12 @@ The `context` field is powerful — use it to give the agent background that hel
|
|
|
421
472
|
- Simple customer service interactions
|
|
422
473
|
- Leaving voicemails (it will speak the message if voicemail picks up)
|
|
423
474
|
|
|
475
|
+
**Inbound calls:**
|
|
476
|
+
- Answering as a receptionist and routing caller requests to the user via ASK_GUARDIAN
|
|
477
|
+
- Taking messages when the user is unavailable
|
|
478
|
+
- Answering questions the assistant already knows from memory/context
|
|
479
|
+
- Screening calls with guardian voice verification
|
|
480
|
+
|
|
424
481
|
### Things to be aware of
|
|
425
482
|
|
|
426
483
|
- Calls have a maximum duration (configurable via `calls.maxDurationSeconds`, default: 1 hour)
|
|
@@ -439,7 +496,7 @@ All call-related settings can be managed via `vellum config`:
|
|
|
439
496
|
| `calls.maxDurationSeconds` | Maximum call length in seconds | `3600` (1 hour) |
|
|
440
497
|
| `calls.userConsultTimeoutSeconds` | How long to wait for user answers | `120` (2 min) |
|
|
441
498
|
| `calls.disclosure.enabled` | Whether the AI announces itself at call start | `true` |
|
|
442
|
-
| `calls.disclosure.text` | The disclosure message spoken at call start | `"
|
|
499
|
+
| `calls.disclosure.text` | The disclosure message spoken at call start | `"At the very beginning of the call, introduce yourself as an assistant calling on behalf of my human."` |
|
|
443
500
|
| `calls.model` | Override LLM model for call orchestration | *(uses default model)* |
|
|
444
501
|
| `calls.callerIdentity.allowPerCallOverride` | Allow per-call caller identity selection | `true` |
|
|
445
502
|
| `calls.callerIdentity.userNumber` | E.164 phone number for user-number mode | *(empty)* |
|
|
@@ -464,12 +521,20 @@ vellum config set calls.maxDurationSeconds 7200
|
|
|
464
521
|
vellum config set calls.disclosure.enabled false
|
|
465
522
|
|
|
466
523
|
# Custom disclosure message
|
|
467
|
-
vellum config set calls.disclosure.text "Just so you know, this is an
|
|
524
|
+
vellum config set calls.disclosure.text "Just so you know, this is an assistant calling on behalf of my human."
|
|
468
525
|
|
|
469
526
|
# Give more time for user consultation
|
|
470
527
|
vellum config set calls.userConsultTimeoutSeconds 300
|
|
471
528
|
```
|
|
472
529
|
|
|
530
|
+
## Accepted Regressions
|
|
531
|
+
|
|
532
|
+
The following behavioral changes were introduced with the cross-channel guardian architecture (voice-cross-guardian):
|
|
533
|
+
|
|
534
|
+
- **No more mid-call steering via desktop chat.** The call bridge that routed desktop chat messages to the active call has been removed. The desktop chat thread only receives pointer/status messages about the call. To steer a call, use the HTTP API endpoints directly (`POST /v1/calls/:id/instruction`).
|
|
535
|
+
- **No live transcript mirror in the initiating chat.** The initiating desktop conversation no longer receives a real-time mirror of the call transcript. The initiating chat only gets pointer/status messages (call started, call ended, question asked, etc.).
|
|
536
|
+
- **Guardian questions are dispatched cross-channel.** Rather than appearing only in the initiating desktop thread, ASK_GUARDIAN questions are now dispatched to all configured guardian channels (mac desktop, Telegram, SMS) simultaneously. The first channel to respond wins.
|
|
537
|
+
|
|
473
538
|
## Troubleshooting
|
|
474
539
|
|
|
475
540
|
### "Twilio credentials not configured"
|
|
@@ -17,7 +17,7 @@ OAuth uses the official X API v2. It is the most reliable connection method and
|
|
|
17
17
|
|
|
18
18
|
- Supports: **post** and **reply**
|
|
19
19
|
- Read-only operations (timeline, search, home, bookmarks, notifications, likes, followers, following, media) always use the browser path directly, regardless of the strategy setting.
|
|
20
|
-
- Setup:
|
|
20
|
+
- Setup: Collect the OAuth Client ID (and optional Client Secret) from the user in chat using `credential_store` with `action: "prompt"`, then initiate the `twitter_auth_start` IPC flow. See the **First-Use Decision Flow** for the full sequence.
|
|
21
21
|
- Set the strategy: `vellum x strategy set oauth`
|
|
22
22
|
|
|
23
23
|
### Browser session (no developer credentials needed)
|
|
@@ -45,15 +45,31 @@ When the user triggers a Twitter operation and no strategy has been configured y
|
|
|
45
45
|
Look at `oauthConnected`, `browserSessionActive`, `preferredStrategy`, and `strategyConfigured` in the response. If `strategyConfigured` is `false`, the user has not yet chosen a strategy and should be guided through setup.
|
|
46
46
|
|
|
47
47
|
2. **Present both options with trade-offs:**
|
|
48
|
-
- **OAuth**: Most reliable and official. Requires X developer app credentials (OAuth Client ID and optional Client Secret). Supports posting and replying. Set up
|
|
48
|
+
- **OAuth**: Most reliable and official. Requires X developer app credentials (OAuth Client ID and optional Client Secret). Supports posting and replying. Set up right here in the chat.
|
|
49
49
|
- **Browser session**: Quick to start, no developer credentials needed. Supports all operations including reading timelines and searching. Set up with `vellum x refresh`.
|
|
50
50
|
|
|
51
51
|
3. **Ask the user which they prefer.** Do not choose for them.
|
|
52
52
|
|
|
53
53
|
4. **Execute setup for the chosen path:**
|
|
54
|
-
- If OAuth:
|
|
54
|
+
- If OAuth: Collect the credentials in-chat using the secure credential prompt, then connect. Follow the **OAuth Setup Sequence** below.
|
|
55
55
|
- If browser: Run `vellum x refresh` to capture session cookies from Chrome.
|
|
56
56
|
|
|
57
|
+
### OAuth Setup Sequence
|
|
58
|
+
|
|
59
|
+
When the user chooses OAuth, collect their X developer credentials conversationally using the secure UI:
|
|
60
|
+
|
|
61
|
+
1. **Collect the Client ID securely:**
|
|
62
|
+
Call `credential_store` with `action: "prompt"`, `service: "integration:twitter"`, `field: "oauth_client_id"`, `label: "X (Twitter) OAuth Client ID"`, `description: "Enter the Client ID from your X Developer App"`, and `placeholder: "your-client-id"`.
|
|
63
|
+
|
|
64
|
+
2. **Collect the Client Secret (if applicable):**
|
|
65
|
+
Ask the user if their X app uses a confidential client (has a Client Secret). If yes, call `credential_store` with `action: "prompt"`, `service: "integration:twitter"`, `field: "oauth_client_secret"`, `label: "X (Twitter) OAuth Client Secret"`, `description: "Enter the Client Secret from your X Developer App (leave blank if using a public client)"`, and `placeholder: "your-client-secret"`.
|
|
66
|
+
|
|
67
|
+
3. **Initiate the OAuth flow:**
|
|
68
|
+
Send the `twitter_auth_start` IPC message. This opens the X authorization page in the user's browser. Wait for the flow to complete.
|
|
69
|
+
|
|
70
|
+
4. **Confirm success:**
|
|
71
|
+
Tell the user: "Great, your X account is connected! You can always update these credentials from the Settings page."
|
|
72
|
+
|
|
57
73
|
5. **Set the preferred strategy:**
|
|
58
74
|
```bash
|
|
59
75
|
vellum x strategy set <oauth|browser|auto>
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
<svg viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg">
|
|
2
|
+
<rect width="16" height="16" fill="#ffffff"/>
|
|
3
|
+
<rect x="2" y="2" width="12" height="12" fill="#000000"/>
|
|
4
|
+
<rect x="4" y="5" width="2" height="1" fill="#ffffff"/>
|
|
5
|
+
<rect x="5" y="4" width="1" height="3" fill="#ffffff"/>
|
|
6
|
+
<rect x="6" y="5" width="2" height="1" fill="#ffffff"/>
|
|
7
|
+
<rect x="10" y="5" width="2" height="1" fill="#ffffff"/>
|
|
8
|
+
<rect x="11" y="4" width="1" height="3" fill="#ffffff"/>
|
|
9
|
+
<rect x="12" y="5" width="2" height="1" fill="#ffffff"/>
|
|
10
|
+
<rect x="4" y="9" width="1" height="2" fill="#ffffff"/>
|
|
11
|
+
<rect x="5" y="8" width="6" height="1" fill="#ffffff"/>
|
|
12
|
+
<rect x="11" y="9" width="1" height="2" fill="#ffffff"/>
|
|
13
|
+
<rect x="5" y="11" width="6" height="1" fill="#ffffff"/>
|
|
14
|
+
</svg>
|