@vellumai/assistant 0.3.4 → 0.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Dockerfile +2 -0
- package/README.md +88 -2
- package/eslint.config.mjs +31 -0
- package/package.json +1 -1
- package/scripts/ipc/check-swift-decoder-drift.ts +4 -1
- package/scripts/ipc/generate-swift.ts +31 -2
- package/src/__tests__/__snapshots__/ipc-snapshot.test.ts.snap +438 -1
- package/src/__tests__/approval-conversation-turn.test.ts +214 -0
- package/src/__tests__/approval-hardcoded-copy-guard.test.ts +41 -0
- package/src/__tests__/approval-message-composer.test.ts +253 -0
- package/src/__tests__/browser-manager.test.ts +1 -0
- package/src/__tests__/call-conversation-messages.test.ts +130 -0
- package/src/__tests__/call-domain.test.ts +12 -2
- package/src/__tests__/call-orchestrator.test.ts +799 -249
- package/src/__tests__/call-pointer-messages.test.ts +148 -0
- package/src/__tests__/call-recovery.test.ts +3 -0
- package/src/__tests__/call-routes-http.test.ts +32 -2
- package/src/__tests__/call-store.test.ts +3 -0
- package/src/__tests__/channel-approval-routes.test.ts +1277 -98
- package/src/__tests__/channel-approval.test.ts +37 -0
- package/src/__tests__/channel-approvals.test.ts +36 -50
- package/src/__tests__/channel-guardian.test.ts +630 -22
- package/src/__tests__/channel-readiness-service.test.ts +324 -0
- package/src/__tests__/checker.test.ts +14 -7
- package/src/__tests__/clarification-resolver.test.ts +44 -24
- package/src/__tests__/commit-message-enrichment-service.test.ts +9 -4
- package/src/__tests__/computer-use-session-working-dir.test.ts +8 -0
- package/src/__tests__/config-schema.test.ts +14 -8
- package/src/__tests__/context-window-manager.test.ts +30 -2
- package/src/__tests__/contradiction-checker.test.ts +20 -5
- package/src/__tests__/credential-security-invariants.test.ts +7 -2
- package/src/__tests__/daemon-lifecycle.test.ts +13 -12
- package/src/__tests__/db-migration-rollback.test.ts +752 -0
- package/src/__tests__/dictation-mode-detection.test.ts +63 -0
- package/src/__tests__/dynamic-skill-workflow-prompt.test.ts +2 -0
- package/src/__tests__/entity-search.test.ts +615 -0
- package/src/__tests__/fuzzy-match-property.test.ts +5 -5
- package/src/__tests__/guardian-action-store.test.ts +123 -0
- package/src/__tests__/guardian-action-sweep.test.ts +277 -0
- package/src/__tests__/guardian-dispatch.test.ts +389 -0
- package/src/__tests__/guardian-question-copy.test.ts +47 -0
- package/src/__tests__/handlers-telegram-config.test.ts +4 -2
- package/src/__tests__/handlers-twilio-config.test.ts +533 -0
- package/src/__tests__/intent-routing.test.ts +2 -0
- package/src/__tests__/ipc-snapshot.test.ts +291 -1
- package/src/__tests__/memory-upsert-concurrency.test.ts +828 -0
- package/src/__tests__/messaging-send-tool.test.ts +65 -0
- package/src/__tests__/model-intents.test.ts +96 -0
- package/src/__tests__/no-direct-anthropic-sdk-imports.test.ts +42 -0
- package/src/__tests__/oauth2-gateway-transport.test.ts +130 -0
- package/src/__tests__/onboarding-starter-tasks.test.ts +2 -0
- package/src/__tests__/provider-commit-message-generator.test.ts +89 -13
- package/src/__tests__/provider-error-scenarios.test.ts +621 -0
- package/src/__tests__/provider-fail-open-selection.test.ts +119 -0
- package/src/__tests__/qdrant-manager.test.ts +27 -20
- package/src/__tests__/relay-server.test.ts +779 -40
- package/src/__tests__/run-orchestrator-assistant-events.test.ts +6 -0
- package/src/__tests__/run-orchestrator.test.ts +42 -4
- package/src/__tests__/runtime-runs-http.test.ts +17 -1
- package/src/__tests__/runtime-runs.test.ts +16 -0
- package/src/__tests__/schedule-store.test.ts +18 -4
- package/src/__tests__/scheduler-recurrence.test.ts +13 -4
- package/src/__tests__/session-abort-tool-results.test.ts +6 -0
- package/src/__tests__/session-agent-loop.test.ts +857 -0
- package/src/__tests__/session-conflict-gate.test.ts +6 -0
- package/src/__tests__/session-pre-run-repair.test.ts +6 -0
- package/src/__tests__/session-profile-injection.test.ts +6 -0
- package/src/__tests__/session-provider-retry-repair.test.ts +6 -0
- package/src/__tests__/session-queue.test.ts +6 -0
- package/src/__tests__/session-runtime-assembly.test.ts +321 -13
- package/src/__tests__/session-slash-known.test.ts +6 -0
- package/src/__tests__/session-slash-queue.test.ts +6 -0
- package/src/__tests__/session-slash-unknown.test.ts +6 -0
- package/src/__tests__/session-surfaces-task-progress.test.ts +2 -0
- package/src/__tests__/session-tool-setup-app-refresh.test.ts +1 -0
- package/src/__tests__/session-tool-setup-memory-scope.test.ts +1 -0
- package/src/__tests__/session-tool-setup-side-effect-flag.test.ts +1 -0
- package/src/__tests__/session-workspace-injection.test.ts +6 -0
- package/src/__tests__/session-workspace-tool-tracking.test.ts +6 -0
- package/src/__tests__/skills.test.ts +2 -0
- package/src/__tests__/sms-messaging-provider.test.ts +126 -0
- package/src/__tests__/starter-task-flow.test.ts +2 -0
- package/src/__tests__/swarm-dag-pathological.test.ts +535 -0
- package/src/__tests__/system-prompt.test.ts +2 -0
- package/src/__tests__/task-management-tools.test.ts +2 -2
- package/src/__tests__/task-runner.test.ts +14 -4
- package/src/__tests__/terminal-tools.test.ts +25 -19
- package/src/__tests__/tool-execution-abort-cleanup.test.ts +545 -0
- package/src/__tests__/tool-executor-shell-integration.test.ts +11 -11
- package/src/__tests__/tool-executor.test.ts +23 -24
- package/src/__tests__/trust-store.test.ts +3 -3
- package/src/__tests__/twilio-rest.test.ts +29 -0
- package/src/__tests__/twilio-routes-elevenlabs.test.ts +3 -0
- package/src/__tests__/twilio-routes-twiml.test.ts +11 -0
- package/src/__tests__/twilio-routes.test.ts +167 -11
- package/src/__tests__/twitter-cli-error-shaping.test.ts +2 -2
- package/src/__tests__/user-reference.test.ts +2 -0
- package/src/__tests__/voice-quality.test.ts +222 -0
- package/src/__tests__/web-search.test.ts +46 -30
- package/src/__tests__/work-item-output.test.ts +110 -0
- package/src/agent/loop.ts +1 -1
- package/src/agent-heartbeat/agent-heartbeat-service.ts +2 -10
- package/src/amazon/client.ts +1418 -0
- package/src/amazon/request-extractor.ts +135 -0
- package/src/amazon/session.ts +109 -0
- package/src/autonomy/autonomy-store.ts +5 -5
- package/src/browser-extension-relay/client.ts +124 -0
- package/src/browser-extension-relay/protocol.ts +63 -0
- package/src/browser-extension-relay/server.ts +177 -0
- package/src/bundler/app-bundler.ts +3 -3
- package/src/bundler/bundle-signer.ts +1 -1
- package/src/bundler/signature-verifier.ts +1 -1
- package/src/calls/call-conversation-messages.ts +33 -0
- package/src/calls/call-domain.ts +114 -10
- package/src/calls/call-orchestrator.ts +268 -59
- package/src/calls/call-pointer-messages.ts +53 -0
- package/src/calls/call-recovery.ts +3 -8
- package/src/calls/call-store.ts +69 -87
- package/src/calls/elevenlabs-config.ts +3 -2
- package/src/calls/guardian-action-sweep.ts +105 -0
- package/src/calls/guardian-dispatch.ts +203 -0
- package/src/calls/guardian-question-copy.ts +133 -0
- package/src/calls/relay-server.ts +466 -8
- package/src/calls/speaker-identification.ts +1 -1
- package/src/calls/twilio-config.ts +22 -14
- package/src/calls/twilio-provider.ts +6 -4
- package/src/calls/twilio-rest.ts +308 -7
- package/src/calls/twilio-routes.ts +65 -12
- package/src/calls/types.ts +3 -1
- package/src/channels/types.ts +25 -0
- package/src/cli/amazon.ts +815 -0
- package/src/cli/config-commands.ts +2 -2
- package/src/cli/core-commands.ts +4 -3
- package/src/cli/influencer.ts +244 -0
- package/src/cli/map.ts +89 -6
- package/src/cli.ts +1 -1
- package/src/config/agent-schema.ts +171 -0
- package/src/config/bundled-skills/amazon/SKILL.md +127 -0
- package/src/config/bundled-skills/amazon/icon.svg +13 -0
- package/src/config/bundled-skills/api-mapping/SKILL.md +78 -0
- package/src/config/bundled-skills/browser/SKILL.md +1 -0
- package/src/config/bundled-skills/browser/TOOLS.json +17 -0
- package/src/config/bundled-skills/browser/tools/browser-wait-for-download.ts +25 -0
- package/src/config/bundled-skills/doordash/SKILL.md +51 -51
- package/src/config/bundled-skills/email-setup/SKILL.md +14 -5
- package/src/config/bundled-skills/google-oauth-setup/SKILL.md +183 -0
- package/src/config/bundled-skills/influencer/SKILL.md +144 -0
- package/src/config/bundled-skills/knowledge-graph/SKILL.md +15 -0
- package/src/config/bundled-skills/knowledge-graph/TOOLS.json +56 -0
- package/src/config/bundled-skills/knowledge-graph/tools/graph-query.ts +185 -0
- package/src/config/bundled-skills/macos-automation/icon.svg +12 -0
- package/src/config/bundled-skills/media-processing/SKILL.md +176 -0
- package/src/config/bundled-skills/media-processing/TOOLS.json +230 -0
- package/src/config/bundled-skills/media-processing/__tests__/concurrency-pool.test.ts +77 -0
- package/src/config/bundled-skills/media-processing/__tests__/cost-tracker.test.ts +69 -0
- package/src/config/bundled-skills/media-processing/__tests__/preprocess.test.ts +303 -0
- package/src/config/bundled-skills/media-processing/services/concurrency-pool.ts +55 -0
- package/src/config/bundled-skills/media-processing/services/cost-tracker.ts +86 -0
- package/src/config/bundled-skills/media-processing/services/gemini-map.ts +339 -0
- package/src/config/bundled-skills/media-processing/services/preprocess.ts +551 -0
- package/src/config/bundled-skills/media-processing/services/processing-pipeline.ts +259 -0
- package/src/config/bundled-skills/media-processing/services/reduce.ts +197 -0
- package/src/config/bundled-skills/media-processing/tools/analyze-keyframes.ts +136 -0
- package/src/config/bundled-skills/media-processing/tools/extract-keyframes.ts +59 -0
- package/src/config/bundled-skills/media-processing/tools/generate-clip.ts +195 -0
- package/src/config/bundled-skills/media-processing/tools/ingest-media.ts +197 -0
- package/src/config/bundled-skills/media-processing/tools/media-diagnostics.ts +143 -0
- package/src/config/bundled-skills/media-processing/tools/media-status.ts +75 -0
- package/src/config/bundled-skills/media-processing/tools/query-media-events.ts +65 -0
- package/src/config/bundled-skills/messaging/SKILL.md +33 -8
- package/src/config/bundled-skills/messaging/tools/messaging-analyze-style.ts +4 -7
- package/src/config/bundled-skills/messaging/tools/messaging-reply.ts +2 -1
- package/src/config/bundled-skills/messaging/tools/messaging-send.ts +5 -1
- package/src/config/bundled-skills/phone-calls/SKILL.md +88 -23
- package/src/config/bundled-skills/twitter/SKILL.md +19 -3
- package/src/config/bundled-skills/twitter/icon.svg +14 -0
- package/src/config/bundled-tool-registry.ts +310 -0
- package/src/config/calls-schema.ts +181 -0
- package/src/config/core-schema.ts +309 -0
- package/src/config/defaults.ts +28 -3
- package/src/config/env-registry.ts +162 -0
- package/src/config/env.ts +175 -0
- package/src/config/loader.ts +6 -6
- package/src/config/memory-schema.ts +528 -0
- package/src/config/sandbox-schema.ts +55 -0
- package/src/config/schema.ts +158 -1133
- package/src/config/skill-state.ts +1 -1
- package/src/config/skills-schema.ts +32 -0
- package/src/config/skills.ts +35 -24
- package/src/config/system-prompt.ts +131 -56
- package/src/config/templates/IDENTITY.md +2 -2
- package/src/config/templates/SOUL.md +1 -1
- package/src/config/types.ts +1 -0
- package/src/config/user-reference.ts +4 -9
- package/src/config/vellum-skills/catalog.json +6 -7
- package/src/config/vellum-skills/chatgpt-import/tools/chatgpt-import.ts +5 -1
- package/src/config/vellum-skills/slack-oauth-setup/SKILL.md +4 -3
- package/src/config/vellum-skills/sms-setup/SKILL.md +216 -0
- package/src/config/vellum-skills/twilio-setup/SKILL.md +40 -8
- package/src/context/window-manager.ts +27 -7
- package/src/daemon/approval-generators.ts +186 -0
- package/src/daemon/approved-devices-store.ts +140 -0
- package/src/daemon/assistant-attachments.ts +1 -1
- package/src/daemon/classifier.ts +35 -32
- package/src/daemon/config-watcher.ts +1 -1
- package/src/daemon/daemon-control.ts +217 -0
- package/src/daemon/handlers/apps.ts +2 -3
- package/src/daemon/handlers/config-channels.ts +158 -0
- package/src/daemon/handlers/config-inbox.ts +540 -0
- package/src/daemon/handlers/config-ingress.ts +231 -0
- package/src/daemon/handlers/config-integrations.ts +258 -0
- package/src/daemon/handlers/config-model.ts +143 -0
- package/src/daemon/handlers/config-parental.ts +163 -0
- package/src/daemon/handlers/config-scheduling.ts +172 -0
- package/src/daemon/handlers/config-slack.ts +92 -0
- package/src/daemon/handlers/config-telegram.ts +301 -0
- package/src/daemon/handlers/config-tools.ts +177 -0
- package/src/daemon/handlers/config-trust.ts +104 -0
- package/src/daemon/handlers/config-twilio.ts +1080 -0
- package/src/daemon/handlers/config.ts +53 -1689
- package/src/daemon/handlers/diagnostics.ts +1 -1
- package/src/daemon/handlers/dictation.ts +180 -0
- package/src/daemon/handlers/documents.ts +18 -32
- package/src/daemon/handlers/identity.ts +14 -23
- package/src/daemon/handlers/index.ts +11 -0
- package/src/daemon/handlers/misc.ts +3 -5
- package/src/daemon/handlers/pairing.ts +98 -0
- package/src/daemon/handlers/sessions.ts +56 -5
- package/src/daemon/handlers/shared.ts +6 -1
- package/src/daemon/handlers/skills.ts +1 -1
- package/src/daemon/handlers/twitter-auth.ts +2 -0
- package/src/daemon/handlers/work-items.ts +17 -9
- package/src/daemon/handlers/workspace-files.ts +4 -3
- package/src/daemon/install-cli-launchers.ts +113 -0
- package/src/daemon/ipc-contract/apps.ts +356 -0
- package/src/daemon/ipc-contract/browser.ts +74 -0
- package/src/daemon/ipc-contract/computer-use.ts +151 -0
- package/src/daemon/ipc-contract/diagnostics.ts +56 -0
- package/src/daemon/ipc-contract/documents.ts +74 -0
- package/src/daemon/ipc-contract/inbox.ts +209 -0
- package/src/daemon/ipc-contract/integrations.ts +284 -0
- package/src/daemon/ipc-contract/memory.ts +48 -0
- package/src/daemon/ipc-contract/messages.ts +211 -0
- package/src/daemon/ipc-contract/pairing.ts +45 -0
- package/src/daemon/ipc-contract/parental-control.ts +95 -0
- package/src/daemon/ipc-contract/schedules.ts +97 -0
- package/src/daemon/ipc-contract/sessions.ts +315 -0
- package/src/daemon/ipc-contract/shared.ts +42 -0
- package/src/daemon/ipc-contract/skills.ts +120 -0
- package/src/daemon/ipc-contract/subagents.ts +58 -0
- package/src/daemon/ipc-contract/surfaces.ts +250 -0
- package/src/daemon/ipc-contract/trust.ts +60 -0
- package/src/daemon/ipc-contract/work-items.ts +225 -0
- package/src/daemon/ipc-contract/workspace.ts +113 -0
- package/src/daemon/ipc-contract-inventory.json +70 -0
- package/src/daemon/ipc-contract-inventory.ts +55 -29
- package/src/daemon/ipc-contract.ts +229 -2426
- package/src/daemon/ipc-protocol.ts +1 -1
- package/src/daemon/ipc-validate.ts +7 -0
- package/src/daemon/lifecycle.ts +97 -377
- package/src/daemon/pairing-store.ts +177 -0
- package/src/daemon/providers-setup.ts +43 -0
- package/src/daemon/ride-shotgun-handler.ts +68 -3
- package/src/daemon/server.ts +66 -46
- package/src/daemon/session-agent-loop-handlers.ts +421 -0
- package/src/daemon/session-agent-loop.ts +117 -275
- package/src/daemon/session-dynamic-profile.ts +1 -1
- package/src/daemon/session-history.ts +1 -1
- package/src/daemon/session-media-retry.ts +1 -1
- package/src/daemon/session-messaging.ts +37 -2
- package/src/daemon/session-notifiers.ts +5 -25
- package/src/daemon/session-process.ts +99 -59
- package/src/daemon/session-queue-manager.ts +96 -4
- package/src/daemon/session-runtime-assembly.ts +199 -10
- package/src/daemon/session-surfaces.ts +19 -4
- package/src/daemon/session-tool-setup.ts +30 -30
- package/src/daemon/session-workspace.ts +1 -1
- package/src/daemon/session.ts +35 -2
- package/src/daemon/shutdown-handlers.ts +122 -0
- package/src/daemon/trace-emitter.ts +1 -1
- package/src/daemon/watch-handler.ts +36 -33
- package/src/doordash/cart-queries.ts +787 -0
- package/src/doordash/client.ts +144 -127
- package/src/doordash/order-queries.ts +85 -0
- package/src/doordash/queries.ts +10 -1308
- package/src/doordash/search-queries.ts +203 -0
- package/src/doordash/session.ts +3 -2
- package/src/doordash/store-queries.ts +246 -0
- package/src/doordash/types.ts +367 -0
- package/src/email/providers/agentmail.ts +2 -1
- package/src/email/providers/index.ts +3 -2
- package/src/email/service.ts +3 -2
- package/src/errors.ts +43 -0
- package/src/home-base/prebuilt/seed.ts +1 -1
- package/src/hooks/cli.ts +6 -5
- package/src/hooks/config.ts +6 -8
- package/src/hooks/discovery.ts +6 -5
- package/src/hooks/manager.ts +4 -3
- package/src/hooks/runner.ts +2 -2
- package/src/hooks/templates.ts +5 -5
- package/src/inbound/public-ingress-urls.ts +6 -4
- package/src/index.ts +4 -2
- package/src/influencer/client.ts +1104 -0
- package/src/instrument.ts +4 -3
- package/src/logfire.ts +4 -3
- package/src/memory/admin.ts +25 -35
- package/src/memory/attachments-store.ts +4 -7
- package/src/memory/channel-delivery-store.ts +30 -1
- package/src/memory/channel-guardian-store.ts +202 -2
- package/src/memory/clarification-resolver.ts +37 -33
- package/src/memory/conflict-store.ts +67 -61
- package/src/memory/contradiction-checker.ts +141 -117
- package/src/memory/conversation-store.ts +335 -51
- package/src/memory/db-connection.ts +27 -4
- package/src/memory/db-init.ts +265 -4
- package/src/memory/db.ts +14 -1
- package/src/memory/embedding-backend.ts +27 -5
- package/src/memory/embedding-ollama.ts +2 -1
- package/src/memory/entity-extractor.ts +38 -35
- package/src/memory/guardian-action-store.ts +430 -0
- package/src/memory/inbox-escalation-projection.ts +59 -0
- package/src/memory/inbox-thread-store.ts +218 -0
- package/src/memory/ingress-invite-store.ts +338 -0
- package/src/memory/ingress-member-store.ts +350 -0
- package/src/memory/items-extractor.ts +91 -97
- package/src/memory/job-handlers/index-maintenance.ts +3 -3
- package/src/memory/job-handlers/media-processing.ts +69 -0
- package/src/memory/job-handlers/summarization.ts +32 -26
- package/src/memory/job-utils.ts +3 -10
- package/src/memory/jobs-store.ts +8 -10
- package/src/memory/jobs-worker.ts +55 -36
- package/src/memory/media-store.ts +759 -0
- package/src/memory/migrations/001-job-deferrals.ts +45 -0
- package/src/memory/migrations/002-tool-invocations-fk.ts +43 -0
- package/src/memory/migrations/003-memory-fts-backfill.ts +24 -0
- package/src/memory/migrations/004-entity-relation-dedup.ts +87 -0
- package/src/memory/migrations/005-fingerprint-scope-unique.ts +80 -0
- package/src/memory/migrations/006-scope-salted-fingerprints.ts +62 -0
- package/src/memory/migrations/007-assistant-id-to-self.ts +254 -0
- package/src/memory/migrations/008-remove-assistant-id-columns.ts +208 -0
- package/src/memory/migrations/009-llm-usage-events-drop-assistant-id.ts +83 -0
- package/src/memory/migrations/010-ext-conv-bindings-channel-chat-unique.ts +56 -0
- package/src/memory/migrations/011-call-sessions-provider-sid-dedup.ts +63 -0
- package/src/memory/migrations/012-call-sessions-add-initiated-from.ts +19 -0
- package/src/memory/migrations/013-guardian-action-tables.ts +68 -0
- package/src/memory/migrations/014-backfill-inbox-thread-state.ts +76 -0
- package/src/memory/migrations/015-drop-active-search-index.ts +27 -0
- package/src/memory/migrations/016-memory-segments-indexes.ts +11 -0
- package/src/memory/migrations/017-memory-items-indexes.ts +10 -0
- package/src/memory/migrations/018-remaining-table-indexes.ts +13 -0
- package/src/memory/migrations/index.ts +24 -0
- package/src/memory/migrations/registry.ts +79 -0
- package/src/memory/migrations/validate-migration-state.ts +69 -0
- package/src/memory/qdrant-manager.ts +49 -8
- package/src/memory/query-builder.ts +1 -1
- package/src/memory/raw-query.ts +119 -0
- package/src/memory/recall-cache.ts +4 -1
- package/src/memory/retriever.ts +165 -47
- package/src/memory/schema-migration.ts +25 -984
- package/src/memory/schema.ts +228 -7
- package/src/memory/search/entity.ts +205 -31
- package/src/memory/search/lexical.ts +81 -52
- package/src/memory/search/ranking.ts +27 -23
- package/src/memory/search/semantic.ts +157 -19
- package/src/memory/search/types.ts +24 -0
- package/src/memory/shared-app-links-store.ts +4 -5
- package/src/memory/validation.ts +19 -0
- package/src/messaging/draft-store.ts +5 -6
- package/src/messaging/provider-types.ts +2 -0
- package/src/messaging/providers/sms/adapter.ts +201 -0
- package/src/messaging/providers/sms/client.ts +93 -0
- package/src/messaging/providers/sms/types.ts +7 -0
- package/src/messaging/providers/telegram-bot/adapter.ts +2 -5
- package/src/messaging/providers/whatsapp/adapter.ts +136 -0
- package/src/messaging/providers/whatsapp/client.ts +67 -0
- package/src/messaging/style-analyzer.ts +5 -4
- package/src/messaging/thread-summarizer.ts +61 -69
- package/src/messaging/triage-engine.ts +62 -71
- package/src/migrations/config-merge.ts +53 -0
- package/src/migrations/data-layout.ts +68 -0
- package/src/migrations/data-merge.ts +33 -0
- package/src/migrations/hooks-merge.ts +90 -0
- package/src/migrations/index.ts +6 -0
- package/src/migrations/log.ts +23 -0
- package/src/migrations/skills-merge.ts +33 -0
- package/src/migrations/workspace-layout.ts +79 -0
- package/src/permissions/checker.ts +133 -11
- package/src/permissions/prompter.ts +14 -0
- package/src/permissions/shell-identity.ts +31 -1
- package/src/permissions/trust-store.ts +21 -1
- package/src/providers/anthropic/client.ts +4 -4
- package/src/providers/failover.ts +2 -2
- package/src/providers/model-intents.ts +70 -0
- package/src/providers/ollama/client.ts +2 -1
- package/src/providers/provider-send-message.ts +176 -0
- package/src/providers/registry.ts +71 -30
- package/src/providers/retry.ts +35 -1
- package/src/providers/types.ts +12 -1
- package/src/runtime/approval-conversation-turn.ts +97 -0
- package/src/runtime/approval-message-composer.ts +253 -0
- package/src/runtime/channel-approval-parser.ts +36 -2
- package/src/runtime/channel-approvals.ts +11 -24
- package/src/runtime/channel-guardian-service.ts +88 -21
- package/src/runtime/channel-readiness-service.ts +418 -0
- package/src/runtime/channel-readiness-types.ts +35 -0
- package/src/runtime/channel-retry-sweep.ts +184 -0
- package/src/runtime/guardian-context-resolver.ts +108 -0
- package/src/runtime/http-server.ts +275 -717
- package/src/runtime/http-types.ts +59 -3
- package/src/runtime/middleware/auth.ts +116 -0
- package/src/runtime/middleware/error-handler.ts +33 -0
- package/src/runtime/middleware/twilio-validation.ts +127 -0
- package/src/runtime/routes/app-routes.ts +1 -1
- package/src/runtime/routes/call-routes.ts +51 -7
- package/src/runtime/routes/channel-delivery-routes.ts +170 -0
- package/src/runtime/routes/channel-guardian-routes.ts +1191 -0
- package/src/runtime/routes/channel-inbound-routes.ts +1152 -0
- package/src/runtime/routes/channel-route-shared.ts +144 -0
- package/src/runtime/routes/channel-routes.ts +32 -1588
- package/src/runtime/routes/conversation-routes.ts +50 -7
- package/src/runtime/routes/events-routes.ts +2 -2
- package/src/runtime/routes/identity-routes.ts +126 -0
- package/src/runtime/routes/pairing-routes.ts +143 -0
- package/src/runtime/routes/run-routes.ts +15 -1
- package/src/runtime/run-orchestrator.ts +86 -35
- package/src/schedule/schedule-store.ts +36 -32
- package/src/schedule/scheduler.ts +3 -3
- package/src/security/encrypted-store.ts +5 -7
- package/src/security/oauth2.ts +45 -15
- package/src/security/parental-control-store.ts +183 -0
- package/src/security/secret-allowlist.ts +4 -3
- package/src/security/secret-scanner.ts +5 -5
- package/src/security/secure-keys.ts +1 -1
- package/src/security/token-manager.ts +3 -2
- package/src/services/vercel-deploy.ts +6 -2
- package/src/skills/tool-manifest.ts +3 -3
- package/src/skills/vellum-catalog-remote.ts +75 -16
- package/src/slack/slack-webhook.ts +2 -1
- package/src/swarm/orchestrator.ts +92 -1
- package/src/swarm/router-planner.ts +6 -9
- package/src/swarm/worker-prompts.ts +9 -12
- package/src/tasks/task-compiler.ts +19 -28
- package/src/tasks/task-runner.ts +1 -1
- package/src/tools/assets/materialize.ts +2 -2
- package/src/tools/assets/search.ts +15 -14
- package/src/tools/browser/__tests__/auth-detector.test.ts +1 -0
- package/src/tools/browser/auto-navigate.ts +1 -0
- package/src/tools/browser/browser-execution.ts +10 -1
- package/src/tools/browser/browser-manager.ts +119 -4
- package/src/tools/browser/network-recorder.ts +5 -0
- package/src/tools/calls/call-start.ts +1 -0
- package/src/tools/credentials/broker.ts +11 -2
- package/src/tools/credentials/metadata-store.ts +18 -14
- package/src/tools/credentials/post-connect-hooks.ts +61 -0
- package/src/tools/credentials/vault.ts +49 -23
- package/src/tools/execution-target.ts +11 -1
- package/src/tools/executor.ts +68 -9
- package/src/tools/host-terminal/cli-discover.ts +1 -1
- package/src/tools/network/script-proxy/http-forwarder.ts +1 -1
- package/src/tools/network/script-proxy/mitm-handler.ts +1 -1
- package/src/tools/network/script-proxy/server.ts +1 -1
- package/src/tools/network/script-proxy/session-manager.ts +6 -5
- package/src/tools/network/web-fetch.ts +18 -2
- package/src/tools/network/web-search.ts +8 -4
- package/src/tools/reminder/reminder-store.ts +14 -15
- package/src/tools/schedule/create.ts +1 -0
- package/src/tools/schedule/list.ts +2 -1
- package/src/tools/shared/filesystem/file-ops-service.ts +5 -7
- package/src/tools/skills/skill-script-runner.ts +24 -9
- package/src/tools/skills/skill-tool-factory.ts +1 -0
- package/src/tools/tasks/work-item-enqueue.ts +2 -2
- package/src/tools/terminal/evaluate-typescript.ts +21 -12
- package/src/tools/terminal/parser.ts +50 -0
- package/src/tools/types.ts +2 -0
- package/src/tools/watcher/delete.ts +6 -0
- package/src/tools/weather/service.ts +1 -1
- package/src/twitter/client.ts +190 -24
- package/src/twitter/router.ts +1 -1
- package/src/twitter/session.ts +4 -3
- package/src/util/clipboard.ts +1 -1
- package/src/util/errors.ts +65 -8
- package/src/util/fs.ts +40 -0
- package/src/util/json.ts +10 -0
- package/src/util/log-redact.ts +189 -0
- package/src/util/logger.ts +19 -17
- package/src/util/object.ts +3 -0
- package/src/util/platform.ts +105 -363
- package/src/util/pricing.ts +1 -1
- package/src/util/promise-guard.ts +1 -1
- package/src/util/retry.ts +19 -0
- package/src/util/row-mapper.ts +79 -0
- package/src/util/silently.ts +21 -0
- package/src/watcher/engine.ts +5 -1
- package/src/watcher/provider-types.ts +20 -0
- package/src/watcher/providers/github.ts +156 -0
- package/src/watcher/providers/gmail.ts +1 -0
- package/src/watcher/providers/google-calendar.ts +1 -0
- package/src/watcher/providers/linear.ts +460 -0
- package/src/watcher/providers/slack.ts +1 -0
- package/src/work-items/work-item-runner.ts +1 -1
- package/src/workspace/git-service.ts +1 -1
- package/src/workspace/provider-commit-message-generator.ts +51 -22
- package/src/__tests__/call-bridge.test.ts +0 -517
- package/src/__tests__/session-process-bridge.test.ts +0 -244
- package/src/calls/call-bridge.ts +0 -168
- package/src/config/vellum-skills/google-oauth-setup/SKILL.md +0 -199
|
@@ -6,8 +6,9 @@
|
|
|
6
6
|
* streams text tokens back through the RelayConnection for real-time TTS.
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
|
-
import Anthropic from '@anthropic-ai/sdk';
|
|
10
9
|
import { getConfig } from '../config/loader.js';
|
|
10
|
+
import { resolveConfiguredProvider } from '../providers/provider-send-message.js';
|
|
11
|
+
import type { ProviderEvent } from '../providers/types.js';
|
|
11
12
|
import { resolveUserReference } from '../config/user-reference.js';
|
|
12
13
|
import { getLogger } from '../util/logger.js';
|
|
13
14
|
import {
|
|
@@ -21,36 +22,87 @@ import { getMaxCallDurationMs, getUserConsultationTimeoutMs, SILENCE_TIMEOUT_MS
|
|
|
21
22
|
import type { RelayConnection } from './relay-server.js';
|
|
22
23
|
import { registerCallOrchestrator, unregisterCallOrchestrator, fireCallQuestionNotifier, fireCallCompletionNotifier, fireCallTranscriptNotifier } from './call-state.js';
|
|
23
24
|
import type { PromptSpeakerContext } from './speaker-identification.js';
|
|
25
|
+
import { addPointerMessage, formatDuration } from './call-pointer-messages.js';
|
|
26
|
+
import { persistCallCompletionMessage } from './call-conversation-messages.js';
|
|
27
|
+
import * as conversationStore from '../memory/conversation-store.js';
|
|
28
|
+
import { dispatchGuardianQuestion } from './guardian-dispatch.js';
|
|
29
|
+
import type { ServerMessage } from '../daemon/ipc-contract.js';
|
|
30
|
+
import {
|
|
31
|
+
buildGuardianContextBlock,
|
|
32
|
+
type GuardianRuntimeContext,
|
|
33
|
+
} from '../daemon/session-runtime-assembly.js';
|
|
24
34
|
|
|
25
35
|
const log = getLogger('call-orchestrator');
|
|
26
36
|
|
|
27
37
|
type OrchestratorState = 'idle' | 'processing' | 'waiting_on_user' | 'speaking';
|
|
28
38
|
|
|
29
|
-
const
|
|
39
|
+
const ASK_GUARDIAN_CAPTURE_REGEX = /\[ASK_GUARDIAN:\s*(.+?)\]/;
|
|
40
|
+
const ASK_GUARDIAN_MARKER_REGEX = /\[ASK_GUARDIAN:\s*.+?\]/g;
|
|
41
|
+
const USER_ANSWERED_MARKER_REGEX = /\[USER_ANSWERED:\s*.+?\]/g;
|
|
42
|
+
const USER_INSTRUCTION_MARKER_REGEX = /\[USER_INSTRUCTION:\s*.+?\]/g;
|
|
43
|
+
const CALL_OPENING_MARKER_REGEX = /\[CALL_OPENING\]/g;
|
|
44
|
+
const CALL_OPENING_ACK_MARKER_REGEX = /\[CALL_OPENING_ACK\]/g;
|
|
45
|
+
const END_CALL_MARKER_REGEX = /\[END_CALL\]/g;
|
|
46
|
+
const CALL_OPENING_MARKER = '[CALL_OPENING]';
|
|
47
|
+
const CALL_OPENING_ACK_MARKER = '[CALL_OPENING_ACK]';
|
|
30
48
|
const END_CALL_MARKER = '[END_CALL]';
|
|
31
49
|
|
|
50
|
+
function stripInternalSpeechMarkers(text: string): string {
|
|
51
|
+
return text
|
|
52
|
+
.replace(ASK_GUARDIAN_MARKER_REGEX, '')
|
|
53
|
+
.replace(USER_ANSWERED_MARKER_REGEX, '')
|
|
54
|
+
.replace(USER_INSTRUCTION_MARKER_REGEX, '')
|
|
55
|
+
.replace(CALL_OPENING_MARKER_REGEX, '')
|
|
56
|
+
.replace(CALL_OPENING_ACK_MARKER_REGEX, '')
|
|
57
|
+
.replace(END_CALL_MARKER_REGEX, '');
|
|
58
|
+
}
|
|
59
|
+
|
|
32
60
|
export class CallOrchestrator {
|
|
33
61
|
private callSessionId: string;
|
|
34
62
|
private relay: RelayConnection;
|
|
35
63
|
private state: OrchestratorState = 'idle';
|
|
36
64
|
private conversationHistory: Array<{ role: 'user' | 'assistant'; content: string }> = [];
|
|
37
65
|
private abortController: AbortController = new AbortController();
|
|
38
|
-
private callStartTime: number = Date.now();
|
|
39
66
|
private silenceTimer: ReturnType<typeof setTimeout> | null = null;
|
|
40
67
|
private durationTimer: ReturnType<typeof setTimeout> | null = null;
|
|
41
68
|
private durationWarningTimer: ReturnType<typeof setTimeout> | null = null;
|
|
42
69
|
private consultationTimer: ReturnType<typeof setTimeout> | null = null;
|
|
43
70
|
private durationEndTimer: ReturnType<typeof setTimeout> | null = null;
|
|
44
71
|
private task: string | null;
|
|
72
|
+
/** True when the call session was created via the inbound path (no outbound task). */
|
|
73
|
+
private isInbound: boolean;
|
|
45
74
|
/** Instructions queued while an LLM turn is in-flight or during waiting_on_user */
|
|
46
75
|
private pendingInstructions: string[] = [];
|
|
76
|
+
/** Ensures the call opener is triggered at most once per call. */
|
|
77
|
+
private initialGreetingStarted = false;
|
|
78
|
+
/** Marks that the next caller turn should be treated as an opening acknowledgment. */
|
|
79
|
+
private awaitingOpeningAck = false;
|
|
47
80
|
/** Monotonic run id used to suppress stale turn side effects after interruption. */
|
|
48
81
|
private llmRunVersion = 0;
|
|
49
|
-
|
|
50
|
-
|
|
82
|
+
/** Optional broadcast function for emitting IPC events to connected clients. */
|
|
83
|
+
private broadcast?: (msg: ServerMessage) => void;
|
|
84
|
+
/** Assistant identity for scoping guardian bindings. */
|
|
85
|
+
private assistantId: string;
|
|
86
|
+
/** Guardian trust context for the current caller, when available. */
|
|
87
|
+
private guardianContext: GuardianRuntimeContext | null;
|
|
88
|
+
|
|
89
|
+
constructor(
|
|
90
|
+
callSessionId: string,
|
|
91
|
+
relay: RelayConnection,
|
|
92
|
+
task: string | null,
|
|
93
|
+
opts?: {
|
|
94
|
+
broadcast?: (msg: ServerMessage) => void;
|
|
95
|
+
assistantId?: string;
|
|
96
|
+
guardianContext?: GuardianRuntimeContext;
|
|
97
|
+
},
|
|
98
|
+
) {
|
|
51
99
|
this.callSessionId = callSessionId;
|
|
52
100
|
this.relay = relay;
|
|
53
101
|
this.task = task;
|
|
102
|
+
this.isInbound = !task;
|
|
103
|
+
this.broadcast = opts?.broadcast;
|
|
104
|
+
this.assistantId = opts?.assistantId ?? 'self';
|
|
105
|
+
this.guardianContext = opts?.guardianContext ?? null;
|
|
54
106
|
this.startDurationTimer();
|
|
55
107
|
this.resetSilenceTimer();
|
|
56
108
|
registerCallOrchestrator(callSessionId, this);
|
|
@@ -63,6 +115,30 @@ export class CallOrchestrator {
|
|
|
63
115
|
return this.state;
|
|
64
116
|
}
|
|
65
117
|
|
|
118
|
+
/**
|
|
119
|
+
* Update guardian trust context for subsequent LLM turns.
|
|
120
|
+
*/
|
|
121
|
+
setGuardianContext(ctx: GuardianRuntimeContext | null): void {
|
|
122
|
+
this.guardianContext = ctx;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* Kick off the first outbound call utterance from the assistant.
|
|
127
|
+
*/
|
|
128
|
+
async startInitialGreeting(): Promise<void> {
|
|
129
|
+
if (this.initialGreetingStarted) return;
|
|
130
|
+
if (this.state !== 'idle') return;
|
|
131
|
+
|
|
132
|
+
this.initialGreetingStarted = true;
|
|
133
|
+
this.resetSilenceTimer();
|
|
134
|
+
this.conversationHistory.push({ role: 'user', content: CALL_OPENING_MARKER });
|
|
135
|
+
await this.runLlm();
|
|
136
|
+
const lastMessage = this.conversationHistory[this.conversationHistory.length - 1];
|
|
137
|
+
if (lastMessage?.role === 'assistant') {
|
|
138
|
+
this.awaitingOpeningAck = true;
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
|
|
66
142
|
/**
|
|
67
143
|
* Handle a final caller utterance from the ConversationRelay.
|
|
68
144
|
*/
|
|
@@ -74,9 +150,42 @@ export class CallOrchestrator {
|
|
|
74
150
|
this.abortController = new AbortController();
|
|
75
151
|
}
|
|
76
152
|
|
|
153
|
+
// Strip the one-shot [CALL_OPENING] marker from conversation history
|
|
154
|
+
// so it doesn't leak into subsequent LLM requests after barge-in.
|
|
155
|
+
// This runs unconditionally because the standard Twilio barge-in path
|
|
156
|
+
// calls handleInterrupt() first (setting state to 'idle') before
|
|
157
|
+
// handleCallerUtterance — so interruptedInFlight would be false even
|
|
158
|
+
// though an interrupt just occurred.
|
|
159
|
+
// Without this, the consecutive-user merge path below would append
|
|
160
|
+
// the caller's transcript to the synthetic "[CALL_OPENING]" message,
|
|
161
|
+
// causing the model to re-run opener behavior instead of responding
|
|
162
|
+
// directly to the caller.
|
|
163
|
+
// If the marker-only seed message becomes empty, remove it entirely:
|
|
164
|
+
// Anthropic rejects any user turn with empty content.
|
|
165
|
+
for (let i = 0; i < this.conversationHistory.length; i++) {
|
|
166
|
+
const entry = this.conversationHistory[i];
|
|
167
|
+
if (!entry.content.includes(CALL_OPENING_MARKER)) continue;
|
|
168
|
+
const stripped = entry.content.replace(CALL_OPENING_MARKER_REGEX, '').trim();
|
|
169
|
+
if (stripped.length === 0) {
|
|
170
|
+
this.conversationHistory.splice(i, 1);
|
|
171
|
+
i--;
|
|
172
|
+
} else {
|
|
173
|
+
entry.content = stripped;
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
|
|
77
177
|
this.state = 'processing';
|
|
78
178
|
this.resetSilenceTimer();
|
|
79
179
|
const callerContent = this.formatCallerUtterance(transcript, speaker);
|
|
180
|
+
const shouldMarkOpeningAck = this.awaitingOpeningAck;
|
|
181
|
+
if (shouldMarkOpeningAck) {
|
|
182
|
+
this.awaitingOpeningAck = false;
|
|
183
|
+
}
|
|
184
|
+
const callerTurnContent = shouldMarkOpeningAck
|
|
185
|
+
? callerContent.length > 0
|
|
186
|
+
? `${CALL_OPENING_ACK_MARKER}\n${callerContent}`
|
|
187
|
+
: CALL_OPENING_ACK_MARKER
|
|
188
|
+
: callerContent;
|
|
80
189
|
|
|
81
190
|
// Preserve strict role alternation for Anthropic. If the last message
|
|
82
191
|
// is already user-role (e.g. interrupted run never appended assistant,
|
|
@@ -84,11 +193,14 @@ export class CallOrchestrator {
|
|
|
84
193
|
// this utterance into that same user turn.
|
|
85
194
|
const lastMessage = this.conversationHistory[this.conversationHistory.length - 1];
|
|
86
195
|
if (lastMessage?.role === 'user') {
|
|
87
|
-
|
|
196
|
+
const existingContent = lastMessage.content.trim();
|
|
197
|
+
lastMessage.content = existingContent.length > 0
|
|
198
|
+
? `${lastMessage.content}\n${callerTurnContent}`
|
|
199
|
+
: callerTurnContent;
|
|
88
200
|
} else {
|
|
89
201
|
this.conversationHistory.push({
|
|
90
202
|
role: 'user',
|
|
91
|
-
content:
|
|
203
|
+
content: callerTurnContent,
|
|
92
204
|
});
|
|
93
205
|
}
|
|
94
206
|
|
|
@@ -209,30 +321,83 @@ export class CallOrchestrator {
|
|
|
209
321
|
|
|
210
322
|
// ── Private ──────────────────────────────────────────────────────
|
|
211
323
|
|
|
324
|
+
private buildGuardianPromptSection(): string[] {
|
|
325
|
+
if (!this.guardianContext) return [];
|
|
326
|
+
return [
|
|
327
|
+
'',
|
|
328
|
+
'GUARDIAN ACTOR CONTEXT (authoritative):',
|
|
329
|
+
buildGuardianContextBlock(this.guardianContext),
|
|
330
|
+
'- Treat `actor_role` as source-of-truth for whether this caller is the verified guardian.',
|
|
331
|
+
'- If `actor_role` is `guardian`, the current caller is verified for this assistant on voice.',
|
|
332
|
+
'- If `actor_role` is `non-guardian` or `unverified_channel`, do not imply the caller is verified.',
|
|
333
|
+
];
|
|
334
|
+
}
|
|
335
|
+
|
|
212
336
|
private buildSystemPrompt(): string {
|
|
213
337
|
const config = getConfig();
|
|
214
338
|
const disclosureRule = config.calls.disclosure.enabled
|
|
215
339
|
? `1. ${config.calls.disclosure.text}`
|
|
216
340
|
: '1. Begin the conversation naturally.';
|
|
217
341
|
|
|
342
|
+
if (this.isInbound) {
|
|
343
|
+
return this.buildInboundSystemPrompt(disclosureRule);
|
|
344
|
+
}
|
|
345
|
+
|
|
218
346
|
return [
|
|
219
347
|
`You are on a live phone call on behalf of ${resolveUserReference()}.`,
|
|
220
348
|
this.task ? `Task: ${this.task}` : '',
|
|
221
349
|
'',
|
|
222
350
|
'You are speaking directly to the person who answered the phone.',
|
|
223
351
|
'Respond naturally and conversationally — speak as you would in a real phone conversation.',
|
|
352
|
+
...this.buildGuardianPromptSection(),
|
|
224
353
|
'',
|
|
225
354
|
'IMPORTANT RULES:',
|
|
226
355
|
'0. When introducing yourself, refer to yourself as an assistant. Avoid the phrase "AI assistant" unless directly asked.',
|
|
227
356
|
disclosureRule,
|
|
228
357
|
'2. Be concise — phone conversations should be brief and natural.',
|
|
229
|
-
'3. If the callee asks something you don\'t know, include [
|
|
358
|
+
'3. If the callee asks something you don\'t know, include [ASK_GUARDIAN: your question here] in your response along with a hold message like "Let me check on that for you."',
|
|
230
359
|
'4. If the callee provides information preceded by [USER_ANSWERED: ...], use that answer naturally in the conversation.',
|
|
231
360
|
'5. If you see [USER_INSTRUCTION: ...], treat it as a high-priority steering directive from your user. Follow the instruction immediately, adjusting your approach or response accordingly.',
|
|
232
361
|
'6. When the call\'s purpose is fulfilled, include [END_CALL] in your response along with a polite goodbye.',
|
|
233
362
|
'7. Do not make up information — ask the user if unsure.',
|
|
234
363
|
'8. Keep responses short — 1-3 sentences is ideal for phone conversation.',
|
|
235
364
|
'9. When caller text includes [SPEAKER id="..." label="..."], treat each speaker as a distinct person and personalize responses using that speaker\'s prior context in this call.',
|
|
365
|
+
'10. If the latest user turn is [CALL_OPENING], generate a natural, context-specific opener: briefly introduce yourself once as an assistant, state why you are calling using the Task context, and ask a short permission/check-in question. Vary the wording; do not use a fixed template.',
|
|
366
|
+
'11. If the latest user turn includes [CALL_OPENING_ACK], treat it as the callee acknowledging your opener and continue the conversation naturally without re-introducing yourself or repeating the initial check-in question.',
|
|
367
|
+
'12. Do not repeat your introduction within the same call unless the callee explicitly asks who you are.',
|
|
368
|
+
]
|
|
369
|
+
.filter(Boolean)
|
|
370
|
+
.join('\n');
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
/**
|
|
374
|
+
* Build a system prompt tailored for inbound calls where the caller
|
|
375
|
+
* reached out to us. The assistant greets naturally and helps the
|
|
376
|
+
* caller with whatever they need, rather than delivering an outbound
|
|
377
|
+
* task message.
|
|
378
|
+
*/
|
|
379
|
+
private buildInboundSystemPrompt(disclosureRule: string): string {
|
|
380
|
+
return [
|
|
381
|
+
`You are on a live phone call, answering an incoming call on behalf of ${resolveUserReference()}.`,
|
|
382
|
+
'',
|
|
383
|
+
'The caller dialed in to reach you. You do not have a specific task — your role is to greet them warmly, find out what they need, and assist them.',
|
|
384
|
+
'Respond naturally and conversationally — speak as you would in a real phone conversation.',
|
|
385
|
+
...this.buildGuardianPromptSection(),
|
|
386
|
+
'',
|
|
387
|
+
'IMPORTANT RULES:',
|
|
388
|
+
'0. When introducing yourself, refer to yourself as an assistant. Avoid the phrase "AI assistant" unless directly asked.',
|
|
389
|
+
disclosureRule,
|
|
390
|
+
'2. Be concise — phone conversations should be brief and natural.',
|
|
391
|
+
'3. If the caller asks something you don\'t know or need to verify, include [ASK_GUARDIAN: your question here] in your response along with a hold message like "Let me check on that for you."',
|
|
392
|
+
'4. If information is provided preceded by [USER_ANSWERED: ...], use that answer naturally in the conversation.',
|
|
393
|
+
'5. If you see [USER_INSTRUCTION: ...], treat it as a high-priority steering directive from your user. Follow the instruction immediately, adjusting your approach or response accordingly.',
|
|
394
|
+
'6. When the caller indicates they are done or the conversation reaches a natural conclusion, include [END_CALL] in your response along with a polite goodbye.',
|
|
395
|
+
'7. Do not make up information — ask the user if unsure.',
|
|
396
|
+
'8. Keep responses short — 1-3 sentences is ideal for phone conversation.',
|
|
397
|
+
'9. When caller text includes [SPEAKER id="..." label="..."], treat each speaker as a distinct person and personalize responses using that speaker\'s prior context in this call.',
|
|
398
|
+
'10. If the latest user turn is [CALL_OPENING], greet the caller warmly and ask how you can help. For example: "Hello, this is [name]\'s assistant. How can I help you today?" Vary the wording; do not use a fixed template.',
|
|
399
|
+
'11. If the latest user turn includes [CALL_OPENING_ACK], treat it as the caller acknowledging your greeting and continue the conversation naturally.',
|
|
400
|
+
'12. Do not repeat your introduction within the same call unless the caller explicitly asks who you are.',
|
|
236
401
|
]
|
|
237
402
|
.filter(Boolean)
|
|
238
403
|
.join('\n');
|
|
@@ -242,7 +407,7 @@ export class CallOrchestrator {
|
|
|
242
407
|
if (!speaker) return transcript;
|
|
243
408
|
const safeId = speaker.speakerId.replaceAll('"', '\'');
|
|
244
409
|
const safeLabel = speaker.speakerLabel.replaceAll('"', '\'');
|
|
245
|
-
const confidencePart = speaker.speakerConfidence
|
|
410
|
+
const confidencePart = speaker.speakerConfidence != null ? ` confidence="${speaker.speakerConfidence.toFixed(2)}"` : '';
|
|
246
411
|
return `[SPEAKER id="${safeId}" label="${safeLabel}" source="${speaker.source}"${confidencePart}] ${transcript}`;
|
|
247
412
|
}
|
|
248
413
|
|
|
@@ -251,37 +416,31 @@ export class CallOrchestrator {
|
|
|
251
416
|
* the response back through the relay.
|
|
252
417
|
*/
|
|
253
418
|
private async runLlm(): Promise<void> {
|
|
254
|
-
const
|
|
255
|
-
|
|
256
|
-
|
|
419
|
+
const config = getConfig();
|
|
420
|
+
const resolved = resolveConfiguredProvider();
|
|
421
|
+
if (!resolved) {
|
|
422
|
+
log.error({ callSessionId: this.callSessionId }, 'No provider available');
|
|
257
423
|
this.relay.sendTextToken('I\'m sorry, I\'m having a technical issue. Please try again later.', true);
|
|
258
424
|
this.state = 'idle';
|
|
259
425
|
return;
|
|
260
426
|
}
|
|
427
|
+
const { provider } = resolved;
|
|
261
428
|
|
|
262
|
-
const client = new Anthropic({ apiKey });
|
|
263
429
|
const runVersion = ++this.llmRunVersion;
|
|
264
430
|
const runSignal = this.abortController.signal;
|
|
265
431
|
|
|
266
432
|
try {
|
|
267
433
|
this.state = 'speaking';
|
|
268
434
|
|
|
269
|
-
|
|
435
|
+
// Only override the model when the user has explicitly configured one
|
|
436
|
+
// AND the selected provider matches the configured provider. Forwarding
|
|
437
|
+
// a provider-specific model to a fallback provider would cause
|
|
438
|
+
// cross-provider 4xx errors (e.g., sending "gpt-5.2" to Anthropic).
|
|
439
|
+
const callModel = !resolved.usedFallbackPrimary
|
|
440
|
+
? (config.calls.model?.trim() || undefined)
|
|
441
|
+
: undefined;
|
|
270
442
|
|
|
271
|
-
|
|
272
|
-
{
|
|
273
|
-
model: callModel,
|
|
274
|
-
max_tokens: 512,
|
|
275
|
-
system: this.buildSystemPrompt(),
|
|
276
|
-
messages: this.conversationHistory.map((m) => ({
|
|
277
|
-
role: m.role,
|
|
278
|
-
content: m.content,
|
|
279
|
-
})),
|
|
280
|
-
},
|
|
281
|
-
{ signal: runSignal },
|
|
282
|
-
);
|
|
283
|
-
|
|
284
|
-
// Buffer incoming tokens so we can strip control markers ([ASK_USER:...], [END_CALL])
|
|
443
|
+
// Buffer incoming tokens so we can strip control markers ([ASK_GUARDIAN:...], [END_CALL])
|
|
285
444
|
// before they reach TTS. We hold text whenever an unmatched '[' appears, since it
|
|
286
445
|
// could be the start of a control marker.
|
|
287
446
|
let ttsBuffer = '';
|
|
@@ -308,14 +467,24 @@ export class CallOrchestrator {
|
|
|
308
467
|
// The check must be bidirectional:
|
|
309
468
|
// - When the buffer is shorter than the prefix (e.g. "[ASK"), the
|
|
310
469
|
// buffer is a prefix of the control tag → hold it.
|
|
311
|
-
// - When the buffer is longer than the prefix (e.g. "[
|
|
470
|
+
// - When the buffer is longer than the prefix (e.g. "[ASK_GUARDIAN: what"),
|
|
312
471
|
// the buffer starts with the control tag prefix → hold it (the
|
|
313
472
|
// variable-length payload hasn't been closed yet).
|
|
314
473
|
const afterBracket = ttsBuffer;
|
|
315
474
|
const couldBeControl =
|
|
316
|
-
'[
|
|
475
|
+
'[ASK_GUARDIAN:'.startsWith(afterBracket) ||
|
|
476
|
+
'[USER_ANSWERED:'.startsWith(afterBracket) ||
|
|
477
|
+
'[USER_INSTRUCTION:'.startsWith(afterBracket) ||
|
|
478
|
+
'[CALL_OPENING]'.startsWith(afterBracket) ||
|
|
479
|
+
'[CALL_OPENING_ACK]'.startsWith(afterBracket) ||
|
|
317
480
|
'[END_CALL]'.startsWith(afterBracket) ||
|
|
318
|
-
afterBracket.startsWith('[
|
|
481
|
+
afterBracket.startsWith('[ASK_GUARDIAN:') ||
|
|
482
|
+
afterBracket.startsWith('[USER_ANSWERED:') ||
|
|
483
|
+
afterBracket.startsWith('[USER_INSTRUCTION:') ||
|
|
484
|
+
afterBracket === '[CALL_OPENING' ||
|
|
485
|
+
afterBracket.startsWith('[CALL_OPENING]') ||
|
|
486
|
+
afterBracket === '[CALL_OPENING_ACK' ||
|
|
487
|
+
afterBracket.startsWith('[CALL_OPENING_ACK]') ||
|
|
319
488
|
afterBracket === '[END_CALL' ||
|
|
320
489
|
afterBracket.startsWith('[END_CALL]');
|
|
321
490
|
|
|
@@ -335,26 +504,33 @@ export class CallOrchestrator {
|
|
|
335
504
|
}
|
|
336
505
|
};
|
|
337
506
|
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
507
|
+
const response = await provider.sendMessage(
|
|
508
|
+
this.conversationHistory.map((m) => ({
|
|
509
|
+
role: m.role as 'user' | 'assistant',
|
|
510
|
+
content: [{ type: 'text' as const, text: m.content }],
|
|
511
|
+
})),
|
|
512
|
+
[], // no tools
|
|
513
|
+
this.buildSystemPrompt(),
|
|
514
|
+
{
|
|
515
|
+
config: {
|
|
516
|
+
...(callModel ? { model: callModel } : {}),
|
|
517
|
+
max_tokens: 512,
|
|
518
|
+
},
|
|
519
|
+
onEvent: (event: ProviderEvent) => {
|
|
520
|
+
if (!this.isCurrentRun(runVersion)) return;
|
|
521
|
+
if (event.type === 'text_delta') {
|
|
522
|
+
ttsBuffer += event.text;
|
|
523
|
+
ttsBuffer = stripInternalSpeechMarkers(ttsBuffer);
|
|
524
|
+
flushSafeText(false);
|
|
525
|
+
}
|
|
526
|
+
},
|
|
527
|
+
signal: runSignal,
|
|
528
|
+
},
|
|
529
|
+
);
|
|
354
530
|
if (!this.isCurrentRun(runVersion)) return;
|
|
355
531
|
|
|
356
532
|
// Final sweep: strip any remaining control markers from the buffer
|
|
357
|
-
ttsBuffer = ttsBuffer
|
|
533
|
+
ttsBuffer = stripInternalSpeechMarkers(ttsBuffer);
|
|
358
534
|
if (ttsBuffer.length > 0) {
|
|
359
535
|
this.relay.sendTextToken(ttsBuffer, false);
|
|
360
536
|
}
|
|
@@ -362,28 +538,35 @@ export class CallOrchestrator {
|
|
|
362
538
|
// Signal end of this turn's speech
|
|
363
539
|
this.relay.sendTextToken('', true);
|
|
364
540
|
|
|
365
|
-
const responseText =
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
.join('') || '';
|
|
541
|
+
const responseText = response.content
|
|
542
|
+
.filter((b): b is { type: 'text'; text: string } => b.type === 'text')
|
|
543
|
+
.map((b) => b.text)
|
|
544
|
+
.join('') || '';
|
|
370
545
|
|
|
371
546
|
// Record the assistant response
|
|
372
547
|
this.conversationHistory.push({ role: 'assistant', content: responseText });
|
|
373
548
|
recordCallEvent(this.callSessionId, 'assistant_spoke', { text: responseText });
|
|
374
|
-
const spokenText = responseText
|
|
549
|
+
const spokenText = stripInternalSpeechMarkers(responseText).trim();
|
|
375
550
|
if (spokenText.length > 0) {
|
|
376
551
|
const session = getCallSession(this.callSessionId);
|
|
377
552
|
if (session) {
|
|
553
|
+
// Persist assistant transcript to the voice conversation so it
|
|
554
|
+
// survives even when no live daemon Session is listening.
|
|
555
|
+
conversationStore.addMessage(
|
|
556
|
+
session.conversationId,
|
|
557
|
+
'assistant',
|
|
558
|
+
JSON.stringify([{ type: 'text', text: spokenText }]),
|
|
559
|
+
{ userMessageChannel: 'voice', assistantMessageChannel: 'voice' },
|
|
560
|
+
);
|
|
378
561
|
fireCallTranscriptNotifier(session.conversationId, this.callSessionId, 'assistant', spokenText);
|
|
379
562
|
}
|
|
380
563
|
}
|
|
381
564
|
|
|
382
|
-
// Check for
|
|
383
|
-
const askMatch = responseText.match(
|
|
565
|
+
// Check for ASK_GUARDIAN pattern
|
|
566
|
+
const askMatch = responseText.match(ASK_GUARDIAN_CAPTURE_REGEX);
|
|
384
567
|
if (askMatch) {
|
|
385
568
|
const questionText = askMatch[1];
|
|
386
|
-
createPendingQuestion(this.callSessionId, questionText);
|
|
569
|
+
const pendingQuestion = createPendingQuestion(this.callSessionId, questionText);
|
|
387
570
|
this.state = 'waiting_on_user';
|
|
388
571
|
updateCallSession(this.callSessionId, { status: 'waiting_on_user' });
|
|
389
572
|
recordCallEvent(this.callSessionId, 'user_question_asked', { question: questionText });
|
|
@@ -392,6 +575,15 @@ export class CallOrchestrator {
|
|
|
392
575
|
const session = getCallSession(this.callSessionId);
|
|
393
576
|
if (session) {
|
|
394
577
|
fireCallQuestionNotifier(session.conversationId, this.callSessionId, questionText);
|
|
578
|
+
|
|
579
|
+
// Dispatch guardian action request to all configured channels
|
|
580
|
+
void dispatchGuardianQuestion({
|
|
581
|
+
callSessionId: this.callSessionId,
|
|
582
|
+
conversationId: session.conversationId,
|
|
583
|
+
assistantId: this.assistantId,
|
|
584
|
+
pendingQuestion,
|
|
585
|
+
broadcast: this.broadcast,
|
|
586
|
+
});
|
|
395
587
|
}
|
|
396
588
|
|
|
397
589
|
// Set a consultation timeout
|
|
@@ -422,11 +614,19 @@ export class CallOrchestrator {
|
|
|
422
614
|
updateCallSession(this.callSessionId, { status: 'completed', endedAt: Date.now() });
|
|
423
615
|
recordCallEvent(this.callSessionId, 'call_ended', { reason: 'completed' });
|
|
424
616
|
|
|
425
|
-
// Notify the conversation
|
|
426
|
-
// into a terminal call state.
|
|
617
|
+
// Notify the voice conversation
|
|
427
618
|
if (shouldNotifyCompletion && currentSession) {
|
|
619
|
+
persistCallCompletionMessage(currentSession.conversationId, this.callSessionId);
|
|
428
620
|
fireCallCompletionNotifier(currentSession.conversationId, this.callSessionId);
|
|
429
621
|
}
|
|
622
|
+
|
|
623
|
+
// Post a pointer message in the initiating conversation
|
|
624
|
+
if (currentSession?.initiatedFromConversationId) {
|
|
625
|
+
const durationMs = currentSession.startedAt ? Date.now() - currentSession.startedAt : 0;
|
|
626
|
+
addPointerMessage(currentSession.initiatedFromConversationId, 'completed', currentSession.toNumber, {
|
|
627
|
+
duration: durationMs > 0 ? formatDuration(durationMs) : undefined,
|
|
628
|
+
});
|
|
629
|
+
}
|
|
430
630
|
this.state = 'idle';
|
|
431
631
|
return;
|
|
432
632
|
}
|
|
@@ -534,8 +734,17 @@ export class CallOrchestrator {
|
|
|
534
734
|
updateCallSession(this.callSessionId, { status: 'completed', endedAt: Date.now() });
|
|
535
735
|
recordCallEvent(this.callSessionId, 'call_ended', { reason: 'max_duration' });
|
|
536
736
|
if (shouldNotifyCompletion && currentSession) {
|
|
737
|
+
persistCallCompletionMessage(currentSession.conversationId, this.callSessionId);
|
|
537
738
|
fireCallCompletionNotifier(currentSession.conversationId, this.callSessionId);
|
|
538
739
|
}
|
|
740
|
+
|
|
741
|
+
// Post a pointer message in the initiating conversation
|
|
742
|
+
if (currentSession?.initiatedFromConversationId) {
|
|
743
|
+
const durationMs = currentSession.startedAt ? Date.now() - currentSession.startedAt : 0;
|
|
744
|
+
addPointerMessage(currentSession.initiatedFromConversationId, 'completed', currentSession.toNumber, {
|
|
745
|
+
duration: durationMs > 0 ? formatDuration(durationMs) : undefined,
|
|
746
|
+
});
|
|
747
|
+
}
|
|
539
748
|
}, 3000);
|
|
540
749
|
}, maxDurationMs);
|
|
541
750
|
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Concise pointer/status messages posted to the initiating conversation
|
|
3
|
+
* so the user sees call lifecycle events without the full transcript
|
|
4
|
+
* (which lives in the dedicated voice conversation).
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import * as conversationStore from '../memory/conversation-store.js';
|
|
8
|
+
|
|
9
|
+
export type PointerEvent = 'started' | 'completed' | 'failed';
|
|
10
|
+
|
|
11
|
+
export function addPointerMessage(
|
|
12
|
+
conversationId: string,
|
|
13
|
+
event: PointerEvent,
|
|
14
|
+
phoneNumber: string,
|
|
15
|
+
extra?: { duration?: string; reason?: string; verificationCode?: string },
|
|
16
|
+
): void {
|
|
17
|
+
let text: string;
|
|
18
|
+
switch (event) {
|
|
19
|
+
case 'started':
|
|
20
|
+
text = extra?.verificationCode
|
|
21
|
+
? `\u{1F4DE} Call to ${phoneNumber} started. Verification code: ${extra.verificationCode}`
|
|
22
|
+
: `\u{1F4DE} Call to ${phoneNumber} started. See voice thread for details.`;
|
|
23
|
+
break;
|
|
24
|
+
case 'completed':
|
|
25
|
+
text = extra?.duration
|
|
26
|
+
? `\u{1F4DE} Call to ${phoneNumber} completed (${extra.duration}).`
|
|
27
|
+
: `\u{1F4DE} Call to ${phoneNumber} completed.`;
|
|
28
|
+
break;
|
|
29
|
+
case 'failed':
|
|
30
|
+
text = extra?.reason
|
|
31
|
+
? `\u{1F4DE} Call to ${phoneNumber} failed: ${extra.reason}.`
|
|
32
|
+
: `\u{1F4DE} Call to ${phoneNumber} failed.`;
|
|
33
|
+
break;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
conversationStore.addMessage(
|
|
37
|
+
conversationId,
|
|
38
|
+
'assistant',
|
|
39
|
+
JSON.stringify([{ type: 'text', text }]),
|
|
40
|
+
{ userMessageChannel: 'voice', assistantMessageChannel: 'voice' },
|
|
41
|
+
);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Format a duration in milliseconds into a human-friendly string.
|
|
46
|
+
*/
|
|
47
|
+
export function formatDuration(ms: number): string {
|
|
48
|
+
const totalSeconds = Math.round(ms / 1000);
|
|
49
|
+
if (totalSeconds < 60) return `${totalSeconds}s`;
|
|
50
|
+
const minutes = Math.floor(totalSeconds / 60);
|
|
51
|
+
const seconds = totalSeconds % 60;
|
|
52
|
+
return seconds > 0 ? `${minutes}m ${seconds}s` : `${minutes}m`;
|
|
53
|
+
}
|
|
@@ -8,9 +8,10 @@
|
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
10
|
import { getLogger } from '../util/logger.js';
|
|
11
|
+
import { isTerminalState } from './call-state-machine.js';
|
|
11
12
|
import { listRecoverableCalls, updateCallSession, expirePendingQuestions } from './call-store.js';
|
|
12
|
-
import type { VoiceProvider } from './voice-provider.js';
|
|
13
13
|
import type { CallStatus } from './types.js';
|
|
14
|
+
import type { VoiceProvider } from './voice-provider.js';
|
|
14
15
|
|
|
15
16
|
type Logger = ReturnType<typeof getLogger>;
|
|
16
17
|
|
|
@@ -57,12 +58,6 @@ function mapProviderStatus(providerStatus: string): CallStatus | null {
|
|
|
57
58
|
}
|
|
58
59
|
}
|
|
59
60
|
|
|
60
|
-
/**
|
|
61
|
-
* Check whether a CallStatus is terminal (no further transitions allowed).
|
|
62
|
-
*/
|
|
63
|
-
function isTerminal(status: CallStatus): boolean {
|
|
64
|
-
return status === 'completed' || status === 'failed' || status === 'cancelled';
|
|
65
|
-
}
|
|
66
61
|
|
|
67
62
|
/**
|
|
68
63
|
* Reconcile all non-terminal call sessions at daemon startup.
|
|
@@ -159,7 +154,7 @@ export async function reconcileCallsOnStartup(
|
|
|
159
154
|
continue;
|
|
160
155
|
}
|
|
161
156
|
|
|
162
|
-
if (
|
|
157
|
+
if (isTerminalState(mappedStatus)) {
|
|
163
158
|
// Provider says the call has ended
|
|
164
159
|
log.info(
|
|
165
160
|
{ callSessionId: session.id, providerStatus, mappedStatus },
|