@vellumai/assistant 0.3.5 → 0.3.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +51 -0
- package/eslint.config.mjs +31 -0
- package/package.json +1 -1
- package/scripts/ipc/check-swift-decoder-drift.ts +4 -1
- package/scripts/ipc/generate-swift.ts +18 -2
- package/src/__tests__/__snapshots__/ipc-snapshot.test.ts.snap +338 -1
- package/src/__tests__/approval-conversation-turn.test.ts +214 -0
- package/src/__tests__/browser-manager.test.ts +1 -0
- package/src/__tests__/call-conversation-messages.test.ts +130 -0
- package/src/__tests__/call-orchestrator.test.ts +752 -271
- package/src/__tests__/call-pointer-messages.test.ts +148 -0
- package/src/__tests__/call-recovery.test.ts +3 -0
- package/src/__tests__/call-routes-http.test.ts +5 -0
- package/src/__tests__/call-store.test.ts +3 -0
- package/src/__tests__/channel-approval-routes.test.ts +1260 -85
- package/src/__tests__/channel-approval.test.ts +37 -0
- package/src/__tests__/channel-approvals.test.ts +4 -65
- package/src/__tests__/channel-guardian.test.ts +556 -0
- package/src/__tests__/channel-readiness-service.test.ts +74 -7
- package/src/__tests__/checker.test.ts +14 -7
- package/src/__tests__/clarification-resolver.test.ts +44 -24
- package/src/__tests__/commit-message-enrichment-service.test.ts +9 -4
- package/src/__tests__/computer-use-session-working-dir.test.ts +8 -0
- package/src/__tests__/config-schema.test.ts +12 -7
- package/src/__tests__/context-window-manager.test.ts +30 -2
- package/src/__tests__/contradiction-checker.test.ts +20 -5
- package/src/__tests__/credential-security-invariants.test.ts +6 -2
- package/src/__tests__/db-migration-rollback.test.ts +752 -0
- package/src/__tests__/dynamic-skill-workflow-prompt.test.ts +2 -0
- package/src/__tests__/fuzzy-match-property.test.ts +5 -5
- package/src/__tests__/guardian-action-store.test.ts +123 -0
- package/src/__tests__/guardian-action-sweep.test.ts +277 -0
- package/src/__tests__/guardian-dispatch.test.ts +389 -0
- package/src/__tests__/guardian-question-copy.test.ts +47 -0
- package/src/__tests__/handlers-telegram-config.test.ts +4 -2
- package/src/__tests__/handlers-twilio-config.test.ts +126 -0
- package/src/__tests__/intent-routing.test.ts +2 -0
- package/src/__tests__/ipc-snapshot.test.ts +228 -1
- package/src/__tests__/memory-upsert-concurrency.test.ts +828 -0
- package/src/__tests__/model-intents.test.ts +96 -0
- package/src/__tests__/no-direct-anthropic-sdk-imports.test.ts +42 -0
- package/src/__tests__/oauth2-gateway-transport.test.ts +130 -0
- package/src/__tests__/onboarding-starter-tasks.test.ts +2 -0
- package/src/__tests__/provider-commit-message-generator.test.ts +89 -13
- package/src/__tests__/provider-error-scenarios.test.ts +621 -0
- package/src/__tests__/provider-fail-open-selection.test.ts +119 -0
- package/src/__tests__/qdrant-manager.test.ts +27 -20
- package/src/__tests__/relay-server.test.ts +779 -40
- package/src/__tests__/run-orchestrator-assistant-events.test.ts +2 -0
- package/src/__tests__/run-orchestrator.test.ts +20 -4
- package/src/__tests__/runtime-runs-http.test.ts +17 -1
- package/src/__tests__/runtime-runs.test.ts +16 -0
- package/src/__tests__/schedule-store.test.ts +18 -4
- package/src/__tests__/scheduler-recurrence.test.ts +13 -4
- package/src/__tests__/session-abort-tool-results.test.ts +6 -0
- package/src/__tests__/session-agent-loop.test.ts +857 -0
- package/src/__tests__/session-conflict-gate.test.ts +6 -0
- package/src/__tests__/session-pre-run-repair.test.ts +6 -0
- package/src/__tests__/session-profile-injection.test.ts +6 -0
- package/src/__tests__/session-provider-retry-repair.test.ts +6 -0
- package/src/__tests__/session-queue.test.ts +6 -0
- package/src/__tests__/session-runtime-assembly.test.ts +237 -13
- package/src/__tests__/session-slash-known.test.ts +6 -0
- package/src/__tests__/session-slash-queue.test.ts +6 -0
- package/src/__tests__/session-slash-unknown.test.ts +6 -0
- package/src/__tests__/session-surfaces-task-progress.test.ts +2 -0
- package/src/__tests__/session-tool-setup-app-refresh.test.ts +1 -0
- package/src/__tests__/session-tool-setup-memory-scope.test.ts +1 -0
- package/src/__tests__/session-tool-setup-side-effect-flag.test.ts +1 -0
- package/src/__tests__/session-workspace-injection.test.ts +6 -0
- package/src/__tests__/session-workspace-tool-tracking.test.ts +6 -0
- package/src/__tests__/skills.test.ts +2 -0
- package/src/__tests__/sms-messaging-provider.test.ts +2 -1
- package/src/__tests__/starter-task-flow.test.ts +2 -0
- package/src/__tests__/swarm-dag-pathological.test.ts +535 -0
- package/src/__tests__/system-prompt.test.ts +2 -0
- package/src/__tests__/task-management-tools.test.ts +2 -2
- package/src/__tests__/task-runner.test.ts +14 -4
- package/src/__tests__/terminal-tools.test.ts +25 -19
- package/src/__tests__/tool-execution-abort-cleanup.test.ts +545 -0
- package/src/__tests__/tool-executor-shell-integration.test.ts +11 -11
- package/src/__tests__/tool-executor.test.ts +23 -24
- package/src/__tests__/trust-store.test.ts +3 -3
- package/src/__tests__/twilio-rest.test.ts +29 -0
- package/src/__tests__/twilio-routes-elevenlabs.test.ts +3 -0
- package/src/__tests__/twilio-routes-twiml.test.ts +11 -0
- package/src/__tests__/twilio-routes.test.ts +141 -21
- package/src/__tests__/user-reference.test.ts +2 -0
- package/src/__tests__/voice-quality.test.ts +222 -0
- package/src/__tests__/web-search.test.ts +45 -29
- package/src/agent/loop.ts +1 -1
- package/src/agent-heartbeat/agent-heartbeat-service.ts +2 -10
- package/src/amazon/client.ts +1418 -0
- package/src/amazon/request-extractor.ts +135 -0
- package/src/amazon/session.ts +109 -0
- package/src/autonomy/autonomy-store.ts +5 -5
- package/src/browser-extension-relay/client.ts +124 -0
- package/src/browser-extension-relay/protocol.ts +63 -0
- package/src/browser-extension-relay/server.ts +177 -0
- package/src/bundler/app-bundler.ts +3 -3
- package/src/bundler/bundle-signer.ts +1 -1
- package/src/bundler/signature-verifier.ts +1 -1
- package/src/calls/call-conversation-messages.ts +33 -0
- package/src/calls/call-domain.ts +106 -5
- package/src/calls/call-orchestrator.ts +252 -54
- package/src/calls/call-pointer-messages.ts +53 -0
- package/src/calls/call-recovery.ts +3 -8
- package/src/calls/call-store.ts +69 -87
- package/src/calls/elevenlabs-config.ts +3 -2
- package/src/calls/guardian-action-sweep.ts +105 -0
- package/src/calls/guardian-dispatch.ts +203 -0
- package/src/calls/guardian-question-copy.ts +133 -0
- package/src/calls/relay-server.ts +466 -8
- package/src/calls/speaker-identification.ts +1 -1
- package/src/calls/twilio-config.ts +7 -5
- package/src/calls/twilio-provider.ts +6 -4
- package/src/calls/twilio-rest.ts +40 -15
- package/src/calls/twilio-routes.ts +60 -45
- package/src/calls/types.ts +3 -1
- package/src/channels/types.ts +25 -0
- package/src/cli/amazon.ts +815 -0
- package/src/cli/config-commands.ts +2 -2
- package/src/cli/core-commands.ts +4 -3
- package/src/cli/influencer.ts +244 -0
- package/src/cli/map.ts +89 -6
- package/src/cli.ts +1 -1
- package/src/config/agent-schema.ts +171 -0
- package/src/config/bundled-skills/amazon/SKILL.md +127 -0
- package/src/config/bundled-skills/amazon/icon.svg +13 -0
- package/src/config/bundled-skills/api-mapping/SKILL.md +78 -0
- package/src/config/bundled-skills/browser/SKILL.md +1 -0
- package/src/config/bundled-skills/browser/TOOLS.json +17 -0
- package/src/config/bundled-skills/browser/tools/browser-wait-for-download.ts +25 -0
- package/src/config/bundled-skills/doordash/SKILL.md +51 -51
- package/src/config/bundled-skills/email-setup/SKILL.md +14 -5
- package/src/config/bundled-skills/google-oauth-setup/SKILL.md +183 -0
- package/src/config/bundled-skills/influencer/SKILL.md +144 -0
- package/src/config/bundled-skills/macos-automation/icon.svg +12 -0
- package/src/config/bundled-skills/media-processing/SKILL.md +72 -95
- package/src/config/bundled-skills/media-processing/TOOLS.json +57 -147
- package/src/config/bundled-skills/media-processing/__tests__/concurrency-pool.test.ts +77 -0
- package/src/config/bundled-skills/media-processing/__tests__/cost-tracker.test.ts +69 -0
- package/src/config/bundled-skills/media-processing/__tests__/preprocess.test.ts +303 -0
- package/src/config/bundled-skills/media-processing/services/concurrency-pool.ts +55 -0
- package/src/config/bundled-skills/media-processing/services/cost-tracker.ts +86 -0
- package/src/config/bundled-skills/media-processing/services/gemini-map.ts +339 -0
- package/src/config/bundled-skills/media-processing/services/preprocess.ts +551 -0
- package/src/config/bundled-skills/media-processing/services/processing-pipeline.ts +7 -9
- package/src/config/bundled-skills/media-processing/services/reduce.ts +197 -0
- package/src/config/bundled-skills/media-processing/tools/analyze-keyframes.ts +88 -253
- package/src/config/bundled-skills/media-processing/tools/extract-keyframes.ts +22 -153
- package/src/config/bundled-skills/media-processing/tools/generate-clip.ts +2 -2
- package/src/config/bundled-skills/media-processing/tools/media-diagnostics.ts +28 -51
- package/src/config/bundled-skills/media-processing/tools/query-media-events.ts +35 -270
- package/src/config/bundled-skills/messaging/SKILL.md +12 -2
- package/src/config/bundled-skills/messaging/tools/messaging-analyze-style.ts +4 -7
- package/src/config/bundled-skills/messaging/tools/messaging-reply.ts +2 -1
- package/src/config/bundled-skills/phone-calls/SKILL.md +86 -21
- package/src/config/bundled-skills/twitter/icon.svg +14 -0
- package/src/config/bundled-tool-registry.ts +310 -0
- package/src/config/calls-schema.ts +181 -0
- package/src/config/core-schema.ts +309 -0
- package/src/config/defaults.ts +27 -3
- package/src/config/env-registry.ts +169 -0
- package/src/config/env.ts +175 -0
- package/src/config/loader.ts +6 -6
- package/src/config/memory-schema.ts +528 -0
- package/src/config/sandbox-schema.ts +55 -0
- package/src/config/schema.ts +157 -1138
- package/src/config/skill-state.ts +1 -1
- package/src/config/skills-schema.ts +32 -0
- package/src/config/skills.ts +35 -24
- package/src/config/system-prompt.ts +107 -56
- package/src/config/templates/SOUL.md +1 -1
- package/src/config/types.ts +1 -0
- package/src/config/user-reference.ts +4 -9
- package/src/config/vellum-skills/catalog.json +0 -7
- package/src/config/vellum-skills/chatgpt-import/tools/chatgpt-import.ts +5 -1
- package/src/config/vellum-skills/slack-oauth-setup/SKILL.md +1 -0
- package/src/config/vellum-skills/sms-setup/SKILL.md +112 -14
- package/src/context/window-manager.ts +27 -7
- package/src/daemon/approval-generators.ts +186 -0
- package/src/daemon/approved-devices-store.ts +140 -0
- package/src/daemon/assistant-attachments.ts +1 -1
- package/src/daemon/classifier.ts +35 -32
- package/src/daemon/config-watcher.ts +1 -1
- package/src/daemon/daemon-control.ts +254 -0
- package/src/daemon/handlers/apps.ts +2 -3
- package/src/daemon/handlers/config-channels.ts +158 -0
- package/src/daemon/handlers/config-inbox.ts +540 -0
- package/src/daemon/handlers/config-ingress.ts +231 -0
- package/src/daemon/handlers/config-integrations.ts +258 -0
- package/src/daemon/handlers/config-model.ts +143 -0
- package/src/daemon/handlers/config-parental.ts +163 -0
- package/src/daemon/handlers/config-scheduling.ts +172 -0
- package/src/daemon/handlers/config-slack.ts +92 -0
- package/src/daemon/handlers/config-telegram.ts +301 -0
- package/src/daemon/handlers/config-tools.ts +177 -0
- package/src/daemon/handlers/config-trust.ts +104 -0
- package/src/daemon/handlers/config-twilio.ts +1080 -0
- package/src/daemon/handlers/config.ts +53 -2463
- package/src/daemon/handlers/diagnostics.ts +1 -1
- package/src/daemon/handlers/dictation.ts +4 -6
- package/src/daemon/handlers/documents.ts +18 -32
- package/src/daemon/handlers/index.ts +9 -0
- package/src/daemon/handlers/misc.ts +3 -5
- package/src/daemon/handlers/pairing.ts +98 -0
- package/src/daemon/handlers/sessions.ts +74 -5
- package/src/daemon/handlers/shared.ts +3 -1
- package/src/daemon/handlers/skills.ts +1 -1
- package/src/daemon/handlers/twitter-auth.ts +2 -0
- package/src/daemon/handlers/work-items.ts +2 -2
- package/src/daemon/handlers/workspace-files.ts +4 -3
- package/src/daemon/install-cli-launchers.ts +113 -0
- package/src/daemon/ipc-contract/apps.ts +356 -0
- package/src/daemon/ipc-contract/browser.ts +74 -0
- package/src/daemon/ipc-contract/computer-use.ts +151 -0
- package/src/daemon/ipc-contract/diagnostics.ts +56 -0
- package/src/daemon/ipc-contract/documents.ts +74 -0
- package/src/daemon/ipc-contract/inbox.ts +209 -0
- package/src/daemon/ipc-contract/integrations.ts +284 -0
- package/src/daemon/ipc-contract/memory.ts +48 -0
- package/src/daemon/ipc-contract/messages.ts +211 -0
- package/src/daemon/ipc-contract/pairing.ts +45 -0
- package/src/daemon/ipc-contract/parental-control.ts +95 -0
- package/src/daemon/ipc-contract/schedules.ts +97 -0
- package/src/daemon/ipc-contract/sessions.ts +321 -0
- package/src/daemon/ipc-contract/shared.ts +42 -0
- package/src/daemon/ipc-contract/skills.ts +120 -0
- package/src/daemon/ipc-contract/subagents.ts +58 -0
- package/src/daemon/ipc-contract/surfaces.ts +250 -0
- package/src/daemon/ipc-contract/trust.ts +60 -0
- package/src/daemon/ipc-contract/work-items.ts +225 -0
- package/src/daemon/ipc-contract/workspace.ts +113 -0
- package/src/daemon/ipc-contract-inventory.json +62 -0
- package/src/daemon/ipc-contract-inventory.ts +55 -29
- package/src/daemon/ipc-contract.ts +227 -2527
- package/src/daemon/ipc-protocol.ts +1 -1
- package/src/daemon/ipc-validate.ts +7 -0
- package/src/daemon/lifecycle.ts +97 -379
- package/src/daemon/pairing-store.ts +177 -0
- package/src/daemon/providers-setup.ts +43 -0
- package/src/daemon/ride-shotgun-handler.ts +67 -2
- package/src/daemon/server.ts +60 -44
- package/src/daemon/session-agent-loop-handlers.ts +421 -0
- package/src/daemon/session-agent-loop.ts +113 -275
- package/src/daemon/session-dynamic-profile.ts +1 -1
- package/src/daemon/session-history.ts +1 -1
- package/src/daemon/session-media-retry.ts +1 -1
- package/src/daemon/session-messaging.ts +37 -2
- package/src/daemon/session-notifiers.ts +5 -25
- package/src/daemon/session-process.ts +99 -59
- package/src/daemon/session-queue-manager.ts +98 -4
- package/src/daemon/session-runtime-assembly.ts +149 -15
- package/src/daemon/session-surfaces.ts +26 -4
- package/src/daemon/session-tool-setup.ts +28 -30
- package/src/daemon/session-workspace.ts +1 -1
- package/src/daemon/session.ts +24 -1
- package/src/daemon/shutdown-handlers.ts +122 -0
- package/src/daemon/trace-emitter.ts +1 -1
- package/src/daemon/watch-handler.ts +36 -33
- package/src/doordash/cart-queries.ts +787 -0
- package/src/doordash/client.ts +144 -127
- package/src/doordash/order-queries.ts +85 -0
- package/src/doordash/queries.ts +10 -1308
- package/src/doordash/search-queries.ts +203 -0
- package/src/doordash/session.ts +3 -2
- package/src/doordash/store-queries.ts +246 -0
- package/src/doordash/types.ts +367 -0
- package/src/email/providers/agentmail.ts +2 -1
- package/src/email/providers/index.ts +3 -2
- package/src/email/service.ts +3 -2
- package/src/errors.ts +43 -0
- package/src/home-base/prebuilt/seed.ts +1 -1
- package/src/hooks/cli.ts +6 -5
- package/src/hooks/config.ts +6 -8
- package/src/hooks/discovery.ts +6 -5
- package/src/hooks/manager.ts +4 -3
- package/src/hooks/runner.ts +2 -2
- package/src/hooks/templates.ts +5 -5
- package/src/inbound/public-ingress-urls.ts +3 -1
- package/src/index.ts +4 -2
- package/src/influencer/client.ts +1104 -0
- package/src/instrument.ts +4 -3
- package/src/logfire.ts +4 -3
- package/src/memory/admin.ts +25 -35
- package/src/memory/attachments-store.ts +4 -7
- package/src/memory/channel-delivery-store.ts +30 -1
- package/src/memory/channel-guardian-store.ts +200 -1
- package/src/memory/clarification-resolver.ts +37 -33
- package/src/memory/conflict-store.ts +67 -61
- package/src/memory/contradiction-checker.ts +141 -117
- package/src/memory/conversation-store.ts +335 -51
- package/src/memory/db-connection.ts +27 -4
- package/src/memory/db-init.ts +121 -4
- package/src/memory/db.ts +14 -1
- package/src/memory/embedding-backend.ts +27 -5
- package/src/memory/embedding-ollama.ts +2 -1
- package/src/memory/entity-extractor.ts +38 -35
- package/src/memory/guardian-action-store.ts +430 -0
- package/src/memory/inbox-escalation-projection.ts +59 -0
- package/src/memory/inbox-thread-store.ts +218 -0
- package/src/memory/ingress-invite-store.ts +338 -0
- package/src/memory/ingress-member-store.ts +350 -0
- package/src/memory/items-extractor.ts +91 -97
- package/src/memory/job-handlers/index-maintenance.ts +3 -3
- package/src/memory/job-handlers/media-processing.ts +11 -42
- package/src/memory/job-handlers/summarization.ts +32 -26
- package/src/memory/job-utils.ts +3 -10
- package/src/memory/jobs-store.ts +6 -9
- package/src/memory/jobs-worker.ts +51 -36
- package/src/memory/migrations/001-job-deferrals.ts +45 -0
- package/src/memory/migrations/002-tool-invocations-fk.ts +43 -0
- package/src/memory/migrations/003-memory-fts-backfill.ts +24 -0
- package/src/memory/migrations/004-entity-relation-dedup.ts +87 -0
- package/src/memory/migrations/005-fingerprint-scope-unique.ts +80 -0
- package/src/memory/migrations/006-scope-salted-fingerprints.ts +62 -0
- package/src/memory/migrations/007-assistant-id-to-self.ts +254 -0
- package/src/memory/migrations/008-remove-assistant-id-columns.ts +208 -0
- package/src/memory/migrations/009-llm-usage-events-drop-assistant-id.ts +83 -0
- package/src/memory/migrations/010-ext-conv-bindings-channel-chat-unique.ts +56 -0
- package/src/memory/migrations/011-call-sessions-provider-sid-dedup.ts +63 -0
- package/src/memory/migrations/012-call-sessions-add-initiated-from.ts +19 -0
- package/src/memory/migrations/013-guardian-action-tables.ts +68 -0
- package/src/memory/migrations/014-backfill-inbox-thread-state.ts +76 -0
- package/src/memory/migrations/015-drop-active-search-index.ts +27 -0
- package/src/memory/migrations/016-memory-segments-indexes.ts +11 -0
- package/src/memory/migrations/017-memory-items-indexes.ts +12 -0
- package/src/memory/migrations/018-remaining-table-indexes.ts +13 -0
- package/src/memory/migrations/index.ts +24 -0
- package/src/memory/migrations/registry.ts +79 -0
- package/src/memory/migrations/validate-migration-state.ts +69 -0
- package/src/memory/qdrant-manager.ts +49 -8
- package/src/memory/query-builder.ts +1 -1
- package/src/memory/raw-query.ts +119 -0
- package/src/memory/recall-cache.ts +4 -1
- package/src/memory/retriever.ts +163 -47
- package/src/memory/schema-migration.ts +25 -984
- package/src/memory/schema.ts +130 -7
- package/src/memory/search/entity.ts +10 -19
- package/src/memory/search/lexical.ts +81 -52
- package/src/memory/search/ranking.ts +21 -22
- package/src/memory/search/semantic.ts +157 -19
- package/src/memory/shared-app-links-store.ts +4 -5
- package/src/memory/validation.ts +19 -0
- package/src/messaging/draft-store.ts +5 -6
- package/src/messaging/providers/sms/adapter.ts +3 -6
- package/src/messaging/providers/telegram-bot/adapter.ts +2 -5
- package/src/messaging/providers/whatsapp/adapter.ts +136 -0
- package/src/messaging/providers/whatsapp/client.ts +67 -0
- package/src/messaging/style-analyzer.ts +5 -4
- package/src/messaging/thread-summarizer.ts +61 -69
- package/src/messaging/triage-engine.ts +62 -71
- package/src/migrations/config-merge.ts +53 -0
- package/src/migrations/data-layout.ts +68 -0
- package/src/migrations/data-merge.ts +33 -0
- package/src/migrations/hooks-merge.ts +90 -0
- package/src/migrations/index.ts +6 -0
- package/src/migrations/log.ts +23 -0
- package/src/migrations/skills-merge.ts +33 -0
- package/src/migrations/workspace-layout.ts +79 -0
- package/src/permissions/checker.ts +126 -11
- package/src/permissions/prompter.ts +14 -0
- package/src/permissions/shell-identity.ts +31 -1
- package/src/permissions/trust-store.ts +21 -1
- package/src/providers/anthropic/client.ts +4 -4
- package/src/providers/failover.ts +2 -2
- package/src/providers/model-intents.ts +70 -0
- package/src/providers/ollama/client.ts +2 -1
- package/src/providers/provider-send-message.ts +176 -0
- package/src/providers/registry.ts +71 -30
- package/src/providers/retry.ts +35 -1
- package/src/providers/types.ts +12 -1
- package/src/runtime/approval-conversation-turn.ts +97 -0
- package/src/runtime/approval-message-composer.ts +115 -5
- package/src/runtime/assistant-event-hub.ts +3 -1
- package/src/runtime/channel-approval-parser.ts +36 -2
- package/src/runtime/channel-approvals.ts +0 -21
- package/src/runtime/channel-guardian-service.ts +48 -7
- package/src/runtime/channel-readiness-service.ts +160 -34
- package/src/runtime/channel-readiness-types.ts +10 -4
- package/src/runtime/channel-retry-sweep.ts +184 -0
- package/src/runtime/guardian-context-resolver.ts +108 -0
- package/src/runtime/http-server.ts +289 -745
- package/src/runtime/http-types.ts +56 -3
- package/src/runtime/middleware/auth.ts +116 -0
- package/src/runtime/middleware/error-handler.ts +33 -0
- package/src/runtime/middleware/twilio-validation.ts +127 -0
- package/src/runtime/routes/app-routes.ts +1 -1
- package/src/runtime/routes/call-routes.ts +49 -6
- package/src/runtime/routes/channel-delivery-routes.ts +170 -0
- package/src/runtime/routes/channel-guardian-routes.ts +1191 -0
- package/src/runtime/routes/channel-inbound-routes.ts +1152 -0
- package/src/runtime/routes/channel-route-shared.ts +144 -0
- package/src/runtime/routes/channel-routes.ts +32 -1634
- package/src/runtime/routes/conversation-routes.ts +50 -7
- package/src/runtime/routes/events-routes.ts +2 -2
- package/src/runtime/routes/identity-routes.ts +126 -0
- package/src/runtime/routes/pairing-routes.ts +144 -0
- package/src/runtime/routes/run-routes.ts +15 -1
- package/src/runtime/run-orchestrator.ts +52 -34
- package/src/schedule/schedule-store.ts +36 -32
- package/src/schedule/scheduler.ts +3 -3
- package/src/security/encrypted-store.ts +5 -7
- package/src/security/oauth2.ts +45 -15
- package/src/security/parental-control-store.ts +183 -0
- package/src/security/secret-allowlist.ts +4 -3
- package/src/security/secret-scanner.ts +5 -5
- package/src/security/secure-keys.ts +1 -1
- package/src/security/token-manager.ts +3 -2
- package/src/services/vercel-deploy.ts +6 -2
- package/src/skills/tool-manifest.ts +3 -3
- package/src/skills/vellum-catalog-remote.ts +75 -16
- package/src/slack/slack-webhook.ts +2 -1
- package/src/swarm/orchestrator.ts +92 -1
- package/src/swarm/router-planner.ts +6 -9
- package/src/swarm/worker-prompts.ts +9 -12
- package/src/tasks/task-compiler.ts +19 -28
- package/src/tasks/task-runner.ts +1 -1
- package/src/tools/assets/search.ts +15 -14
- package/src/tools/browser/__tests__/auth-detector.test.ts +1 -0
- package/src/tools/browser/auto-navigate.ts +1 -0
- package/src/tools/browser/browser-execution.ts +13 -1
- package/src/tools/browser/browser-manager.ts +119 -4
- package/src/tools/browser/network-recorder.ts +5 -0
- package/src/tools/credentials/broker.ts +11 -2
- package/src/tools/credentials/metadata-store.ts +18 -14
- package/src/tools/credentials/post-connect-hooks.ts +61 -0
- package/src/tools/credentials/vault.ts +49 -23
- package/src/tools/executor.ts +80 -18
- package/src/tools/host-terminal/cli-discover.ts +1 -1
- package/src/tools/network/script-proxy/http-forwarder.ts +1 -1
- package/src/tools/network/script-proxy/mitm-handler.ts +1 -1
- package/src/tools/network/script-proxy/server.ts +1 -1
- package/src/tools/network/script-proxy/session-manager.ts +6 -5
- package/src/tools/network/web-fetch.ts +18 -2
- package/src/tools/network/web-search.ts +7 -3
- package/src/tools/reminder/reminder-store.ts +14 -15
- package/src/tools/schedule/create.ts +1 -0
- package/src/tools/schedule/list.ts +2 -1
- package/src/tools/shared/filesystem/file-ops-service.ts +5 -7
- package/src/tools/skills/skill-script-runner.ts +24 -9
- package/src/tools/skills/skill-tool-factory.ts +1 -0
- package/src/tools/tasks/work-item-enqueue.ts +2 -2
- package/src/tools/terminal/evaluate-typescript.ts +21 -12
- package/src/tools/terminal/parser.ts +50 -0
- package/src/tools/watcher/delete.ts +6 -0
- package/src/tools/weather/service.ts +1 -1
- package/src/twitter/client.ts +190 -24
- package/src/twitter/session.ts +4 -3
- package/src/util/clipboard.ts +1 -1
- package/src/util/errors.ts +65 -8
- package/src/util/fs.ts +40 -0
- package/src/util/json.ts +10 -0
- package/src/util/log-redact.ts +189 -0
- package/src/util/logger.ts +25 -18
- package/src/util/object.ts +3 -0
- package/src/util/platform.ts +72 -365
- package/src/util/pricing.ts +1 -1
- package/src/util/promise-guard.ts +1 -1
- package/src/util/retry.ts +19 -0
- package/src/util/row-mapper.ts +79 -0
- package/src/util/silently.ts +21 -0
- package/src/watcher/engine.ts +5 -1
- package/src/watcher/provider-types.ts +20 -0
- package/src/watcher/providers/github.ts +156 -0
- package/src/watcher/providers/gmail.ts +1 -0
- package/src/watcher/providers/google-calendar.ts +1 -0
- package/src/watcher/providers/linear.ts +460 -0
- package/src/watcher/providers/slack.ts +1 -0
- package/src/work-items/work-item-runner.ts +1 -1
- package/src/workspace/git-service.ts +1 -1
- package/src/workspace/provider-commit-message-generator.ts +51 -22
- package/src/__tests__/call-bridge.test.ts +0 -517
- package/src/__tests__/session-process-bridge.test.ts +0 -244
- package/src/calls/call-bridge.ts +0 -168
- package/src/config/bundled-skills/media-processing/services/capability-registry.ts +0 -137
- package/src/config/bundled-skills/media-processing/services/event-detection-service.ts +0 -280
- package/src/config/bundled-skills/media-processing/services/feedback-aggregation.ts +0 -144
- package/src/config/bundled-skills/media-processing/services/feedback-store.ts +0 -136
- package/src/config/bundled-skills/media-processing/services/retrieval-service.ts +0 -95
- package/src/config/bundled-skills/media-processing/services/timeline-service.ts +0 -267
- package/src/config/bundled-skills/media-processing/tools/detect-events.ts +0 -110
- package/src/config/bundled-skills/media-processing/tools/recalibrate.ts +0 -235
- package/src/config/bundled-skills/media-processing/tools/select-tracking-profile.ts +0 -142
- package/src/config/bundled-skills/media-processing/tools/submit-feedback.ts +0 -150
- package/src/config/vellum-skills/google-oauth-setup/SKILL.md +0 -199
|
@@ -6,8 +6,9 @@
|
|
|
6
6
|
* streams text tokens back through the RelayConnection for real-time TTS.
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
|
-
import Anthropic from '@anthropic-ai/sdk';
|
|
10
9
|
import { getConfig } from '../config/loader.js';
|
|
10
|
+
import { resolveConfiguredProvider } from '../providers/provider-send-message.js';
|
|
11
|
+
import type { ProviderEvent } from '../providers/types.js';
|
|
11
12
|
import { resolveUserReference } from '../config/user-reference.js';
|
|
12
13
|
import { getLogger } from '../util/logger.js';
|
|
13
14
|
import {
|
|
@@ -21,23 +22,38 @@ import { getMaxCallDurationMs, getUserConsultationTimeoutMs, SILENCE_TIMEOUT_MS
|
|
|
21
22
|
import type { RelayConnection } from './relay-server.js';
|
|
22
23
|
import { registerCallOrchestrator, unregisterCallOrchestrator, fireCallQuestionNotifier, fireCallCompletionNotifier, fireCallTranscriptNotifier } from './call-state.js';
|
|
23
24
|
import type { PromptSpeakerContext } from './speaker-identification.js';
|
|
25
|
+
import { addPointerMessage, formatDuration } from './call-pointer-messages.js';
|
|
26
|
+
import { persistCallCompletionMessage } from './call-conversation-messages.js';
|
|
27
|
+
import * as conversationStore from '../memory/conversation-store.js';
|
|
28
|
+
import { dispatchGuardianQuestion } from './guardian-dispatch.js';
|
|
29
|
+
import type { ServerMessage } from '../daemon/ipc-contract.js';
|
|
30
|
+
import {
|
|
31
|
+
buildGuardianContextBlock,
|
|
32
|
+
type GuardianRuntimeContext,
|
|
33
|
+
} from '../daemon/session-runtime-assembly.js';
|
|
24
34
|
|
|
25
35
|
const log = getLogger('call-orchestrator');
|
|
26
36
|
|
|
27
37
|
type OrchestratorState = 'idle' | 'processing' | 'waiting_on_user' | 'speaking';
|
|
28
38
|
|
|
29
|
-
const
|
|
30
|
-
const
|
|
39
|
+
const ASK_GUARDIAN_CAPTURE_REGEX = /\[ASK_GUARDIAN:\s*(.+?)\]/;
|
|
40
|
+
const ASK_GUARDIAN_MARKER_REGEX = /\[ASK_GUARDIAN:\s*.+?\]/g;
|
|
31
41
|
const USER_ANSWERED_MARKER_REGEX = /\[USER_ANSWERED:\s*.+?\]/g;
|
|
32
42
|
const USER_INSTRUCTION_MARKER_REGEX = /\[USER_INSTRUCTION:\s*.+?\]/g;
|
|
43
|
+
const CALL_OPENING_MARKER_REGEX = /\[CALL_OPENING\]/g;
|
|
44
|
+
const CALL_OPENING_ACK_MARKER_REGEX = /\[CALL_OPENING_ACK\]/g;
|
|
33
45
|
const END_CALL_MARKER_REGEX = /\[END_CALL\]/g;
|
|
46
|
+
const CALL_OPENING_MARKER = '[CALL_OPENING]';
|
|
47
|
+
const CALL_OPENING_ACK_MARKER = '[CALL_OPENING_ACK]';
|
|
34
48
|
const END_CALL_MARKER = '[END_CALL]';
|
|
35
49
|
|
|
36
50
|
function stripInternalSpeechMarkers(text: string): string {
|
|
37
51
|
return text
|
|
38
|
-
.replace(
|
|
52
|
+
.replace(ASK_GUARDIAN_MARKER_REGEX, '')
|
|
39
53
|
.replace(USER_ANSWERED_MARKER_REGEX, '')
|
|
40
54
|
.replace(USER_INSTRUCTION_MARKER_REGEX, '')
|
|
55
|
+
.replace(CALL_OPENING_MARKER_REGEX, '')
|
|
56
|
+
.replace(CALL_OPENING_ACK_MARKER_REGEX, '')
|
|
41
57
|
.replace(END_CALL_MARKER_REGEX, '');
|
|
42
58
|
}
|
|
43
59
|
|
|
@@ -47,22 +63,46 @@ export class CallOrchestrator {
|
|
|
47
63
|
private state: OrchestratorState = 'idle';
|
|
48
64
|
private conversationHistory: Array<{ role: 'user' | 'assistant'; content: string }> = [];
|
|
49
65
|
private abortController: AbortController = new AbortController();
|
|
50
|
-
private callStartTime: number = Date.now();
|
|
51
66
|
private silenceTimer: ReturnType<typeof setTimeout> | null = null;
|
|
52
67
|
private durationTimer: ReturnType<typeof setTimeout> | null = null;
|
|
53
68
|
private durationWarningTimer: ReturnType<typeof setTimeout> | null = null;
|
|
54
69
|
private consultationTimer: ReturnType<typeof setTimeout> | null = null;
|
|
55
70
|
private durationEndTimer: ReturnType<typeof setTimeout> | null = null;
|
|
56
71
|
private task: string | null;
|
|
72
|
+
/** True when the call session was created via the inbound path (no outbound task). */
|
|
73
|
+
private isInbound: boolean;
|
|
57
74
|
/** Instructions queued while an LLM turn is in-flight or during waiting_on_user */
|
|
58
75
|
private pendingInstructions: string[] = [];
|
|
76
|
+
/** Ensures the call opener is triggered at most once per call. */
|
|
77
|
+
private initialGreetingStarted = false;
|
|
78
|
+
/** Marks that the next caller turn should be treated as an opening acknowledgment. */
|
|
79
|
+
private awaitingOpeningAck = false;
|
|
59
80
|
/** Monotonic run id used to suppress stale turn side effects after interruption. */
|
|
60
81
|
private llmRunVersion = 0;
|
|
61
|
-
|
|
62
|
-
|
|
82
|
+
/** Optional broadcast function for emitting IPC events to connected clients. */
|
|
83
|
+
private broadcast?: (msg: ServerMessage) => void;
|
|
84
|
+
/** Assistant identity for scoping guardian bindings. */
|
|
85
|
+
private assistantId: string;
|
|
86
|
+
/** Guardian trust context for the current caller, when available. */
|
|
87
|
+
private guardianContext: GuardianRuntimeContext | null;
|
|
88
|
+
|
|
89
|
+
constructor(
|
|
90
|
+
callSessionId: string,
|
|
91
|
+
relay: RelayConnection,
|
|
92
|
+
task: string | null,
|
|
93
|
+
opts?: {
|
|
94
|
+
broadcast?: (msg: ServerMessage) => void;
|
|
95
|
+
assistantId?: string;
|
|
96
|
+
guardianContext?: GuardianRuntimeContext;
|
|
97
|
+
},
|
|
98
|
+
) {
|
|
63
99
|
this.callSessionId = callSessionId;
|
|
64
100
|
this.relay = relay;
|
|
65
101
|
this.task = task;
|
|
102
|
+
this.isInbound = !task;
|
|
103
|
+
this.broadcast = opts?.broadcast;
|
|
104
|
+
this.assistantId = opts?.assistantId ?? 'self';
|
|
105
|
+
this.guardianContext = opts?.guardianContext ?? null;
|
|
66
106
|
this.startDurationTimer();
|
|
67
107
|
this.resetSilenceTimer();
|
|
68
108
|
registerCallOrchestrator(callSessionId, this);
|
|
@@ -75,6 +115,30 @@ export class CallOrchestrator {
|
|
|
75
115
|
return this.state;
|
|
76
116
|
}
|
|
77
117
|
|
|
118
|
+
/**
|
|
119
|
+
* Update guardian trust context for subsequent LLM turns.
|
|
120
|
+
*/
|
|
121
|
+
setGuardianContext(ctx: GuardianRuntimeContext | null): void {
|
|
122
|
+
this.guardianContext = ctx;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* Kick off the first outbound call utterance from the assistant.
|
|
127
|
+
*/
|
|
128
|
+
async startInitialGreeting(): Promise<void> {
|
|
129
|
+
if (this.initialGreetingStarted) return;
|
|
130
|
+
if (this.state !== 'idle') return;
|
|
131
|
+
|
|
132
|
+
this.initialGreetingStarted = true;
|
|
133
|
+
this.resetSilenceTimer();
|
|
134
|
+
this.conversationHistory.push({ role: 'user', content: CALL_OPENING_MARKER });
|
|
135
|
+
await this.runLlm();
|
|
136
|
+
const lastMessage = this.conversationHistory[this.conversationHistory.length - 1];
|
|
137
|
+
if (lastMessage?.role === 'assistant') {
|
|
138
|
+
this.awaitingOpeningAck = true;
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
|
|
78
142
|
/**
|
|
79
143
|
* Handle a final caller utterance from the ConversationRelay.
|
|
80
144
|
*/
|
|
@@ -86,9 +150,42 @@ export class CallOrchestrator {
|
|
|
86
150
|
this.abortController = new AbortController();
|
|
87
151
|
}
|
|
88
152
|
|
|
153
|
+
// Strip the one-shot [CALL_OPENING] marker from conversation history
|
|
154
|
+
// so it doesn't leak into subsequent LLM requests after barge-in.
|
|
155
|
+
// This runs unconditionally because the standard Twilio barge-in path
|
|
156
|
+
// calls handleInterrupt() first (setting state to 'idle') before
|
|
157
|
+
// handleCallerUtterance — so interruptedInFlight would be false even
|
|
158
|
+
// though an interrupt just occurred.
|
|
159
|
+
// Without this, the consecutive-user merge path below would append
|
|
160
|
+
// the caller's transcript to the synthetic "[CALL_OPENING]" message,
|
|
161
|
+
// causing the model to re-run opener behavior instead of responding
|
|
162
|
+
// directly to the caller.
|
|
163
|
+
// If the marker-only seed message becomes empty, remove it entirely:
|
|
164
|
+
// Anthropic rejects any user turn with empty content.
|
|
165
|
+
for (let i = 0; i < this.conversationHistory.length; i++) {
|
|
166
|
+
const entry = this.conversationHistory[i];
|
|
167
|
+
if (!entry.content.includes(CALL_OPENING_MARKER)) continue;
|
|
168
|
+
const stripped = entry.content.replace(CALL_OPENING_MARKER_REGEX, '').trim();
|
|
169
|
+
if (stripped.length === 0) {
|
|
170
|
+
this.conversationHistory.splice(i, 1);
|
|
171
|
+
i--;
|
|
172
|
+
} else {
|
|
173
|
+
entry.content = stripped;
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
|
|
89
177
|
this.state = 'processing';
|
|
90
178
|
this.resetSilenceTimer();
|
|
91
179
|
const callerContent = this.formatCallerUtterance(transcript, speaker);
|
|
180
|
+
const shouldMarkOpeningAck = this.awaitingOpeningAck;
|
|
181
|
+
if (shouldMarkOpeningAck) {
|
|
182
|
+
this.awaitingOpeningAck = false;
|
|
183
|
+
}
|
|
184
|
+
const callerTurnContent = shouldMarkOpeningAck
|
|
185
|
+
? callerContent.length > 0
|
|
186
|
+
? `${CALL_OPENING_ACK_MARKER}\n${callerContent}`
|
|
187
|
+
: CALL_OPENING_ACK_MARKER
|
|
188
|
+
: callerContent;
|
|
92
189
|
|
|
93
190
|
// Preserve strict role alternation for Anthropic. If the last message
|
|
94
191
|
// is already user-role (e.g. interrupted run never appended assistant,
|
|
@@ -96,11 +193,14 @@ export class CallOrchestrator {
|
|
|
96
193
|
// this utterance into that same user turn.
|
|
97
194
|
const lastMessage = this.conversationHistory[this.conversationHistory.length - 1];
|
|
98
195
|
if (lastMessage?.role === 'user') {
|
|
99
|
-
|
|
196
|
+
const existingContent = lastMessage.content.trim();
|
|
197
|
+
lastMessage.content = existingContent.length > 0
|
|
198
|
+
? `${lastMessage.content}\n${callerTurnContent}`
|
|
199
|
+
: callerTurnContent;
|
|
100
200
|
} else {
|
|
101
201
|
this.conversationHistory.push({
|
|
102
202
|
role: 'user',
|
|
103
|
-
content:
|
|
203
|
+
content: callerTurnContent,
|
|
104
204
|
});
|
|
105
205
|
}
|
|
106
206
|
|
|
@@ -221,30 +321,83 @@ export class CallOrchestrator {
|
|
|
221
321
|
|
|
222
322
|
// ── Private ──────────────────────────────────────────────────────
|
|
223
323
|
|
|
324
|
+
private buildGuardianPromptSection(): string[] {
|
|
325
|
+
if (!this.guardianContext) return [];
|
|
326
|
+
return [
|
|
327
|
+
'',
|
|
328
|
+
'GUARDIAN ACTOR CONTEXT (authoritative):',
|
|
329
|
+
buildGuardianContextBlock(this.guardianContext),
|
|
330
|
+
'- Treat `actor_role` as source-of-truth for whether this caller is the verified guardian.',
|
|
331
|
+
'- If `actor_role` is `guardian`, the current caller is verified for this assistant on voice.',
|
|
332
|
+
'- If `actor_role` is `non-guardian` or `unverified_channel`, do not imply the caller is verified.',
|
|
333
|
+
];
|
|
334
|
+
}
|
|
335
|
+
|
|
224
336
|
private buildSystemPrompt(): string {
|
|
225
337
|
const config = getConfig();
|
|
226
338
|
const disclosureRule = config.calls.disclosure.enabled
|
|
227
339
|
? `1. ${config.calls.disclosure.text}`
|
|
228
340
|
: '1. Begin the conversation naturally.';
|
|
229
341
|
|
|
342
|
+
if (this.isInbound) {
|
|
343
|
+
return this.buildInboundSystemPrompt(disclosureRule);
|
|
344
|
+
}
|
|
345
|
+
|
|
230
346
|
return [
|
|
231
347
|
`You are on a live phone call on behalf of ${resolveUserReference()}.`,
|
|
232
348
|
this.task ? `Task: ${this.task}` : '',
|
|
233
349
|
'',
|
|
234
350
|
'You are speaking directly to the person who answered the phone.',
|
|
235
351
|
'Respond naturally and conversationally — speak as you would in a real phone conversation.',
|
|
352
|
+
...this.buildGuardianPromptSection(),
|
|
236
353
|
'',
|
|
237
354
|
'IMPORTANT RULES:',
|
|
238
355
|
'0. When introducing yourself, refer to yourself as an assistant. Avoid the phrase "AI assistant" unless directly asked.',
|
|
239
356
|
disclosureRule,
|
|
240
357
|
'2. Be concise — phone conversations should be brief and natural.',
|
|
241
|
-
'3. If the callee asks something you don\'t know, include [
|
|
358
|
+
'3. If the callee asks something you don\'t know, include [ASK_GUARDIAN: your question here] in your response along with a hold message like "Let me check on that for you."',
|
|
242
359
|
'4. If the callee provides information preceded by [USER_ANSWERED: ...], use that answer naturally in the conversation.',
|
|
243
360
|
'5. If you see [USER_INSTRUCTION: ...], treat it as a high-priority steering directive from your user. Follow the instruction immediately, adjusting your approach or response accordingly.',
|
|
244
361
|
'6. When the call\'s purpose is fulfilled, include [END_CALL] in your response along with a polite goodbye.',
|
|
245
362
|
'7. Do not make up information — ask the user if unsure.',
|
|
246
363
|
'8. Keep responses short — 1-3 sentences is ideal for phone conversation.',
|
|
247
364
|
'9. When caller text includes [SPEAKER id="..." label="..."], treat each speaker as a distinct person and personalize responses using that speaker\'s prior context in this call.',
|
|
365
|
+
'10. If the latest user turn is [CALL_OPENING], generate a natural, context-specific opener: briefly introduce yourself once as an assistant, state why you are calling using the Task context, and ask a short permission/check-in question. Vary the wording; do not use a fixed template.',
|
|
366
|
+
'11. If the latest user turn includes [CALL_OPENING_ACK], treat it as the callee acknowledging your opener and continue the conversation naturally without re-introducing yourself or repeating the initial check-in question.',
|
|
367
|
+
'12. Do not repeat your introduction within the same call unless the callee explicitly asks who you are.',
|
|
368
|
+
]
|
|
369
|
+
.filter(Boolean)
|
|
370
|
+
.join('\n');
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
/**
|
|
374
|
+
* Build a system prompt tailored for inbound calls where the caller
|
|
375
|
+
* reached out to us. The assistant greets naturally and helps the
|
|
376
|
+
* caller with whatever they need, rather than delivering an outbound
|
|
377
|
+
* task message.
|
|
378
|
+
*/
|
|
379
|
+
private buildInboundSystemPrompt(disclosureRule: string): string {
|
|
380
|
+
return [
|
|
381
|
+
`You are on a live phone call, answering an incoming call on behalf of ${resolveUserReference()}.`,
|
|
382
|
+
'',
|
|
383
|
+
'The caller dialed in to reach you. You do not have a specific task — your role is to greet them warmly, find out what they need, and assist them.',
|
|
384
|
+
'Respond naturally and conversationally — speak as you would in a real phone conversation.',
|
|
385
|
+
...this.buildGuardianPromptSection(),
|
|
386
|
+
'',
|
|
387
|
+
'IMPORTANT RULES:',
|
|
388
|
+
'0. When introducing yourself, refer to yourself as an assistant. Avoid the phrase "AI assistant" unless directly asked.',
|
|
389
|
+
disclosureRule,
|
|
390
|
+
'2. Be concise — phone conversations should be brief and natural.',
|
|
391
|
+
'3. If the caller asks something you don\'t know or need to verify, include [ASK_GUARDIAN: your question here] in your response along with a hold message like "Let me check on that for you."',
|
|
392
|
+
'4. If information is provided preceded by [USER_ANSWERED: ...], use that answer naturally in the conversation.',
|
|
393
|
+
'5. If you see [USER_INSTRUCTION: ...], treat it as a high-priority steering directive from your user. Follow the instruction immediately, adjusting your approach or response accordingly.',
|
|
394
|
+
'6. When the caller indicates they are done or the conversation reaches a natural conclusion, include [END_CALL] in your response along with a polite goodbye.',
|
|
395
|
+
'7. Do not make up information — ask the user if unsure.',
|
|
396
|
+
'8. Keep responses short — 1-3 sentences is ideal for phone conversation.',
|
|
397
|
+
'9. When caller text includes [SPEAKER id="..." label="..."], treat each speaker as a distinct person and personalize responses using that speaker\'s prior context in this call.',
|
|
398
|
+
'10. If the latest user turn is [CALL_OPENING], greet the caller warmly and ask how you can help. For example: "Hello, this is [name]\'s assistant. How can I help you today?" Vary the wording; do not use a fixed template.',
|
|
399
|
+
'11. If the latest user turn includes [CALL_OPENING_ACK], treat it as the caller acknowledging your greeting and continue the conversation naturally.',
|
|
400
|
+
'12. Do not repeat your introduction within the same call unless the caller explicitly asks who you are.',
|
|
248
401
|
]
|
|
249
402
|
.filter(Boolean)
|
|
250
403
|
.join('\n');
|
|
@@ -254,7 +407,7 @@ export class CallOrchestrator {
|
|
|
254
407
|
if (!speaker) return transcript;
|
|
255
408
|
const safeId = speaker.speakerId.replaceAll('"', '\'');
|
|
256
409
|
const safeLabel = speaker.speakerLabel.replaceAll('"', '\'');
|
|
257
|
-
const confidencePart = speaker.speakerConfidence
|
|
410
|
+
const confidencePart = speaker.speakerConfidence != null ? ` confidence="${speaker.speakerConfidence.toFixed(2)}"` : '';
|
|
258
411
|
return `[SPEAKER id="${safeId}" label="${safeLabel}" source="${speaker.source}"${confidencePart}] ${transcript}`;
|
|
259
412
|
}
|
|
260
413
|
|
|
@@ -263,37 +416,31 @@ export class CallOrchestrator {
|
|
|
263
416
|
* the response back through the relay.
|
|
264
417
|
*/
|
|
265
418
|
private async runLlm(): Promise<void> {
|
|
266
|
-
const
|
|
267
|
-
|
|
268
|
-
|
|
419
|
+
const config = getConfig();
|
|
420
|
+
const resolved = resolveConfiguredProvider();
|
|
421
|
+
if (!resolved) {
|
|
422
|
+
log.error({ callSessionId: this.callSessionId }, 'No provider available');
|
|
269
423
|
this.relay.sendTextToken('I\'m sorry, I\'m having a technical issue. Please try again later.', true);
|
|
270
424
|
this.state = 'idle';
|
|
271
425
|
return;
|
|
272
426
|
}
|
|
427
|
+
const { provider } = resolved;
|
|
273
428
|
|
|
274
|
-
const client = new Anthropic({ apiKey });
|
|
275
429
|
const runVersion = ++this.llmRunVersion;
|
|
276
430
|
const runSignal = this.abortController.signal;
|
|
277
431
|
|
|
278
432
|
try {
|
|
279
433
|
this.state = 'speaking';
|
|
280
434
|
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
messages: this.conversationHistory.map((m) => ({
|
|
289
|
-
role: m.role,
|
|
290
|
-
content: m.content,
|
|
291
|
-
})),
|
|
292
|
-
},
|
|
293
|
-
{ signal: runSignal },
|
|
294
|
-
);
|
|
435
|
+
// Only override the model when the user has explicitly configured one
|
|
436
|
+
// AND the selected provider matches the configured provider. Forwarding
|
|
437
|
+
// a provider-specific model to a fallback provider would cause
|
|
438
|
+
// cross-provider 4xx errors (e.g., sending "gpt-5.2" to Anthropic).
|
|
439
|
+
const callModel = !resolved.usedFallbackPrimary
|
|
440
|
+
? (config.calls.model?.trim() || undefined)
|
|
441
|
+
: undefined;
|
|
295
442
|
|
|
296
|
-
// Buffer incoming tokens so we can strip control markers ([
|
|
443
|
+
// Buffer incoming tokens so we can strip control markers ([ASK_GUARDIAN:...], [END_CALL])
|
|
297
444
|
// before they reach TTS. We hold text whenever an unmatched '[' appears, since it
|
|
298
445
|
// could be the start of a control marker.
|
|
299
446
|
let ttsBuffer = '';
|
|
@@ -320,18 +467,24 @@ export class CallOrchestrator {
|
|
|
320
467
|
// The check must be bidirectional:
|
|
321
468
|
// - When the buffer is shorter than the prefix (e.g. "[ASK"), the
|
|
322
469
|
// buffer is a prefix of the control tag → hold it.
|
|
323
|
-
// - When the buffer is longer than the prefix (e.g. "[
|
|
470
|
+
// - When the buffer is longer than the prefix (e.g. "[ASK_GUARDIAN: what"),
|
|
324
471
|
// the buffer starts with the control tag prefix → hold it (the
|
|
325
472
|
// variable-length payload hasn't been closed yet).
|
|
326
473
|
const afterBracket = ttsBuffer;
|
|
327
474
|
const couldBeControl =
|
|
328
|
-
'[
|
|
475
|
+
'[ASK_GUARDIAN:'.startsWith(afterBracket) ||
|
|
329
476
|
'[USER_ANSWERED:'.startsWith(afterBracket) ||
|
|
330
477
|
'[USER_INSTRUCTION:'.startsWith(afterBracket) ||
|
|
478
|
+
'[CALL_OPENING]'.startsWith(afterBracket) ||
|
|
479
|
+
'[CALL_OPENING_ACK]'.startsWith(afterBracket) ||
|
|
331
480
|
'[END_CALL]'.startsWith(afterBracket) ||
|
|
332
|
-
afterBracket.startsWith('[
|
|
481
|
+
afterBracket.startsWith('[ASK_GUARDIAN:') ||
|
|
333
482
|
afterBracket.startsWith('[USER_ANSWERED:') ||
|
|
334
483
|
afterBracket.startsWith('[USER_INSTRUCTION:') ||
|
|
484
|
+
afterBracket === '[CALL_OPENING' ||
|
|
485
|
+
afterBracket.startsWith('[CALL_OPENING]') ||
|
|
486
|
+
afterBracket === '[CALL_OPENING_ACK' ||
|
|
487
|
+
afterBracket.startsWith('[CALL_OPENING_ACK]') ||
|
|
335
488
|
afterBracket === '[END_CALL' ||
|
|
336
489
|
afterBracket.startsWith('[END_CALL]');
|
|
337
490
|
|
|
@@ -351,17 +504,29 @@ export class CallOrchestrator {
|
|
|
351
504
|
}
|
|
352
505
|
};
|
|
353
506
|
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
507
|
+
const response = await provider.sendMessage(
|
|
508
|
+
this.conversationHistory.map((m) => ({
|
|
509
|
+
role: m.role as 'user' | 'assistant',
|
|
510
|
+
content: [{ type: 'text' as const, text: m.content }],
|
|
511
|
+
})),
|
|
512
|
+
[], // no tools
|
|
513
|
+
this.buildSystemPrompt(),
|
|
514
|
+
{
|
|
515
|
+
config: {
|
|
516
|
+
...(callModel ? { model: callModel } : {}),
|
|
517
|
+
max_tokens: 512,
|
|
518
|
+
},
|
|
519
|
+
onEvent: (event: ProviderEvent) => {
|
|
520
|
+
if (!this.isCurrentRun(runVersion)) return;
|
|
521
|
+
if (event.type === 'text_delta') {
|
|
522
|
+
ttsBuffer += event.text;
|
|
523
|
+
ttsBuffer = stripInternalSpeechMarkers(ttsBuffer);
|
|
524
|
+
flushSafeText(false);
|
|
525
|
+
}
|
|
526
|
+
},
|
|
527
|
+
signal: runSignal,
|
|
528
|
+
},
|
|
529
|
+
);
|
|
365
530
|
if (!this.isCurrentRun(runVersion)) return;
|
|
366
531
|
|
|
367
532
|
// Final sweep: strip any remaining control markers from the buffer
|
|
@@ -373,11 +538,10 @@ export class CallOrchestrator {
|
|
|
373
538
|
// Signal end of this turn's speech
|
|
374
539
|
this.relay.sendTextToken('', true);
|
|
375
540
|
|
|
376
|
-
const responseText =
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
.join('') || '';
|
|
541
|
+
const responseText = response.content
|
|
542
|
+
.filter((b): b is { type: 'text'; text: string } => b.type === 'text')
|
|
543
|
+
.map((b) => b.text)
|
|
544
|
+
.join('') || '';
|
|
381
545
|
|
|
382
546
|
// Record the assistant response
|
|
383
547
|
this.conversationHistory.push({ role: 'assistant', content: responseText });
|
|
@@ -386,15 +550,23 @@ export class CallOrchestrator {
|
|
|
386
550
|
if (spokenText.length > 0) {
|
|
387
551
|
const session = getCallSession(this.callSessionId);
|
|
388
552
|
if (session) {
|
|
553
|
+
// Persist assistant transcript to the voice conversation so it
|
|
554
|
+
// survives even when no live daemon Session is listening.
|
|
555
|
+
conversationStore.addMessage(
|
|
556
|
+
session.conversationId,
|
|
557
|
+
'assistant',
|
|
558
|
+
JSON.stringify([{ type: 'text', text: spokenText }]),
|
|
559
|
+
{ userMessageChannel: 'voice', assistantMessageChannel: 'voice' },
|
|
560
|
+
);
|
|
389
561
|
fireCallTranscriptNotifier(session.conversationId, this.callSessionId, 'assistant', spokenText);
|
|
390
562
|
}
|
|
391
563
|
}
|
|
392
564
|
|
|
393
|
-
// Check for
|
|
394
|
-
const askMatch = responseText.match(
|
|
565
|
+
// Check for ASK_GUARDIAN pattern
|
|
566
|
+
const askMatch = responseText.match(ASK_GUARDIAN_CAPTURE_REGEX);
|
|
395
567
|
if (askMatch) {
|
|
396
568
|
const questionText = askMatch[1];
|
|
397
|
-
createPendingQuestion(this.callSessionId, questionText);
|
|
569
|
+
const pendingQuestion = createPendingQuestion(this.callSessionId, questionText);
|
|
398
570
|
this.state = 'waiting_on_user';
|
|
399
571
|
updateCallSession(this.callSessionId, { status: 'waiting_on_user' });
|
|
400
572
|
recordCallEvent(this.callSessionId, 'user_question_asked', { question: questionText });
|
|
@@ -403,6 +575,15 @@ export class CallOrchestrator {
|
|
|
403
575
|
const session = getCallSession(this.callSessionId);
|
|
404
576
|
if (session) {
|
|
405
577
|
fireCallQuestionNotifier(session.conversationId, this.callSessionId, questionText);
|
|
578
|
+
|
|
579
|
+
// Dispatch guardian action request to all configured channels
|
|
580
|
+
void dispatchGuardianQuestion({
|
|
581
|
+
callSessionId: this.callSessionId,
|
|
582
|
+
conversationId: session.conversationId,
|
|
583
|
+
assistantId: this.assistantId,
|
|
584
|
+
pendingQuestion,
|
|
585
|
+
broadcast: this.broadcast,
|
|
586
|
+
});
|
|
406
587
|
}
|
|
407
588
|
|
|
408
589
|
// Set a consultation timeout
|
|
@@ -433,11 +614,19 @@ export class CallOrchestrator {
|
|
|
433
614
|
updateCallSession(this.callSessionId, { status: 'completed', endedAt: Date.now() });
|
|
434
615
|
recordCallEvent(this.callSessionId, 'call_ended', { reason: 'completed' });
|
|
435
616
|
|
|
436
|
-
// Notify the conversation
|
|
437
|
-
// into a terminal call state.
|
|
617
|
+
// Notify the voice conversation
|
|
438
618
|
if (shouldNotifyCompletion && currentSession) {
|
|
619
|
+
persistCallCompletionMessage(currentSession.conversationId, this.callSessionId);
|
|
439
620
|
fireCallCompletionNotifier(currentSession.conversationId, this.callSessionId);
|
|
440
621
|
}
|
|
622
|
+
|
|
623
|
+
// Post a pointer message in the initiating conversation
|
|
624
|
+
if (currentSession?.initiatedFromConversationId) {
|
|
625
|
+
const durationMs = currentSession.startedAt ? Date.now() - currentSession.startedAt : 0;
|
|
626
|
+
addPointerMessage(currentSession.initiatedFromConversationId, 'completed', currentSession.toNumber, {
|
|
627
|
+
duration: durationMs > 0 ? formatDuration(durationMs) : undefined,
|
|
628
|
+
});
|
|
629
|
+
}
|
|
441
630
|
this.state = 'idle';
|
|
442
631
|
return;
|
|
443
632
|
}
|
|
@@ -545,8 +734,17 @@ export class CallOrchestrator {
|
|
|
545
734
|
updateCallSession(this.callSessionId, { status: 'completed', endedAt: Date.now() });
|
|
546
735
|
recordCallEvent(this.callSessionId, 'call_ended', { reason: 'max_duration' });
|
|
547
736
|
if (shouldNotifyCompletion && currentSession) {
|
|
737
|
+
persistCallCompletionMessage(currentSession.conversationId, this.callSessionId);
|
|
548
738
|
fireCallCompletionNotifier(currentSession.conversationId, this.callSessionId);
|
|
549
739
|
}
|
|
740
|
+
|
|
741
|
+
// Post a pointer message in the initiating conversation
|
|
742
|
+
if (currentSession?.initiatedFromConversationId) {
|
|
743
|
+
const durationMs = currentSession.startedAt ? Date.now() - currentSession.startedAt : 0;
|
|
744
|
+
addPointerMessage(currentSession.initiatedFromConversationId, 'completed', currentSession.toNumber, {
|
|
745
|
+
duration: durationMs > 0 ? formatDuration(durationMs) : undefined,
|
|
746
|
+
});
|
|
747
|
+
}
|
|
550
748
|
}, 3000);
|
|
551
749
|
}, maxDurationMs);
|
|
552
750
|
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Concise pointer/status messages posted to the initiating conversation
|
|
3
|
+
* so the user sees call lifecycle events without the full transcript
|
|
4
|
+
* (which lives in the dedicated voice conversation).
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import * as conversationStore from '../memory/conversation-store.js';
|
|
8
|
+
|
|
9
|
+
export type PointerEvent = 'started' | 'completed' | 'failed';
|
|
10
|
+
|
|
11
|
+
export function addPointerMessage(
|
|
12
|
+
conversationId: string,
|
|
13
|
+
event: PointerEvent,
|
|
14
|
+
phoneNumber: string,
|
|
15
|
+
extra?: { duration?: string; reason?: string; verificationCode?: string },
|
|
16
|
+
): void {
|
|
17
|
+
let text: string;
|
|
18
|
+
switch (event) {
|
|
19
|
+
case 'started':
|
|
20
|
+
text = extra?.verificationCode
|
|
21
|
+
? `\u{1F4DE} Call to ${phoneNumber} started. Verification code: ${extra.verificationCode}`
|
|
22
|
+
: `\u{1F4DE} Call to ${phoneNumber} started. See voice thread for details.`;
|
|
23
|
+
break;
|
|
24
|
+
case 'completed':
|
|
25
|
+
text = extra?.duration
|
|
26
|
+
? `\u{1F4DE} Call to ${phoneNumber} completed (${extra.duration}).`
|
|
27
|
+
: `\u{1F4DE} Call to ${phoneNumber} completed.`;
|
|
28
|
+
break;
|
|
29
|
+
case 'failed':
|
|
30
|
+
text = extra?.reason
|
|
31
|
+
? `\u{1F4DE} Call to ${phoneNumber} failed: ${extra.reason}.`
|
|
32
|
+
: `\u{1F4DE} Call to ${phoneNumber} failed.`;
|
|
33
|
+
break;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
conversationStore.addMessage(
|
|
37
|
+
conversationId,
|
|
38
|
+
'assistant',
|
|
39
|
+
JSON.stringify([{ type: 'text', text }]),
|
|
40
|
+
{ userMessageChannel: 'voice', assistantMessageChannel: 'voice' },
|
|
41
|
+
);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Format a duration in milliseconds into a human-friendly string.
|
|
46
|
+
*/
|
|
47
|
+
export function formatDuration(ms: number): string {
|
|
48
|
+
const totalSeconds = Math.round(ms / 1000);
|
|
49
|
+
if (totalSeconds < 60) return `${totalSeconds}s`;
|
|
50
|
+
const minutes = Math.floor(totalSeconds / 60);
|
|
51
|
+
const seconds = totalSeconds % 60;
|
|
52
|
+
return seconds > 0 ? `${minutes}m ${seconds}s` : `${minutes}m`;
|
|
53
|
+
}
|
|
@@ -8,9 +8,10 @@
|
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
10
|
import { getLogger } from '../util/logger.js';
|
|
11
|
+
import { isTerminalState } from './call-state-machine.js';
|
|
11
12
|
import { listRecoverableCalls, updateCallSession, expirePendingQuestions } from './call-store.js';
|
|
12
|
-
import type { VoiceProvider } from './voice-provider.js';
|
|
13
13
|
import type { CallStatus } from './types.js';
|
|
14
|
+
import type { VoiceProvider } from './voice-provider.js';
|
|
14
15
|
|
|
15
16
|
type Logger = ReturnType<typeof getLogger>;
|
|
16
17
|
|
|
@@ -57,12 +58,6 @@ function mapProviderStatus(providerStatus: string): CallStatus | null {
|
|
|
57
58
|
}
|
|
58
59
|
}
|
|
59
60
|
|
|
60
|
-
/**
|
|
61
|
-
* Check whether a CallStatus is terminal (no further transitions allowed).
|
|
62
|
-
*/
|
|
63
|
-
function isTerminal(status: CallStatus): boolean {
|
|
64
|
-
return status === 'completed' || status === 'failed' || status === 'cancelled';
|
|
65
|
-
}
|
|
66
61
|
|
|
67
62
|
/**
|
|
68
63
|
* Reconcile all non-terminal call sessions at daemon startup.
|
|
@@ -159,7 +154,7 @@ export async function reconcileCallsOnStartup(
|
|
|
159
154
|
continue;
|
|
160
155
|
}
|
|
161
156
|
|
|
162
|
-
if (
|
|
157
|
+
if (isTerminalState(mappedStatus)) {
|
|
163
158
|
// Provider says the call has ended
|
|
164
159
|
log.info(
|
|
165
160
|
{ callSessionId: session.id, providerStatus, mappedStatus },
|