@vellumai/assistant 0.11.2 → 0.11.3-dev.202608080427.92f668a
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +69 -9
- package/docs/flux-turn-detection-spike.md +243 -0
- package/docs/stt-provider-onboarding.md +3 -1
- package/docs/trusted-contact-access.md +9 -5
- package/docs/vellum-doctor.md +195 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/channels.ts +50 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/ingress.ts +10 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/remote-web-pairing.ts +6 -0
- package/node_modules/@vellumai/ces-client/src/index.ts +1 -0
- package/node_modules/@vellumai/ces-client/src/rpc-client.ts +19 -2
- package/node_modules/@vellumai/environments/package.json +2 -1
- package/node_modules/@vellumai/environments/src/__tests__/cloud-assistant-hub-url.test.ts +38 -0
- package/node_modules/@vellumai/environments/src/__tests__/install-layout.test.ts +98 -0
- package/node_modules/@vellumai/environments/src/__tests__/package-boundary.test.ts +5 -5
- package/node_modules/@vellumai/environments/src/index.ts +17 -6
- package/node_modules/@vellumai/environments/src/install-layout.ts +49 -0
- package/node_modules/@vellumai/environments/src/seeds.ts +29 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/channels.ts +50 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/ingress.ts +10 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/remote-web-pairing.ts +6 -0
- package/node_modules/@vellumai/gateway-client/src/index.ts +4 -0
- package/node_modules/@vellumai/gateway-client/src/verification-session-contract.ts +91 -6
- package/node_modules/@vellumai/ipc-server-utils/src/endpoint.test.ts +36 -0
- package/node_modules/@vellumai/ipc-server-utils/src/endpoint.ts +142 -0
- package/node_modules/@vellumai/ipc-server-utils/src/index.ts +2 -0
- package/node_modules/@vellumai/ipc-server-utils/src/listen-options.ts +3 -0
- package/node_modules/@vellumai/ipc-server-utils/src/socket-watchdog.test.ts +15 -1
- package/node_modules/@vellumai/ipc-server-utils/src/socket-watchdog.ts +17 -2
- package/node_modules/@vellumai/service-contracts/src/channels.ts +50 -0
- package/node_modules/@vellumai/service-contracts/src/ingress.ts +10 -0
- package/node_modules/@vellumai/service-contracts/src/remote-web-pairing.ts +6 -0
- package/node_modules/@vellumai/slack-text/src/index.ts +16 -2
- package/openapi.yaml +619 -47
- package/package.json +1 -1
- package/scripts/voice-ttft-spike.ts +3 -3
- package/src/__tests__/agent-image-optimize.test.ts +4 -4
- package/src/__tests__/agent-loop-mutable-latest-user-message.test.ts +113 -66
- package/src/__tests__/agent-loop-override-profile.test.ts +38 -48
- package/src/__tests__/agent-loop-subagent-billing-headers.test.ts +166 -0
- package/src/__tests__/agent-wake-disk-pressure-callsite.test.ts +1 -0
- package/src/__tests__/agent-wake-override-profile.test.ts +1 -0
- package/src/__tests__/answered-question-persistence.test.ts +240 -0
- package/src/__tests__/anthropic-provider.test.ts +165 -30
- package/src/__tests__/app-control-flow.test.ts +9 -8
- package/src/__tests__/assistant-event-hub.test.ts +48 -0
- package/src/__tests__/attachments-store-heic-normalize.test.ts +16 -12
- package/src/__tests__/attachments-store.test.ts +130 -68
- package/src/__tests__/attachments.test.ts +36 -36
- package/src/__tests__/background-workers-disk-pressure.test.ts +1 -1
- package/src/__tests__/call-controller.test.ts +120 -0
- package/src/__tests__/call-start-guardian-guard.test.ts +18 -4
- package/src/__tests__/canned-reply-release.test.ts +86 -0
- package/src/__tests__/channel-availability-routes.test.ts +14 -1
- package/src/__tests__/channel-guardian.test.ts +59 -29
- package/src/__tests__/channel-readiness-telegram-remote.test.ts +154 -0
- package/src/__tests__/channel-setup-panel-ack.test.ts +6 -5
- package/src/__tests__/client-os-metadata-persistence.test.ts +103 -4
- package/src/__tests__/compactor-image-manifest-trust.test.ts +1 -1
- package/src/__tests__/config-loader-backfill.test.ts +19 -0
- package/src/__tests__/config-schema.test.ts +150 -7
- package/src/__tests__/consult-deadline.test.ts +65 -0
- package/src/__tests__/conversation-agent-loop.test.ts +438 -2
- package/src/__tests__/conversation-disk-view-integration.test.ts +5 -1
- package/src/__tests__/conversation-disk-view.test.ts +4 -4
- package/src/__tests__/conversation-error.test.ts +22 -3
- package/src/__tests__/conversation-fork-crud.test.ts +5 -1
- package/src/__tests__/conversation-fork-referential.test.ts +352 -0
- package/src/__tests__/conversation-fork-retrospective.test.ts +5 -1
- package/src/__tests__/conversation-group-tools.test.ts +32 -2
- package/src/__tests__/conversation-inference-profile-route.test.ts +27 -1
- package/src/__tests__/conversation-initial-prompt.test.ts +51 -22
- package/src/__tests__/conversation-key-store-bootstrap-cleanup.test.ts +119 -0
- package/src/__tests__/conversation-key-store-origin.test.ts +105 -0
- package/src/__tests__/conversation-list-recency-ordering.test.ts +104 -0
- package/src/__tests__/conversation-origin-at-creation.test.ts +153 -0
- package/src/__tests__/conversation-origin-channel-filter.test.ts +95 -0
- package/src/__tests__/conversation-placement-promotion.test.ts +172 -0
- package/src/__tests__/conversation-process-app-control-preactivation.test.ts +123 -0
- package/src/__tests__/conversation-queue.test.ts +433 -1
- package/src/__tests__/conversation-retry-route.test.ts +3 -1
- package/src/__tests__/conversation-routes-hidden-queue.test.ts +64 -1
- package/src/__tests__/conversation-store-ephemeral.test.ts +4 -0
- package/src/__tests__/conversation-store.test.ts +26 -10
- package/src/__tests__/conversation-summarize-route.test.ts +1 -1
- package/src/__tests__/conversation-surfaces-action-delivery.test.ts +15 -6
- package/src/__tests__/conversation-surfaces-activation-emit.test.ts +6 -5
- package/src/__tests__/conversation-surfaces-app-control.test.ts +12 -11
- package/src/__tests__/conversation-surfaces-app-open.test.ts +11 -5
- package/src/__tests__/conversation-surfaces-data-persist.test.ts +5 -4
- package/src/__tests__/conversation-surfaces-history-restored-completion.test.ts +967 -0
- package/src/__tests__/conversation-surfaces-persisted-info.test.ts +312 -0
- package/src/__tests__/conversation-surfaces-queued-emit.test.ts +6 -5
- package/src/__tests__/conversation-surfaces-standalone-payloads.test.ts +12 -8
- package/src/__tests__/conversation-surfaces-standalone.test.ts +21 -11
- package/src/__tests__/conversation-surfaces-state-update.test.ts +7 -11
- package/src/__tests__/conversation-surfaces-table-action.test.ts +7 -6
- package/src/__tests__/conversation-surfaces-task-progress.test.ts +11 -6
- package/src/__tests__/conversation-tool-setup-app-refresh.test.ts +9 -6
- package/src/__tests__/conversation-tool-setup-attribution.test.ts +70 -7
- package/src/__tests__/conversation-wait-for-idle.test.ts +6 -11
- package/src/__tests__/credential-prompt-route.test.ts +54 -7
- package/src/__tests__/credential-security-invariants.test.ts +1 -0
- package/src/__tests__/cu-unified-flow.test.ts +89 -28
- package/src/__tests__/custom-group-listing-visibility.test.ts +1 -1
- package/src/__tests__/db-schedule-syntax-migration.test.ts +4 -1
- package/src/__tests__/discord-access-request-privacy.test.ts +366 -0
- package/src/__tests__/discord-requester-notice-privacy.test.ts +87 -0
- package/src/__tests__/disk-pressure-guard.test.ts +55 -55
- package/src/__tests__/disk-pressure-routes.test.ts +10 -10
- package/src/__tests__/disk-pressure-tools.test.ts +2 -2
- package/src/__tests__/disk-usage.test.ts +35 -20
- package/src/__tests__/document-sync-tags.test.ts +379 -0
- package/src/__tests__/document-update-default-surface.test.ts +28 -0
- package/src/__tests__/dynamic-skill-background-guard.test.ts +2 -2
- package/src/__tests__/emit-signal-routing-intent.test.ts +3 -4
- package/src/__tests__/ensure-conversation-exists.test.ts +8 -5
- package/src/__tests__/events-dev-bypass-actor.test.ts +294 -1
- package/src/__tests__/evict-conversations-for-reload.test.ts +90 -0
- package/src/__tests__/file-list-tool.test.ts +12 -10
- package/src/__tests__/file-ops-service.test.ts +151 -44
- package/src/__tests__/file-read-container-boundary.test.ts +137 -0
- package/src/__tests__/file-read-tool.test.ts +21 -9
- package/src/__tests__/file-url-path-guard.test.ts +83 -0
- package/src/__tests__/filesystem-tools.test.ts +82 -66
- package/src/__tests__/guardian-outbound-http.test.ts +8 -8
- package/src/__tests__/helpers/gateway-verification-sessions-stub.ts +14 -1
- package/src/__tests__/helpers/hub-clients.ts +42 -0
- package/src/__tests__/helpers/mock-conversation.ts +46 -0
- package/src/__tests__/helpers/verification-sessions-ipc-sim.ts +61 -57
- package/src/__tests__/history-repair.test.ts +112 -12
- package/src/__tests__/host-bash-proxy.test.ts +10 -5
- package/src/__tests__/host-bash-routes.test.ts +67 -0
- package/src/__tests__/host-browser-proxy.test.ts +2 -2
- package/src/__tests__/host-cu-proxy.test.ts +3 -7
- package/src/__tests__/host-file-proxy.test.ts +3 -7
- package/src/__tests__/host-file-read-tool.test.ts +5 -1
- package/src/__tests__/host-transfer-proxy-targeted.test.ts +6 -4
- package/src/__tests__/http-conversation-lineage.test.ts +2 -5
- package/src/__tests__/http-user-message-parity.test.ts +44 -0
- package/src/__tests__/identity-routes.test.ts +13 -13
- package/src/__tests__/image-conversion.test.ts +29 -26
- package/src/__tests__/image-source-path-reinject.test.ts +2 -2
- package/src/__tests__/inference-profile-session-handler.test.ts +95 -0
- package/src/__tests__/inference-profile-session-ipc.test.ts +25 -0
- package/src/__tests__/introduction-card-resolver.test.ts +6 -1
- package/src/__tests__/list-messages-attachments.test.ts +56 -32
- package/src/__tests__/list-messages-background-tool-completion.test.ts +4 -4
- package/src/__tests__/list-messages-client-message-id.test.ts +2 -2
- package/src/__tests__/list-messages-hidden-metadata.test.ts +16 -16
- package/src/__tests__/list-messages-page-latest.test.ts +61 -61
- package/src/__tests__/list-messages-provider-error.test.ts +6 -6
- package/src/__tests__/list-messages-queued.test.ts +49 -6
- package/src/__tests__/list-messages-system-card.test.ts +4 -4
- package/src/__tests__/list-messages-tool-merge.test.ts +9 -9
- package/src/__tests__/llm-resolver-override-or-default.test.ts +141 -0
- package/src/__tests__/llm-usage-store.test.ts +55 -0
- package/src/__tests__/manual-token-reconciliation.test.ts +45 -0
- package/src/__tests__/mcp-list-plugin-servers.test.ts +250 -0
- package/src/__tests__/mcp-tool-annotations-risk.test.ts +133 -0
- package/src/__tests__/media-stream-output.test.ts +175 -0
- package/src/__tests__/media-stream-server-integration.test.ts +36 -0
- package/src/__tests__/media-stream-stt-session.test.ts +67 -0
- package/src/__tests__/migration-export-http.test.ts +1 -1
- package/src/__tests__/migration-import-commit-http.test.ts +1 -1
- package/src/__tests__/migration-import-preflight-http.test.ts +1 -1
- package/src/__tests__/migration-validate-http.test.ts +1 -1
- package/src/__tests__/mtime-cache.test.ts +65 -3
- package/src/__tests__/notification-platform-adapter.test.ts +104 -0
- package/src/__tests__/notification-schedule-notify-dedup.test.ts +1 -1
- package/src/__tests__/oauth-provider-profiles.test.ts +4 -2
- package/src/__tests__/oauth-provider-seed-logos.test.ts +65 -0
- package/src/__tests__/oauth-provider-serializer.test.ts +17 -0
- package/src/__tests__/oauth-providers-routes.test.ts +1 -0
- package/src/__tests__/openai-responses-prompt-cache.test.ts +5 -2
- package/src/__tests__/path-policy.test.ts +164 -1
- package/src/__tests__/persist-media-references.test.ts +2 -2
- package/src/__tests__/persist-unsendable-image-downscale.test.ts +5 -5
- package/src/__tests__/persist-unsendable-image.test.ts +23 -23
- package/src/__tests__/persona-resolver.test.ts +154 -0
- package/src/__tests__/plugin-api-webhook-url.test.ts +170 -0
- package/src/__tests__/plugin-import-boundary-guard.test.ts +6 -0
- package/src/__tests__/processing-flag-persist-failure.test.ts +15 -10
- package/src/__tests__/profile-availability-incomplete.test.ts +113 -0
- package/src/__tests__/profiler-routes.test.ts +13 -13
- package/src/__tests__/prompt-cache-cross-turn-stability.test.ts +593 -0
- package/src/__tests__/proxy-approval-callback.test.ts +3 -3
- package/src/__tests__/queued-message-cancel-and-steer.test.ts +189 -0
- package/src/__tests__/queued-message-steer-actor-scoping.test.ts +164 -0
- package/src/__tests__/resolve-trust-class.test.ts +20 -0
- package/src/__tests__/run-conversation-turn-persistence.test.ts +1 -1
- package/src/__tests__/run-due-schedules.test.ts +174 -0
- package/src/__tests__/runtime-attachment-metadata.test.ts +15 -7
- package/src/__tests__/scaffold-managed-skill-tool.test.ts +166 -0
- package/src/__tests__/schedule-routes.test.ts +730 -8
- package/src/__tests__/schedule-store.test.ts +540 -0
- package/src/__tests__/schedule-tools.test.ts +103 -0
- package/src/__tests__/scheduler-wake.test.ts +5 -0
- package/src/__tests__/secret-routes-platform-proxy.test.ts +29 -1
- package/src/__tests__/secure-keys.test.ts +91 -0
- package/src/__tests__/server-history-render.test.ts +97 -7
- package/src/__tests__/size-guard.test.ts +8 -8
- package/src/__tests__/skill-docs-sync-guard.test.ts +3 -2
- package/src/__tests__/skill-tool-factory.test.ts +247 -0
- package/src/__tests__/skills-install-staging.test.ts +15 -6
- package/src/__tests__/skills.test.ts +32 -0
- package/src/__tests__/slack-edit-ordering-characterization.test.ts +103 -0
- package/src/__tests__/slack-inbound-verification.test.ts +4 -1
- package/src/__tests__/slack-mention-binding-invariance.test.ts +103 -0
- package/src/__tests__/slack-mention-provider-leak-audit.test.ts +135 -0
- package/src/__tests__/starter-task-flow.test.ts +5 -4
- package/src/__tests__/steer-on-enqueue-question.test.ts +3 -2
- package/src/__tests__/stt-language-catalog-parity.test.ts +38 -0
- package/src/__tests__/subagent-allowlist-validation.test.ts +26 -4
- package/src/__tests__/subagent-cron-run-attribution.test.ts +301 -0
- package/src/__tests__/subagent-disposal.test.ts +10 -0
- package/src/__tests__/subagent-fork-notifications.test.ts +10 -0
- package/src/__tests__/subagent-fork-prompt-role.test.ts +85 -11
- package/src/__tests__/subagent-fork-spawn.test.ts +15 -7
- package/src/__tests__/subagent-manager-notify.test.ts +148 -0
- package/src/__tests__/subagent-notify-parent.test.ts +1 -1
- package/src/__tests__/subagent-persistence.test.ts +141 -0
- package/src/__tests__/subagent-role-registry.test.ts +172 -63
- package/src/__tests__/subagent-role-resolution.test.ts +88 -0
- package/src/__tests__/subagent-spawn-and-await.test.ts +48 -0
- package/src/__tests__/subagent-spawn-tool-fork.test.ts +111 -10
- package/src/__tests__/subagent-terminal-message.test.ts +88 -1
- package/src/__tests__/subagent-tool-gate-mode.test.ts +225 -10
- package/src/__tests__/subagent-tools.test.ts +1901 -108
- package/src/__tests__/surface-completion-compaction-boundary.test.ts +124 -0
- package/src/__tests__/surface-completion-in-flight-snapshot.test.ts +298 -0
- package/src/__tests__/system-prompt.test.ts +82 -7
- package/src/__tests__/tool-grant-request-escalation.test.ts +40 -1
- package/src/__tests__/tool-result-spool.test.ts +74 -18
- package/src/__tests__/tool-side-effects-documents.test.ts +174 -0
- package/src/__tests__/tool-side-effects-hook-failures.test.ts +129 -0
- package/src/__tests__/tools-audio-read.test.ts +10 -10
- package/src/__tests__/tools-get-route.test.ts +85 -12
- package/src/__tests__/tools-list-cli.test.ts +171 -0
- package/src/__tests__/top-level-renderer.test.ts +54 -1
- package/src/__tests__/trusted-contact-inline-approval-integration.test.ts +68 -1
- package/src/__tests__/trusted-contact-multichannel.test.ts +4 -1
- package/src/__tests__/turn-tail-chain.test.ts +135 -0
- package/src/__tests__/ui-choice-copy-surfaces.test.ts +6 -4
- package/src/__tests__/ui-visual-surface.test.ts +5 -4
- package/src/__tests__/ui-work-result-surface.test.ts +5 -4
- package/src/__tests__/unified-turn-context-visible-app.test.ts +7 -8
- package/src/__tests__/usage-routes.test.ts +62 -0
- package/src/__tests__/verification-outbound-delivery.test.ts +176 -0
- package/src/__tests__/visible-app-context.test.ts +20 -4
- package/src/__tests__/web-search-history.test.ts +96 -0
- package/src/__tests__/workspace-migration-139-clear-renamed-cost-profile-label.test.ts +6 -6
- package/src/__tests__/workspace-migration-140-repair-seed-pinned-memory-v3-live.test.ts +330 -0
- package/src/__tests__/workspace-migration-142-consolidate-voice-front-door.test.ts +150 -0
- package/src/acp/failure-error.ts +3 -5
- package/src/agent/attachments.ts +26 -24
- package/src/agent/history-repair/history-repair.ts +27 -26
- package/src/agent/image-optimize.ts +6 -6
- package/src/agent/loop.ts +40 -22
- package/src/agent/message-types.ts +3 -3
- package/src/api/constants/document-tools.ts +55 -0
- package/src/api/events/question-answered.ts +53 -0
- package/src/api/events/tool-result.ts +7 -0
- package/src/api/index.ts +13 -0
- package/src/api/responses/conversation-message.ts +11 -0
- package/src/api/surfaces-oauth-connect.test.ts +96 -0
- package/src/api/surfaces.ts +36 -8
- package/src/approvals/guardian-channel-delivery.ts +83 -0
- package/src/approvals/guardian-expiry-notifier.ts +13 -11
- package/src/approvals/guardian-request-resolvers.ts +222 -135
- package/src/calls/__tests__/tts-text-sanitizer.test.ts +13 -0
- package/src/calls/__tests__/voice-session-bridge.test.ts +131 -12
- package/src/calls/__tests__/voice-triage-escalate.test.ts +106 -0
- package/src/calls/call-controller.ts +30 -2
- package/src/calls/call-speech-output.ts +12 -4
- package/src/calls/call-transport.ts +18 -1
- package/src/calls/media-stream-output.ts +53 -5
- package/src/calls/media-stream-server.ts +21 -1
- package/src/calls/media-stream-stt-session.ts +49 -0
- package/src/calls/telephony-synthesis-language.ts +84 -0
- package/src/calls/tts-text-sanitizer.ts +12 -5
- package/src/calls/voice-session-bridge.ts +86 -17
- package/src/calls/voice-triage-escalate.ts +54 -7
- package/src/channels/__tests__/gateway-verification-sessions.test.ts +3 -3
- package/src/channels/__tests__/plugin-channel-declarations.test.ts +161 -0
- package/src/channels/config.ts +20 -2
- package/src/channels/gateway-verification-sessions.ts +7 -2
- package/src/channels/plugin-channel-declarations.ts +108 -0
- package/src/channels/types.ts +37 -1
- package/src/cli/AGENTS.md +5 -2
- package/src/cli/commands/__tests__/cache.test.ts +7 -4
- package/src/cli/commands/__tests__/cli-test-harness.ts +82 -0
- package/src/cli/commands/__tests__/conversations-wake.test.ts +51 -6
- package/src/cli/commands/__tests__/inference-models.test.ts +14 -0
- package/src/cli/commands/__tests__/inference-profiles.test.ts +44 -1
- package/src/cli/commands/__tests__/inference-providers.test.ts +7 -0
- package/src/cli/commands/__tests__/plugins.test.ts +692 -0
- package/src/cli/commands/__tests__/schedules.test.ts +136 -37
- package/src/cli/commands/conversations.ts +7 -3
- package/src/cli/commands/credentials.help.ts +7 -0
- package/src/cli/commands/inference-profiles.ts +9 -3
- package/src/cli/commands/inference.help.ts +16 -2
- package/src/cli/commands/mcp.help.ts +17 -6
- package/src/cli/commands/mcp.ts +9 -0
- package/src/cli/commands/memory/memory-retrospective.ts +18 -0
- package/src/cli/commands/notifications.ts +3 -6
- package/src/cli/commands/oauth/connect-surface-guidance.test.ts +16 -0
- package/src/cli/commands/oauth/connect-surface-guidance.ts +25 -7
- package/src/cli/commands/oauth/connect.test.ts +26 -1
- package/src/cli/commands/oauth/connect.ts +4 -1
- package/src/cli/commands/oauth/index.help.ts +3 -1
- package/src/cli/commands/oauth/providers.ts +3 -0
- package/src/cli/commands/plugins.help.ts +25 -2
- package/src/cli/commands/plugins.ts +289 -10
- package/src/cli/commands/schedules.help.ts +14 -12
- package/src/cli/commands/schedules.ts +19 -5
- package/src/cli/commands/stt.help.ts +27 -2
- package/src/cli/commands/tools.help.ts +4 -2
- package/src/cli/commands/tools.ts +38 -0
- package/src/cli/lib/__tests__/inspect-plugin.test.ts +1 -0
- package/src/cli/lib/__tests__/install-from-github.test.ts +79 -0
- package/src/cli/lib/__tests__/install-from-platform.test.ts +47 -1
- package/src/cli/lib/__tests__/plugin-surfaces.test.ts +86 -7
- package/src/cli/lib/__tests__/upgrade-plugin.test.ts +161 -0
- package/src/cli/lib/install-from-github.ts +61 -0
- package/src/cli/lib/install-from-platform.ts +13 -2
- package/src/cli/lib/plugin-surfaces.ts +124 -4
- package/src/cli/lib/upgrade-plugin.ts +60 -0
- package/src/cli.ts +1 -1
- package/src/config/__tests__/default-profile-catalog.test.ts +11 -1
- package/src/config/__tests__/deployment-context-defaults.test.ts +73 -0
- package/src/config/__tests__/memory-retrospective-schema.test.ts +40 -0
- package/src/config/__tests__/plugin-updates-schema.test.ts +43 -0
- package/src/config/__tests__/profile-tool-support.test.ts +131 -0
- package/src/config/bundled-skills/AGENTS.md +1 -1
- package/src/config/bundled-skills/acp/SKILL.md +6 -7
- package/src/config/bundled-skills/app-builder/SKILL.md +1 -1
- package/src/config/bundled-skills/app-builder/references/RESPONSIVE.md +3 -3
- package/src/config/bundled-skills/media-processing/services/preprocess.ts +14 -4
- package/src/config/bundled-skills/phone-calls/references/CONFIG.md +15 -15
- package/src/config/bundled-skills/schedule/SKILL.md +5 -1
- package/src/config/bundled-skills/schedule/TOOLS.json +2 -2
- package/src/config/bundled-skills/settings/TOOLS.json +3 -3
- package/src/config/bundled-skills/settings/tools/voice-config-update.ts +1 -0
- package/src/config/bundled-skills/subagent/SKILL.md +53 -22
- package/src/config/bundled-skills/subagent/TOOLS.json +14 -13
- package/src/config/bundled-skills/transcribe/tools/transcribe-media.test.ts +22 -1
- package/src/config/bundled-skills/transcribe/tools/transcribe-media.ts +9 -2
- package/src/config/bundled-skills/visualize/SKILL.md +1 -1
- package/src/config/call-site-defaults.ts +4 -5
- package/src/config/default-profile-catalog.ts +9 -5
- package/src/config/feature-flag-registry.json +25 -17
- package/src/config/llm-resolver.ts +45 -15
- package/src/config/loader.ts +75 -1
- package/src/config/profile-tool-support.ts +110 -0
- package/src/config/sanitize-for-transfer.ts +11 -0
- package/src/config/schema.ts +4 -0
- package/src/config/schemas/__tests__/live-voice.test.ts +142 -4
- package/src/config/schemas/__tests__/stt.test.ts +20 -5
- package/src/config/schemas/call-site-catalog.ts +4 -4
- package/src/config/schemas/calls.ts +0 -4
- package/src/config/schemas/channels.ts +13 -0
- package/src/config/schemas/live-voice.ts +82 -23
- package/src/config/schemas/llm.ts +1 -1
- package/src/config/schemas/memory-retrospective.ts +25 -0
- package/src/config/schemas/plugin-updates.ts +59 -0
- package/src/config/schemas/services.ts +8 -4
- package/src/config/schemas/stt.ts +18 -5
- package/src/config/schemas/timeouts.ts +1 -1
- package/src/config/schemas/tts.ts +63 -0
- package/src/contacts/__tests__/guardian-delivery-reader.test.ts +16 -0
- package/src/contacts/guardian-delivery-reader.ts +29 -2
- package/src/context/compactor.ts +4 -4
- package/src/context/post-turn-tool-result-truncation.ts +32 -16
- package/src/context/strip-injections.ts +42 -0
- package/src/context/tool-result-spool.ts +32 -18
- package/src/credential-execution/ces-runtime.ts +3 -0
- package/src/credential-execution/client.ts +22 -1
- package/src/credential-execution/process-manager.test.ts +99 -0
- package/src/credential-execution/process-manager.ts +62 -16
- package/src/daemon/__tests__/abort-null-controller.test.ts +141 -5
- package/src/daemon/__tests__/conversation-surfaces-launch.test.ts +8 -12
- package/src/daemon/__tests__/install-assistant-command.test.ts +234 -0
- package/src/daemon/__tests__/turn-tail-assistant-reply-notify.test.ts +2 -4
- package/src/daemon/__tests__/turn-tail-deleted-conversation.test.ts +20 -29
- package/src/daemon/channel-ui-capability.ts +4 -4
- package/src/daemon/conversation-agent-loop-handlers.ts +117 -61
- package/src/daemon/conversation-agent-loop.ts +318 -148
- package/src/daemon/conversation-attachments.ts +1 -1
- package/src/daemon/conversation-error.ts +8 -1
- package/src/daemon/conversation-initial-prompt.ts +16 -38
- package/src/daemon/conversation-lifecycle.ts +154 -32
- package/src/daemon/conversation-messaging.ts +106 -43
- package/src/daemon/conversation-process.ts +131 -33
- package/src/daemon/conversation-queue-manager.ts +10 -0
- package/src/daemon/conversation-runtime-assembly.ts +9 -4
- package/src/daemon/conversation-store.ts +52 -3
- package/src/daemon/conversation-surface-state.ts +9 -0
- package/src/daemon/conversation-surfaces.ts +403 -210
- package/src/daemon/conversation-tool-setup.ts +56 -7
- package/src/daemon/conversation-turn-finalize.ts +134 -89
- package/src/daemon/conversation.ts +111 -27
- package/src/daemon/disk-pressure-guard-lifecycle.ts +2 -2
- package/src/daemon/disk-pressure-guard.ts +3 -3
- package/src/daemon/doordash-steps.ts +5 -5
- package/src/daemon/external-plugins-bootstrap.ts +7 -1
- package/src/daemon/handlers/config-channels.ts +48 -5
- package/src/daemon/handlers/conversation-history.ts +1 -1
- package/src/daemon/handlers/conversations.ts +62 -17
- package/src/daemon/handlers/shared.ts +12 -0
- package/src/daemon/handlers/skills.ts +11 -1
- package/src/daemon/identity-helpers.ts +38 -10
- package/src/daemon/install-assistant-command.ts +322 -0
- package/src/daemon/lifecycle.ts +14 -3
- package/src/daemon/message-types/conversations.ts +6 -7
- package/src/daemon/message-types/sync.ts +1 -0
- package/src/daemon/persist-media-references.ts +23 -14
- package/src/daemon/process-message.ts +6 -6
- package/src/daemon/tool-setup-types.ts +19 -122
- package/src/daemon/tool-side-effects.ts +17 -94
- package/src/daemon/trust-context-types.ts +39 -0
- package/src/daemon/trust-context.ts +14 -6
- package/src/daemon/turn-tail-chain.ts +73 -0
- package/src/daemon/web-search-history.ts +47 -33
- package/src/heartbeat/heartbeat-service.ts +6 -6
- package/src/inbound/__tests__/oauth-callback-url.test.ts +83 -0
- package/src/inbound/oauth-callback-url.ts +61 -0
- package/src/ipc/__tests__/socket-path.test.ts +21 -0
- package/src/ipc/assistant-server.ts +17 -10
- package/src/ipc/cli-client.ts +10 -1
- package/src/ipc/routes/__tests__/documents-sync-ipc-routes.test.ts +56 -0
- package/src/ipc/routes/documents-sync-ipc-routes.ts +42 -0
- package/src/ipc/socket-cleanup.ts +9 -7
- package/src/ipc/socket-path.ts +4 -96
- package/src/live-voice/__tests__/live-voice-agent-turn.test.ts +1 -104
- package/src/live-voice/__tests__/live-voice-archive.test.ts +16 -16
- package/src/live-voice/__tests__/live-voice-attach-image.test.ts +209 -0
- package/src/live-voice/__tests__/live-voice-connection.test.ts +20 -0
- package/src/live-voice/__tests__/live-voice-events.test.ts +7 -8
- package/src/live-voice/__tests__/live-voice-flux-turn-end.test.ts +932 -0
- package/src/live-voice/__tests__/live-voice-integration.test.ts +12 -5
- package/src/live-voice/__tests__/live-voice-metrics.test.ts +115 -8
- package/src/live-voice/__tests__/live-voice-photo.test.ts +100 -0
- package/src/live-voice/__tests__/live-voice-progress.test.ts +179 -146
- package/src/live-voice/__tests__/live-voice-session-telemetry.test.ts +265 -0
- package/src/live-voice/__tests__/live-voice-stt.test.ts +324 -2
- package/src/live-voice/__tests__/live-voice-triage-escalate.test.ts +131 -2
- package/src/live-voice/__tests__/live-voice-tts-session.test.ts +0 -483
- package/src/live-voice/__tests__/live-voice-tts.test.ts +148 -2
- package/src/live-voice/__tests__/live-voice-vad.test.ts +313 -32
- package/src/live-voice/__tests__/progress-narration.test.ts +214 -0
- package/src/live-voice/__tests__/progress-phrases.test.ts +167 -0
- package/src/live-voice/__tests__/protocol.test.ts +52 -0
- package/src/live-voice/__tests__/runtime-websocket-shell.test.ts +3 -0
- package/src/live-voice/live-voice-archive.ts +13 -13
- package/src/live-voice/live-voice-connection.ts +7 -1
- package/src/live-voice/live-voice-metrics.ts +57 -32
- package/src/live-voice/live-voice-photo.ts +168 -0
- package/src/live-voice/live-voice-session.ts +1137 -351
- package/src/live-voice/live-voice-tts.ts +18 -2
- package/src/live-voice/progress-narration.ts +277 -0
- package/src/live-voice/progress-phrases.ts +105 -2
- package/src/live-voice/protocol.ts +110 -2
- package/src/mcp/__tests__/mcp-auth-orchestrator.test.ts +0 -1
- package/src/mcp/__tests__/mcp-oauth-client-registration.test.ts +196 -0
- package/src/mcp/__tests__/mcp-oauth-provider.test.ts +9 -9
- package/src/mcp/client.ts +14 -7
- package/src/mcp/mcp-auth-orchestrator.ts +13 -22
- package/src/mcp/mcp-oauth-provider.ts +195 -240
- package/src/messaging/providers/__tests__/callback-routing.test.ts +5 -1
- package/src/messaging/providers/__tests__/transport-dispatch.test.ts +66 -4
- package/src/messaging/providers/callback-routing.ts +1 -0
- package/src/messaging/providers/discord/api.ts +312 -0
- package/src/messaging/providers/discord/dm-delivery.test.ts +204 -0
- package/src/messaging/providers/discord/render.test.ts +204 -0
- package/src/messaging/providers/discord/render.ts +195 -0
- package/src/messaging/providers/discord/send.test.ts +179 -0
- package/src/messaging/providers/discord/send.ts +212 -0
- package/src/messaging/providers/discord/transport.ts +81 -0
- package/src/messaging/providers/index.ts +3 -1
- package/src/messaging/providers/retry-policy.test.ts +247 -0
- package/src/messaging/providers/retry-policy.ts +206 -0
- package/src/messaging/providers/slack/mention-source.test.ts +285 -0
- package/src/messaging/providers/slack/mention-source.ts +287 -0
- package/src/messaging/providers/telegram-bot/api.ts +47 -130
- package/src/messaging/providers/whatsapp/api.ts +47 -136
- package/src/monitoring/__tests__/db-integrity-sample.test.ts +4 -1
- package/src/monitoring/__tests__/plugin-auto-update.test.ts +495 -0
- package/src/monitoring/db-integrity-sample.ts +6 -1
- package/src/monitoring/plugin-auto-update.ts +463 -0
- package/src/monitoring/resource-sampler.ts +21 -5
- package/src/monitoring/worker.ts +13 -0
- package/src/notifications/__tests__/assistant-reply-producer.test.ts +375 -2
- package/src/notifications/__tests__/broadcaster.test.ts +37 -1
- package/src/notifications/__tests__/copy-composer.test.ts +165 -0
- package/src/notifications/__tests__/emit-signal-pipeline-failure.test.ts +205 -0
- package/src/notifications/__tests__/notification-utils.test.ts +249 -0
- package/src/notifications/adapters/platform.ts +31 -2
- package/src/notifications/assistant-reply-producer.ts +79 -32
- package/src/notifications/broadcaster.ts +30 -10
- package/src/notifications/copy-composer.ts +69 -0
- package/src/notifications/emit-signal.ts +82 -15
- package/src/notifications/events-store.ts +10 -2
- package/src/notifications/home-feed-side-effect.ts +6 -3
- package/src/notifications/notification-utils.ts +173 -5
- package/src/notifications/signal.ts +13 -0
- package/src/oauth/AGENTS.md +4 -2
- package/src/oauth/byo-connection.ts +6 -1
- package/src/oauth/credential-token-resolver.ts +3 -10
- package/src/oauth/manual-token-connection.ts +29 -64
- package/src/oauth/manual-token-providers.ts +62 -0
- package/src/oauth/provider-serializer.ts +13 -0
- package/src/oauth/seed-providers.ts +62 -11
- package/src/onboarding/onboarding-events-store.ts +91 -1
- package/src/permissions/question-prompter.test.ts +4 -0
- package/src/permissions/question-prompter.ts +21 -4
- package/src/persistence/__tests__/conversation-lineage.test.ts +248 -0
- package/src/persistence/attachments-store.ts +15 -15
- package/src/persistence/conversation-attention-store.ts +30 -10
- package/src/persistence/conversation-bootstrap.ts +17 -0
- package/src/persistence/conversation-crud.ts +363 -97
- package/src/persistence/conversation-group-migration.ts +1 -1
- package/src/persistence/conversation-key-store.ts +38 -10
- package/src/persistence/conversation-lineage.ts +226 -0
- package/src/persistence/conversation-plugin-facade.ts +28 -3
- package/src/persistence/conversation-queries.ts +207 -33
- package/src/persistence/conversation-types.ts +107 -1
- package/src/persistence/jobs-store.ts +32 -8
- package/src/persistence/llm-usage-store.ts +22 -0
- package/src/persistence/migrations/361-normalize-managed-connection-rows.test.ts +168 -0
- package/src/persistence/migrations/361-normalize-managed-connection-rows.ts +76 -0
- package/src/persistence/migrations/362-add-conversation-subagent-kind.test.ts +187 -0
- package/src/persistence/migrations/362-add-conversation-subagent-kind.ts +93 -0
- package/src/persistence/migrations/363-backfill-schedule-inference-profile.test.ts +229 -0
- package/src/persistence/migrations/363-backfill-schedule-inference-profile.ts +127 -0
- package/src/persistence/migrations/364-add-schedule-source-key.test.ts +120 -0
- package/src/persistence/migrations/364-add-schedule-source-key.ts +43 -0
- package/src/persistence/migrations/365-add-conversation-fork-strategy.ts +27 -0
- package/src/persistence/schema/conversations.ts +32 -0
- package/src/persistence/schema/infrastructure.ts +3 -0
- package/src/persistence/steps.ts +30 -0
- package/src/persistence/subagent-store.ts +148 -6
- package/src/platform/managed-speech.test.ts +20 -0
- package/src/platform/managed-speech.ts +9 -0
- package/src/plugin-api/__tests__/oauth-callback-url-export.test.ts +29 -0
- package/src/plugin-api/index.ts +28 -5
- package/src/plugin-api/webhook-url.ts +151 -0
- package/src/plugins/__tests__/installed-plugin-dirs.test.ts +82 -0
- package/src/plugins/__tests__/mcp-servers.test.ts +370 -0
- package/src/plugins/__tests__/source-fingerprint.test.ts +5 -1
- package/src/plugins/collect-source-versions.ts +4 -27
- package/src/plugins/defaults/exploration-drift/hooks/post-tool-use.ts +5 -5
- package/src/plugins/defaults/exploration-drift/package.json +1 -1
- package/src/plugins/defaults/image-recovery/hooks/post-model-call.ts +4 -2
- package/src/plugins/defaults/image-recovery/recover.ts +67 -57
- package/src/plugins/defaults/index.ts +1 -1
- package/src/plugins/defaults/memory/__tests__/conversation-queries.test.ts +64 -18
- package/src/plugins/defaults/memory/__tests__/jobs-worker-outcome-contract.test.ts +242 -0
- package/src/plugins/defaults/memory/__tests__/jobs-worker-retrospective-sweep.test.ts +24 -1
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-enqueue.test.ts +48 -3
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-job.test.ts +511 -19
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-prompt.test.ts +39 -3
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-provider-path.test.ts +493 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-wake-chain.test.ts +5 -4
- package/src/plugins/defaults/memory/__tests__/memory-tier-boundary-guard.test.ts +5 -1
- package/src/plugins/defaults/memory/job-handlers.ts +87 -5
- package/src/plugins/defaults/memory/jobs-worker.ts +60 -6
- package/src/plugins/defaults/memory/memory-retrospective-constants.ts +11 -0
- package/src/plugins/defaults/memory/memory-retrospective-enqueue.ts +29 -0
- package/src/plugins/defaults/memory/memory-retrospective-job.ts +342 -60
- package/src/plugins/defaults/memory/memory-retrospective-prompt.ts +23 -3
- package/src/plugins/defaults/memory/substrate/consolidation-lock.ts +9 -4
- package/src/plugins/defaults/memory/substrate/skill-content.ts +8 -1
- package/src/plugins/defaults/memory/substrate/sweep-job.ts +7 -0
- package/src/plugins/defaults/memory/v1/graph/injection.ts +1 -1
- package/src/plugins/defaults/memory/v2/router.ts +8 -0
- package/src/plugins/defaults/turn-context/unified-turn-context.ts +1 -2
- package/src/plugins/external-plugin-loader.ts +64 -8
- package/src/plugins/installed-plugin-dirs.ts +101 -0
- package/src/plugins/mcp-servers.ts +343 -0
- package/src/plugins/mtime-cache.ts +34 -22
- package/src/plugins/types.ts +69 -8
- package/src/prompts/persona-resolver.ts +6 -2
- package/src/prompts/system-prompt.ts +72 -17
- package/src/providers/__tests__/inference.test.ts +40 -0
- package/src/providers/__tests__/preflight-resolved-config.test.ts +58 -1
- package/src/providers/__tests__/retry-callsite.test.ts +72 -2
- package/src/providers/__tests__/vellum-connection-routing.test.ts +42 -11
- package/src/providers/anthropic/client.ts +112 -95
- package/src/providers/connection-resolution.ts +35 -1
- package/src/providers/gemini/client.ts +5 -5
- package/src/providers/inference/adapter-factory.ts +16 -10
- package/src/providers/inference/auth.ts +16 -0
- package/src/providers/inference/connection-availability.ts +164 -12
- package/src/providers/inference/connections.ts +25 -1
- package/src/providers/media-resolve.ts +21 -11
- package/src/providers/model-catalog.ts +15 -0
- package/src/providers/openai/chat-completions-provider.ts +4 -4
- package/src/providers/openai/responses-provider.ts +12 -25
- package/src/providers/registry.ts +9 -4
- package/src/providers/retry.ts +25 -0
- package/src/providers/server-tool-pairing.ts +134 -0
- package/src/providers/speech-to-text/__tests__/deepgram-flux-frames.test.ts +433 -0
- package/src/providers/speech-to-text/__tests__/deepgram-flux-realtime.test.ts +620 -0
- package/src/providers/speech-to-text/__tests__/provider-catalog.test.ts +34 -0
- package/src/providers/speech-to-text/__tests__/resolve.test.ts +472 -12
- package/src/providers/speech-to-text/deepgram-flux-frames.ts +395 -0
- package/src/providers/speech-to-text/deepgram-flux-realtime.ts +719 -0
- package/src/providers/speech-to-text/deepgram-realtime.test.ts +283 -1
- package/src/providers/speech-to-text/deepgram-realtime.ts +117 -3
- package/src/providers/speech-to-text/deepgram.ts +1 -0
- package/src/providers/speech-to-text/provider-catalog.ts +137 -8
- package/src/providers/speech-to-text/resolve.ts +105 -12
- package/src/providers/speech-to-text/vellum-managed.ts +7 -1
- package/src/providers/speech-to-text/vellum-speech-relay-connection.ts +3 -1
- package/src/providers/types.ts +26 -6
- package/src/providers/vellum-model-routing.ts +6 -7
- package/src/runtime/AGENTS.md +2 -0
- package/src/runtime/__tests__/agent-wake.test.ts +16 -16
- package/src/runtime/__tests__/background-job-runner.test.ts +0 -144
- package/src/runtime/__tests__/desktop-presence.test.ts +162 -0
- package/src/runtime/__tests__/local-actor-identity-force-refresh.test.ts +104 -0
- package/src/runtime/agent-wake.ts +63 -24
- package/src/runtime/assistant-event-hub.ts +110 -12
- package/src/runtime/auth/__tests__/guard-tests.test.ts +2 -0
- package/src/runtime/auth/__tests__/scopes.test.ts +20 -0
- package/src/runtime/auth/same-actor.ts +56 -8
- package/src/runtime/auth/scopes.ts +12 -0
- package/src/runtime/auth/types.ts +2 -0
- package/src/runtime/background-job-runner.ts +5 -27
- package/src/runtime/channel-readiness-service.ts +59 -3
- package/src/runtime/channel-readiness-types.ts +18 -0
- package/src/runtime/desktop-presence.ts +69 -0
- package/src/runtime/http-server.ts +12 -0
- package/src/runtime/local-actor-identity.ts +29 -14
- package/src/runtime/migrations/__tests__/vbundle-symlink-streaming-importer.test.ts +0 -0
- package/src/runtime/routes/__tests__/channel-verification-routes.test.ts +1 -1
- package/src/runtime/routes/__tests__/client-routes.test.ts +186 -14
- package/src/runtime/routes/__tests__/conversation-list-routes.test.ts +547 -1
- package/src/runtime/routes/__tests__/conversation-management-routes.test.ts +162 -7
- package/src/runtime/routes/__tests__/conversation-query-routes.test.ts +14 -10
- package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +340 -3
- package/src/runtime/routes/__tests__/llm-call-sites-routes.test.ts +28 -1
- package/src/runtime/routes/__tests__/plugins-routes.test.ts +217 -1
- package/src/runtime/routes/__tests__/question-routes.test.ts +64 -0
- package/src/runtime/routes/__tests__/retrospective-routes.test.ts +20 -0
- package/src/runtime/routes/__tests__/skill-history-route.test.ts +116 -0
- package/src/runtime/routes/__tests__/sse-actor-principal-heal.test.ts +165 -0
- package/src/runtime/routes/__tests__/stt-routes.test.ts +25 -0
- package/src/runtime/routes/__tests__/surface-action-routes.test.ts +2 -0
- package/src/runtime/routes/__tests__/wake-conversation-routes.test.ts +60 -2
- package/src/runtime/routes/archive-utils.ts +16 -12
- package/src/runtime/routes/attachment-routes.ts +13 -6
- package/src/runtime/routes/canned-reply-release.ts +54 -0
- package/src/runtime/routes/channel-availability-routes.ts +32 -14
- package/src/runtime/routes/channel-verification-routes.ts +2 -27
- package/src/runtime/routes/chatgpt-subscription-auth-routes.ts +5 -0
- package/src/runtime/routes/client-routes.ts +114 -2
- package/src/runtime/routes/conversation-cli-routes.ts +10 -12
- package/src/runtime/routes/conversation-list-routes.ts +59 -10
- package/src/runtime/routes/conversation-management-routes.ts +47 -18
- package/src/runtime/routes/conversation-query-routes.ts +32 -53
- package/src/runtime/routes/conversation-routes.ts +360 -293
- package/src/runtime/routes/credential-prompt-routes.ts +20 -2
- package/src/runtime/routes/debug-routes.ts +4 -1
- package/src/runtime/routes/default-provider-routes.ts +7 -11
- package/src/runtime/routes/documents-routes.ts +44 -1
- package/src/runtime/routes/events-routes.ts +29 -1
- package/src/runtime/routes/host-app-control-routes.ts +2 -0
- package/src/runtime/routes/host-bash-routes.ts +2 -0
- package/src/runtime/routes/host-browser-routes.ts +2 -0
- package/src/runtime/routes/host-cu-routes.ts +2 -0
- package/src/runtime/routes/host-file-routes.ts +2 -0
- package/src/runtime/routes/host-transfer-routes.ts +4 -0
- package/src/runtime/routes/identity-routes.ts +6 -6
- package/src/runtime/routes/inbound-message-handler.ts +5 -5
- package/src/runtime/routes/inbound-stages/acl-enforcement.ts +7 -3
- package/src/runtime/routes/inbound-stages/background-dispatch.ts +14 -0
- package/src/runtime/routes/inbound-stages/bootstrap-intercept.ts +2 -2
- package/src/runtime/routes/inbound-stages/guardian-activation-intercept.test.ts +14 -11
- package/src/runtime/routes/inbound-stages/guardian-activation-intercept.ts +14 -17
- package/src/runtime/routes/inference-profile-availability-guard.ts +171 -0
- package/src/runtime/routes/inference-profile-session-handler.ts +26 -2
- package/src/runtime/routes/inference-profile-session-routes.ts +2 -1
- package/src/runtime/routes/inference-profiles-routes.ts +152 -73
- package/src/runtime/routes/inference-provider-connection-routes.ts +0 -16
- package/src/runtime/routes/llm-call-sites-routes.ts +13 -0
- package/src/runtime/routes/log-export-routes.ts +1 -1
- package/src/runtime/routes/mcp-auth-routes.ts +141 -57
- package/src/runtime/routes/notification-routes.ts +0 -14
- package/src/runtime/routes/oauth-providers.ts +1 -0
- package/src/runtime/routes/plugins-routes.ts +127 -4
- package/src/runtime/routes/profiler-routes.ts +4 -2
- package/src/runtime/routes/question-routes.ts +28 -10
- package/src/runtime/routes/retrospective-routes.ts +9 -3
- package/src/runtime/routes/schedule-routes.ts +210 -12
- package/src/runtime/routes/secret-routes.ts +40 -23
- package/src/runtime/routes/settings-routes.ts +41 -13
- package/src/runtime/routes/skills-routes.ts +79 -0
- package/src/runtime/routes/sse-actor-principal-heal.ts +113 -0
- package/src/runtime/routes/stt-routes.ts +31 -25
- package/src/runtime/routes/ui-snapshot-routes.ts +1 -0
- package/src/runtime/routes/wake-conversation-routes.ts +23 -0
- package/src/runtime/services/__tests__/conversation-serializer.test.ts +1 -0
- package/src/runtime/services/conversation-serializer.ts +24 -9
- package/src/runtime/sync/documents-sidecar-publish.test.ts +101 -0
- package/src/runtime/sync/resource-sync-events.ts +79 -1
- package/src/runtime/sync/worker-daemon-notify.test.ts +34 -0
- package/src/runtime/sync/worker-daemon-notify.ts +44 -6
- package/src/runtime/verification-outbound-actions.ts +250 -222
- package/src/runtime/verification-templates.ts +41 -30
- package/src/schedule/__tests__/plugin-schedule-declarations.test.ts +614 -0
- package/src/schedule/__tests__/plugin-schedule-reconciler.test.ts +1066 -0
- package/src/schedule/inference-profile.ts +64 -3
- package/src/schedule/plugin-schedule-declarations.ts +518 -0
- package/src/schedule/plugin-schedule-reconciler.ts +605 -0
- package/src/schedule/plugin-schedules-gate.ts +23 -0
- package/src/schedule/schedule-store.ts +520 -26
- package/src/schedule/scheduler.ts +38 -0
- package/src/security/credential-backend.ts +51 -4
- package/src/security/encrypted-store.ts +64 -42
- package/src/security/oauth2.ts +6 -22
- package/src/security/secure-keys.ts +55 -7
- package/src/skills/catalog-install.ts +20 -7
- package/src/skills/inline-command-runner.ts +2 -7
- package/src/skills/skill-history.test.ts +269 -0
- package/src/skills/skill-history.ts +250 -0
- package/src/skills/skillssh-registry.ts +1 -1
- package/src/stt/__tests__/daemon-batch-transcriber.test.ts +22 -0
- package/src/stt/__tests__/language-metadata.test.ts +85 -0
- package/src/stt/__tests__/speech-energy.test.ts +79 -0
- package/src/stt/__tests__/types.test.ts +94 -0
- package/src/stt/daemon-batch-transcriber.ts +18 -6
- package/src/stt/language-metadata.ts +65 -0
- package/src/stt/speech-energy.ts +115 -12
- package/src/stt/stt-stream-session.ts +8 -4
- package/src/stt/types.ts +119 -0
- package/src/subagent/__tests__/consult-prompt.test.ts +25 -0
- package/src/subagent/consult-prompt.ts +3 -1
- package/src/subagent/consult-transcript.ts +1 -1
- package/src/subagent/manager.ts +304 -44
- package/src/subagent/progress-events.ts +35 -0
- package/src/subagent/role-resolution.ts +88 -0
- package/src/subagent/types.ts +303 -67
- package/src/subagent/validate-allowlists.ts +3 -4
- package/src/telegram/__tests__/webhook-health.test.ts +73 -1
- package/src/telegram/webhook-health.ts +101 -3
- package/src/telemetry/__tests__/telemetry-event-fixtures.ts +2 -0
- package/src/telemetry/activation-funnel.ts +6 -1
- package/src/telemetry/live-voice-funnel.ts +75 -0
- package/src/telemetry/telemetry-event-sources.ts +6 -0
- package/src/telemetry/telemetry-wire-source.json +1 -1
- package/src/telemetry/telemetry-wire.generated.ts +2 -0
- package/src/telemetry/types.ts +23 -4
- package/src/telemetry/usage-telemetry-reporter.test.ts +4 -0
- package/src/tools/__tests__/subagent-workflow-tool-input-schemas.test.ts +1 -1
- package/src/tools/ask-question/ask-question-tool.test.ts +118 -3
- package/src/tools/ask-question/ask-question-tool.ts +56 -2
- package/src/tools/browser/browser-execution.ts +1 -1
- package/src/tools/calls/call-start.ts +8 -5
- package/src/tools/conversation-groups/move_to_group.ts +10 -12
- package/src/tools/document/document-tool.ts +11 -4
- package/src/tools/execution-timeout.ts +9 -2
- package/src/tools/executor.ts +7 -6
- package/src/tools/filesystem/edit.ts +1 -1
- package/src/tools/filesystem/list.ts +1 -1
- package/src/tools/filesystem/read.ts +7 -13
- package/src/tools/filesystem/search.ts +23 -21
- package/src/tools/filesystem/write.ts +4 -1
- package/src/tools/host-filesystem/edit.ts +7 -2
- package/src/tools/host-filesystem/read.ts +21 -8
- package/src/tools/host-filesystem/write.ts +10 -2
- package/src/tools/host-terminal/host-shell.ts +2 -2
- package/src/tools/mcp/mcp-tool-factory.ts +25 -1
- package/src/tools/permission-checker.ts +8 -3
- package/src/tools/schedule/delete.ts +13 -1
- package/src/tools/schedule/list.ts +14 -3
- package/src/tools/schedule/update.ts +56 -33
- package/src/tools/shared/filesystem/audio-read.ts +6 -4
- package/src/tools/shared/filesystem/file-ops-service.ts +176 -116
- package/src/tools/shared/filesystem/image-read.ts +12 -12
- package/src/tools/shared/filesystem/path-policy.ts +70 -5
- package/src/tools/shared/filesystem/size-guard.ts +6 -6
- package/src/tools/shared/input-misuse.ts +72 -0
- package/src/tools/skills/scaffold-managed.ts +82 -0
- package/src/tools/skills/skill-tool-factory.ts +14 -1
- package/src/tools/subagent/consult-deadline.ts +12 -7
- package/src/tools/subagent/message.ts +3 -1
- package/src/tools/subagent/read.ts +66 -4
- package/src/tools/subagent/spawn.ts +500 -78
- package/src/tools/tool-approval-handler.ts +45 -4
- package/src/tools/types.ts +22 -3
- package/src/tools/ui-surface/surface-shape-docs.ts +1 -1
- package/src/tools/ui-surface/visual-validation.ts +2 -0
- package/src/tts/__tests__/provider-adapters.test.ts +147 -0
- package/src/tts/__tests__/speakable-segments.test.ts +470 -0
- package/src/tts/language-voices.ts +23 -0
- package/src/tts/providers/deepgram-provider.ts +3 -1
- package/src/tts/providers/elevenlabs-provider.ts +73 -1
- package/src/tts/providers/xai-provider.ts +28 -2
- package/src/tts/speakable-segments.ts +293 -23
- package/src/tts/synthesis-stream.ts +7 -0
- package/src/tts/types.ts +7 -0
- package/src/usage/__tests__/subagent-attribution.test.ts +109 -0
- package/src/usage/subagent-attribution.ts +108 -0
- package/src/util/__tests__/language-subtag.test.ts +54 -0
- package/src/util/__tests__/sqlite-retry.test.ts +112 -0
- package/src/util/abort-reasons.ts +16 -0
- package/src/util/ansi.ts +44 -0
- package/src/util/disk-usage.ts +17 -13
- package/src/util/image-conversion.ts +20 -13
- package/src/util/language-subtag.ts +43 -0
- package/src/util/schedule-source-key.ts +23 -0
- package/src/util/unicode.ts +1 -1
- package/src/util/worker-process.ts +7 -1
- package/src/workspace/git-service.ts +33 -9
- package/src/workspace/migrations/140-repair-seed-pinned-memory-v3-live.ts +260 -0
- package/src/workspace/migrations/141-stt-english-default-to-multilingual.ts +111 -0
- package/src/workspace/migrations/142-consolidate-voice-front-door.ts +70 -0
- package/src/workspace/migrations/__tests__/141-stt-english-default-to-multilingual.test.ts +116 -0
- package/src/workspace/migrations/registry.ts +6 -0
- package/src/workspace/top-level-renderer.ts +15 -2
- package/src/__tests__/tool-side-effects-slack-dm.test.ts +0 -357
- package/src/daemon/install-symlink.ts +0 -207
- package/src/live-voice/__tests__/front-decision.test.ts +0 -525
- package/src/live-voice/front-decision.ts +0 -429
- package/src/notifications/deferred-emit.ts +0 -147
|
@@ -38,6 +38,7 @@
|
|
|
38
38
|
* - Media stream `stop` event / WebSocket close -> finalize call.
|
|
39
39
|
*/
|
|
40
40
|
|
|
41
|
+
import { GATEWAY_TUNNEL_LOST_WS_CLOSE_CODE } from "@vellumai/service-contracts/ingress";
|
|
41
42
|
import type { ServerWebSocket } from "bun";
|
|
42
43
|
|
|
43
44
|
import { revokeScopedApprovalGrantsForContext } from "../approvals/scoped-approval-grants.js";
|
|
@@ -80,6 +81,7 @@ import {
|
|
|
80
81
|
type MediaStreamSttSessionCallbacks,
|
|
81
82
|
type MediaStreamSttSessionConfig,
|
|
82
83
|
} from "./media-stream-stt-session.js";
|
|
84
|
+
import { resolveTelephonySynthesisLanguage } from "./telephony-synthesis-language.js";
|
|
83
85
|
import {
|
|
84
86
|
TRUST_UNAVAILABLE_DENY_MESSAGE,
|
|
85
87
|
unresolvedActorTrust,
|
|
@@ -191,6 +193,14 @@ export class MediaStreamCallSession {
|
|
|
191
193
|
/** Number of transcript finals produced (non-empty). */
|
|
192
194
|
private transcriptFinalsProduced = 0;
|
|
193
195
|
|
|
196
|
+
/**
|
|
197
|
+
* Synthesized speech follows the caller's latest detected language when
|
|
198
|
+
* the transcriber tags finals, falling back to the pin-based
|
|
199
|
+
* resolution. Shared by the output adapter and the call controller.
|
|
200
|
+
*/
|
|
201
|
+
private readonly resolveSynthesisLanguage = (): string | undefined =>
|
|
202
|
+
resolveTelephonySynthesisLanguage(this.sttSession.currentLanguage());
|
|
203
|
+
|
|
194
204
|
constructor(
|
|
195
205
|
ws: ServerWebSocket<unknown>,
|
|
196
206
|
callSessionId: string,
|
|
@@ -215,6 +225,8 @@ export class MediaStreamCallSession {
|
|
|
215
225
|
|
|
216
226
|
this.sttSession = new MediaStreamSttSession(sttConfig ?? {}, callbacks);
|
|
217
227
|
|
|
228
|
+
this.output.setSynthesisLanguageResolver(this.resolveSynthesisLanguage);
|
|
229
|
+
|
|
218
230
|
log.info({ callSessionId }, "Media stream call session created");
|
|
219
231
|
}
|
|
220
232
|
|
|
@@ -344,9 +356,16 @@ export class MediaStreamCallSession {
|
|
|
344
356
|
);
|
|
345
357
|
}
|
|
346
358
|
} else {
|
|
359
|
+
// The gateway's close reason does not survive the relay (Bun's
|
|
360
|
+
// WebSocket client drops it), so tunnel loss is decoded from the
|
|
361
|
+
// dedicated close code.
|
|
347
362
|
const detail =
|
|
348
363
|
reason ||
|
|
349
|
-
(code
|
|
364
|
+
(code === GATEWAY_TUNNEL_LOST_WS_CLOSE_CODE
|
|
365
|
+
? "public ingress tunnel disconnected"
|
|
366
|
+
: code
|
|
367
|
+
? `media_stream_closed_${code}`
|
|
368
|
+
: "media_stream_closed_abnormal");
|
|
350
369
|
updateCallSession(this.callSessionId, {
|
|
351
370
|
status: "failed",
|
|
352
371
|
endedAt: Date.now(),
|
|
@@ -673,6 +692,7 @@ export class MediaStreamCallSession {
|
|
|
673
692
|
{
|
|
674
693
|
assistantId: result.assistantId,
|
|
675
694
|
trustContext: result.trustContext,
|
|
695
|
+
resolveSynthesisLanguage: this.resolveSynthesisLanguage,
|
|
676
696
|
},
|
|
677
697
|
);
|
|
678
698
|
this.controller = controller;
|
|
@@ -48,6 +48,7 @@ import type {
|
|
|
48
48
|
SttCallContextHints,
|
|
49
49
|
SttStreamServerEvent,
|
|
50
50
|
} from "../stt/types.js";
|
|
51
|
+
import { baseLanguageSubtag } from "../util/language-subtag.js";
|
|
51
52
|
import { getLogger } from "../util/logger.js";
|
|
52
53
|
import {
|
|
53
54
|
mulawToLinear,
|
|
@@ -220,6 +221,20 @@ export class MediaStreamSttSession {
|
|
|
220
221
|
/** Speech-bearing audio milliseconds since the last streaming final. */
|
|
221
222
|
private utteranceAudioMs = 0;
|
|
222
223
|
|
|
224
|
+
/**
|
|
225
|
+
* The dominant detected-language base subtag of the latest committed
|
|
226
|
+
* streaming final that carried language tags. Overwritten per
|
|
227
|
+
* utterance, matching live voice's per-utterance resolution, so a
|
|
228
|
+
* caller who switches languages retargets synthesis on their next
|
|
229
|
+
* utterance instead of having to outvote the session's history. Only
|
|
230
|
+
* finals that carry transcript text update it: silence finals can
|
|
231
|
+
* carry container-level tags describing no emitted words. Untagged
|
|
232
|
+
* finals keep the previous value; batch transcription reports no
|
|
233
|
+
* language tags, so it stays unset there and any streaming-detected
|
|
234
|
+
* value is cleared when the session settles on batch mode.
|
|
235
|
+
*/
|
|
236
|
+
private latestUtteranceLanguage: string | undefined;
|
|
237
|
+
|
|
223
238
|
constructor(
|
|
224
239
|
config: MediaStreamSttSessionConfig = {},
|
|
225
240
|
callbacks: MediaStreamSttSessionCallbacks = {},
|
|
@@ -304,6 +319,15 @@ export class MediaStreamSttSession {
|
|
|
304
319
|
return this.startupFramesDroppedCount;
|
|
305
320
|
}
|
|
306
321
|
|
|
322
|
+
/**
|
|
323
|
+
* The caller's detected language as of the latest tagged streaming
|
|
324
|
+
* final, as a lowercase base subtag. Undefined until a tagged final
|
|
325
|
+
* commits (batch mode, non-tagging providers, silence).
|
|
326
|
+
*/
|
|
327
|
+
currentLanguage(): string | undefined {
|
|
328
|
+
return this.latestUtteranceLanguage;
|
|
329
|
+
}
|
|
330
|
+
|
|
307
331
|
// ── Event handlers ─────────────────────────────────────────────────
|
|
308
332
|
|
|
309
333
|
private handleStart(event: MediaStreamStartEvent): void {
|
|
@@ -477,6 +501,10 @@ export class MediaStreamSttSession {
|
|
|
477
501
|
*/
|
|
478
502
|
private enterBatchMode(): void {
|
|
479
503
|
this.mode = "batch";
|
|
504
|
+
// Batch transcripts carry no language metadata, so a language detected
|
|
505
|
+
// while streaming would otherwise hint synthesis for the rest of the
|
|
506
|
+
// call. Clear it so the configured pin (or no hint) takes over.
|
|
507
|
+
this.latestUtteranceLanguage = undefined;
|
|
480
508
|
this.startupFrames = [];
|
|
481
509
|
this.capabilityPromise ??= resolveTelephonySttCapability();
|
|
482
510
|
|
|
@@ -510,11 +538,28 @@ export class MediaStreamSttSession {
|
|
|
510
538
|
switch (event.type) {
|
|
511
539
|
case "partial":
|
|
512
540
|
return;
|
|
541
|
+
case "finalized":
|
|
542
|
+
return;
|
|
543
|
+
case "turn-start":
|
|
544
|
+
case "eager-turn-end":
|
|
545
|
+
case "turn-resumed":
|
|
546
|
+
case "turn-end":
|
|
547
|
+
// The local VAD and the provider's boundary finals own turn
|
|
548
|
+
// taking. Listed case-by-case rather than under `default` so the
|
|
549
|
+
// exhaustiveness check flags this site when turn detection is
|
|
550
|
+
// wired in.
|
|
551
|
+
return;
|
|
513
552
|
case "final": {
|
|
514
553
|
const durationMs = Math.round(this.utteranceAudioMs);
|
|
515
554
|
this.utteranceAudioMs = 0;
|
|
516
555
|
const text = event.text.trim();
|
|
517
556
|
if (text.length > 0) {
|
|
557
|
+
// `languages` is dominance-ranked, so the first entry is the
|
|
558
|
+
// utterance's dominant tag.
|
|
559
|
+
const dominant = baseLanguageSubtag(event.languages?.[0]);
|
|
560
|
+
if (dominant !== undefined) {
|
|
561
|
+
this.latestUtteranceLanguage = dominant;
|
|
562
|
+
}
|
|
518
563
|
this.callbacks.onTranscriptFinal?.(text, durationMs);
|
|
519
564
|
}
|
|
520
565
|
return;
|
|
@@ -536,6 +581,10 @@ export class MediaStreamSttSession {
|
|
|
536
581
|
this.enterBatchMode();
|
|
537
582
|
}
|
|
538
583
|
return;
|
|
584
|
+
default: {
|
|
585
|
+
const _exhaustive: never = event;
|
|
586
|
+
return;
|
|
587
|
+
}
|
|
539
588
|
}
|
|
540
589
|
}
|
|
541
590
|
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Synthesis-language resolution for telephony TTS.
|
|
3
|
+
*
|
|
4
|
+
* The telephony call-control prompt instructs the model to speak the
|
|
5
|
+
* caller's language; this resolver produces the matching TTS hint so
|
|
6
|
+
* providers that can enforce a language render the reply in it (the hint
|
|
7
|
+
* is a no-op for providers that cannot). Resolution mirrors live voice's
|
|
8
|
+
* per-utterance turn language (live-voice-session.ts): the caller's
|
|
9
|
+
* latest STT-detected language when the transcriber tags finals, else a
|
|
10
|
+
* monolingual `services.stt.language` pin when the configured provider
|
|
11
|
+
* honors manual language selection, else undefined (no hint, provider
|
|
12
|
+
* default behavior).
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import { getConfig } from "../config/loader.js";
|
|
16
|
+
import type { AssistantConfig } from "../config/types.js";
|
|
17
|
+
import { pinnedListeningLanguage } from "../providers/speech-to-text/provider-catalog.js";
|
|
18
|
+
import { resolveLanguageVoiceOverride } from "../tts/language-voices.js";
|
|
19
|
+
import { resolveTtsConfig } from "../tts/tts-config-resolver.js";
|
|
20
|
+
import type { TtsProviderId } from "../tts/types.js";
|
|
21
|
+
import { baseLanguageSubtag } from "../util/language-subtag.js";
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Resolve the language hint for telephony synthesis as a lowercase base
|
|
25
|
+
* subtag, or undefined when no signal resolves.
|
|
26
|
+
*
|
|
27
|
+
* @param detectedLanguage - The caller language the STT session detected
|
|
28
|
+
* on its latest tagged utterance, when a session is reachable from the
|
|
29
|
+
* call site. Wins over the pin whenever it normalizes to a non-empty
|
|
30
|
+
* tag.
|
|
31
|
+
*/
|
|
32
|
+
export function resolveTelephonySynthesisLanguage(
|
|
33
|
+
detectedLanguage?: string,
|
|
34
|
+
): string | undefined {
|
|
35
|
+
const detected = baseLanguageSubtag(detectedLanguage);
|
|
36
|
+
if (detected !== undefined) {
|
|
37
|
+
return detected;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
let stt: { provider: string; language?: string };
|
|
41
|
+
try {
|
|
42
|
+
stt = getConfig().services.stt;
|
|
43
|
+
} catch {
|
|
44
|
+
// Config unavailable (early startup, minimal test workspaces): no hint.
|
|
45
|
+
return undefined;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
return pinnedListeningLanguage(stt.provider, stt.language);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Resolve the per-language voice for a telephony synthesis request from
|
|
53
|
+
* the synthesizing provider's `services.tts.providers.<id>.languageVoices`
|
|
54
|
+
* map. Keyed by the provider actually synthesizing, which on the
|
|
55
|
+
* media-stream transport may be a playability fallback rather than the
|
|
56
|
+
* configured provider, because voice identifiers are provider-specific.
|
|
57
|
+
* Returns undefined (provider default voice) when the request carries no
|
|
58
|
+
* language, config is unavailable, or the provider's map has no entry.
|
|
59
|
+
*/
|
|
60
|
+
export function resolveTelephonyLanguageVoice(
|
|
61
|
+
providerId: string,
|
|
62
|
+
language: string | undefined,
|
|
63
|
+
): string | undefined {
|
|
64
|
+
if (language === undefined) {
|
|
65
|
+
return undefined;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
let config: AssistantConfig;
|
|
69
|
+
try {
|
|
70
|
+
config = getConfig();
|
|
71
|
+
} catch {
|
|
72
|
+
// Config unavailable (early startup, minimal test workspaces): no override.
|
|
73
|
+
return undefined;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
const { providerConfig } = resolveTtsConfig(
|
|
77
|
+
config,
|
|
78
|
+
providerId as TtsProviderId,
|
|
79
|
+
);
|
|
80
|
+
return resolveLanguageVoiceOverride(
|
|
81
|
+
providerConfig.languageVoices as Record<string, string> | undefined,
|
|
82
|
+
language,
|
|
83
|
+
);
|
|
84
|
+
}
|
|
@@ -49,11 +49,18 @@ export function sanitizeForTts(text: string): string {
|
|
|
49
49
|
result = result.replace(/^[-*]\s+/gm, "");
|
|
50
50
|
|
|
51
51
|
// 7. Italic: *text* or _text_ → text
|
|
52
|
-
// Word-boundary-aware
|
|
53
|
-
//
|
|
54
|
-
//
|
|
55
|
-
|
|
56
|
-
result = result.replace(
|
|
52
|
+
// Word-boundary-aware (Unicode letters/digits, so café and 変数 count as
|
|
53
|
+
// word chars) with non-whitespace content edges, so arithmetic like
|
|
54
|
+
// `5 * 3 and 4 * 2` and identifiers like `my_var` survive. Mirrors the
|
|
55
|
+
// open-span rule in tts/speakable-segments.ts.
|
|
56
|
+
result = result.replace(
|
|
57
|
+
/(?<![\p{L}\p{N}_])\*([^\s*](?:[^*]*[^\s*])?)\*(?![\p{L}\p{N}_])/gu,
|
|
58
|
+
"$1",
|
|
59
|
+
);
|
|
60
|
+
result = result.replace(
|
|
61
|
+
/(?<![\p{L}\p{N}_])_([^\s_](?:[^_]*[^\s_])?)_(?![\p{L}\p{N}_])/gu,
|
|
62
|
+
"$1",
|
|
63
|
+
);
|
|
57
64
|
|
|
58
65
|
// 8. Emojis: strip extended pictographic characters, variation selectors,
|
|
59
66
|
// zero-width joiners, skin tone modifiers, and regional indicator symbols (flags).
|
|
@@ -17,6 +17,7 @@ import type {
|
|
|
17
17
|
import { consumeGrantForInvocation } from "../approvals/approval-primitive.js";
|
|
18
18
|
import type {
|
|
19
19
|
ChannelId,
|
|
20
|
+
ClientOs,
|
|
20
21
|
InterfaceId,
|
|
21
22
|
TurnChannelContext,
|
|
22
23
|
TurnInterfaceContext,
|
|
@@ -33,6 +34,7 @@ import {
|
|
|
33
34
|
recordConversationPersistedSeq,
|
|
34
35
|
updateMessageContent,
|
|
35
36
|
} from "../persistence/conversation-crud.js";
|
|
37
|
+
import { pinnedListeningLanguage } from "../providers/speech-to-text/provider-catalog.js";
|
|
36
38
|
import type { ContentBlock } from "../providers/types.js";
|
|
37
39
|
import { broadcastMessage } from "../runtime/assistant-event-hub.js";
|
|
38
40
|
import { DAEMON_INTERNAL_ASSISTANT_ID } from "../runtime/assistant-scope.js";
|
|
@@ -265,6 +267,22 @@ export interface VoiceTurnOptions {
|
|
|
265
267
|
userMessageInterface?: InterfaceId;
|
|
266
268
|
/** Source interface for persisted assistant messages. Defaults to userMessageInterface. */
|
|
267
269
|
assistantMessageInterface?: InterfaceId;
|
|
270
|
+
/**
|
|
271
|
+
* Analytics attribution for a live-voice turn: which client opened the
|
|
272
|
+
* session, and that session's id. Persisted into the user message's
|
|
273
|
+
* `metadata.client` bag, which `turn-events-store` projects onto
|
|
274
|
+
* `TurnTelemetryEvent.client`, so a live-voice turn is countable per client
|
|
275
|
+
* and joinable to its session's start/end funnel rows.
|
|
276
|
+
*
|
|
277
|
+
* Deliberately separate from {@link userMessageInterface}: that field feeds
|
|
278
|
+
* `resolveChannelCapabilities` and decides what the turn may do, so it is not
|
|
279
|
+
* free to carry attribution. Absent for phone calls and for clients that send
|
|
280
|
+
* no identity on the start frame.
|
|
281
|
+
*/
|
|
282
|
+
voiceTelemetry?: {
|
|
283
|
+
sessionId: string;
|
|
284
|
+
client?: ClientOs;
|
|
285
|
+
};
|
|
268
286
|
/** Per-turn control prompt. Undefined uses the phone prompt; null disables it. */
|
|
269
287
|
voiceControlPrompt?: string | null;
|
|
270
288
|
/** The transcribed caller utterance or synthetic marker. */
|
|
@@ -389,18 +407,6 @@ export interface VoiceTurnHandle {
|
|
|
389
407
|
* on a live phone call" framing (the session system prompt already
|
|
390
408
|
* provides assistant identity) and guardian context (injected separately).
|
|
391
409
|
*/
|
|
392
|
-
/**
|
|
393
|
-
* Steering shared by every voice channel. A sign-in flow opens a browser
|
|
394
|
-
* window mid-call that the caller may be unable to see or complete, whether it
|
|
395
|
-
* is reached through a ui-surface tool or through shell and CLI tools (e.g.
|
|
396
|
-
* `assistant oauth connect`). Tell the model to speak the limitation and defer
|
|
397
|
-
* the flow to text chat instead.
|
|
398
|
-
*
|
|
399
|
-
* This outlives the ui-surface restriction it was written alongside: a
|
|
400
|
-
* live-voice call can now show surfaces, but a browser window handing control
|
|
401
|
-
* to a third party mid-call is a different problem, and one a minimized room
|
|
402
|
-
* does not solve.
|
|
403
|
-
*/
|
|
404
410
|
/**
|
|
405
411
|
* How long a live-voice call waits on an approval before deciding it itself.
|
|
406
412
|
*
|
|
@@ -411,8 +417,49 @@ export interface VoiceTurnHandle {
|
|
|
411
417
|
*/
|
|
412
418
|
const VOICE_APPROVAL_TIMEOUT_MS = 45_000;
|
|
413
419
|
|
|
414
|
-
|
|
415
|
-
|
|
420
|
+
/**
|
|
421
|
+
* Telephony-only steering. A sign-in flow opens a browser window on a screen
|
|
422
|
+
* the caller does not have in front of them, whether it is reached through a
|
|
423
|
+
* ui-surface tool or through shell and CLI tools (e.g. `assistant oauth
|
|
424
|
+
* connect`). Tell the model to speak the limitation and defer the flow to text
|
|
425
|
+
* chat instead.
|
|
426
|
+
*
|
|
427
|
+
* Scoped to the phone because the screen is what decides it. A phone call has
|
|
428
|
+
* no screen, so the `open_url` signal a CLI tool can reach lands somewhere the
|
|
429
|
+
* caller will never see, and that signal bus carries no capability or
|
|
430
|
+
* conversation context, so this rule is the only thing standing in front of it
|
|
431
|
+
* here. A live-voice call is the opposite case: the user is holding the screen,
|
|
432
|
+
* and the room minimizes itself to hand it back (see
|
|
433
|
+
* LIVE_VOICE_SETUP_FLOW_TEACHING).
|
|
434
|
+
*/
|
|
435
|
+
const PHONE_NO_SETUP_FLOWS_RULE =
|
|
436
|
+
"Never start account connections, OAuth or sign-in flows, or any other action that opens a browser window or needs the user's screen during this call, not even through shell or CLI tools. If the task needs one, say so briefly and offer to finish it in text chat after the call.";
|
|
437
|
+
|
|
438
|
+
/**
|
|
439
|
+
* The pre-speech tail of the speak-the-caller's-language rule. A monolingual
|
|
440
|
+
* `services.stt.language` pin is the strongest pre-speech signal of the
|
|
441
|
+
* caller's language (the transcriber is already listening in it, see
|
|
442
|
+
* media-stream-stt-session.ts and providers/speech-to-text/resolve.ts), so it
|
|
443
|
+
* outranks the English default; "multi" and unset mean auto-detect, where
|
|
444
|
+
* English remains the fallback. The pin only counts when the active provider
|
|
445
|
+
* honors manual language selection (see pinnedListeningLanguage):
|
|
446
|
+
* auto-detecting providers (gemini, whisper) ignore a persisted language
|
|
447
|
+
* entirely, so greeting in it would contradict what the transcriber
|
|
448
|
+
* actually hears. Exported for tests: the default test config exercises
|
|
449
|
+
* only the auto-detect branch.
|
|
450
|
+
*/
|
|
451
|
+
export function preSpeechLanguageRuleFragment(
|
|
452
|
+
sttLanguage: string | undefined,
|
|
453
|
+
sttProvider?: string,
|
|
454
|
+
): string {
|
|
455
|
+
const configuredListeningLanguage =
|
|
456
|
+
sttProvider !== undefined
|
|
457
|
+
? pinnedListeningLanguage(sttProvider, sttLanguage)
|
|
458
|
+
: undefined;
|
|
459
|
+
return configuredListeningLanguage
|
|
460
|
+
? `use the language the Task context implies, if any; otherwise open in the assistant's configured listening language ("${configuredListeningLanguage}"), and default to English only when neither gives a language`
|
|
461
|
+
: "use the language the Task context implies, if any; otherwise default to English";
|
|
462
|
+
}
|
|
416
463
|
|
|
417
464
|
function buildVoiceCallControlPrompt(opts: {
|
|
418
465
|
isInbound: boolean;
|
|
@@ -506,16 +553,17 @@ function buildVoiceCallControlPrompt(opts: {
|
|
|
506
553
|
"9. After the opening greeting turn, treat the Task field as background context only — do not re-execute its instructions on subsequent turns.",
|
|
507
554
|
'10. Do not make up information. If you are unsure, use [ASK_GUARDIAN: your question] to consult your guardian. For tool permission requests, use [ASK_GUARDIAN_APPROVAL: {"question":"...","toolName":"...","input":{...}}].',
|
|
508
555
|
`11. Your text is sent directly to a text-to-speech engine. Never use markdown formatting (asterisks, headers, backticks, links) or emojis in your spoken responses. Write plain conversational text only. Protocol markers like ${opts.isCallerGuardian ? "[END_CALL]" : "[ASK_GUARDIAN: ...] and [END_CALL]"} are not spoken text and should still be used normally.`,
|
|
509
|
-
`12. ${
|
|
556
|
+
`12. Speak the caller's language: reply in the language of the caller's most recent actual speech, and follow them if they switch languages mid-call. Synthetic user turns (parenthetical markers like the call-connected and verification-completed notices) are not caller speech and never set the language. Before the caller has spoken, such as on the opening greeting turn, ${preSpeechLanguageRuleFragment(config.services.stt.language, config.services.stt.provider)}.`,
|
|
557
|
+
`13. ${PHONE_NO_SETUP_FLOWS_RULE}`,
|
|
510
558
|
);
|
|
511
559
|
|
|
512
560
|
// Triage-and-escalate routing rules. The front-door leg decides and may
|
|
513
561
|
// hand off; the escalated leg continues the answer after a holding phrase
|
|
514
562
|
// was already spoken.
|
|
515
563
|
if (opts.routingLeg === "front-door") {
|
|
516
|
-
lines.push(`
|
|
564
|
+
lines.push(`14. ${frontDoorRuleWithDigest(opts.unifiedVerdict === true)}`);
|
|
517
565
|
} else if (opts.routingLeg === "escalated") {
|
|
518
|
-
lines.push(`
|
|
566
|
+
lines.push(`14. ${escalatedContinuationRule(opts.spokenEscalationBridge)}`);
|
|
519
567
|
}
|
|
520
568
|
|
|
521
569
|
lines.push("</voice_call_control>");
|
|
@@ -945,6 +993,27 @@ export async function startVoiceTurn(
|
|
|
945
993
|
// `isVoiceSessionUserMessage` for why the channel fields cannot carry it.
|
|
946
994
|
voiceSessionTurn: true,
|
|
947
995
|
...(isHiddenSyntheticPrompt ? { hidden: true } : {}),
|
|
996
|
+
...(opts.voiceTelemetry
|
|
997
|
+
? {
|
|
998
|
+
// Projected onto `TurnTelemetryEvent.client` by
|
|
999
|
+
// `turn-events-store`. `voice_session_id` is what joins these
|
|
1000
|
+
// turn rows to the session's funnel rows, so a session's turn
|
|
1001
|
+
// count is a count of rows carrying its id, never a field
|
|
1002
|
+
// anyone has to keep correct.
|
|
1003
|
+
//
|
|
1004
|
+
// `os` is the standard per-platform dimension the HTTP send
|
|
1005
|
+
// path already fills from the same `detectClientOs()` value, so
|
|
1006
|
+
// a voice turn reports its platform in the column existing turn
|
|
1007
|
+
// analytics read rather than one only voice knows about.
|
|
1008
|
+
client: {
|
|
1009
|
+
voice: true,
|
|
1010
|
+
voice_session_id: opts.voiceTelemetry.sessionId,
|
|
1011
|
+
...(opts.voiceTelemetry.client
|
|
1012
|
+
? { os: opts.voiceTelemetry.client }
|
|
1013
|
+
: {}),
|
|
1014
|
+
},
|
|
1015
|
+
}
|
|
1016
|
+
: {}),
|
|
948
1017
|
},
|
|
949
1018
|
});
|
|
950
1019
|
return persistResult.id;
|
|
@@ -26,6 +26,8 @@
|
|
|
26
26
|
* cap/fallback policy. LiveVoiceSession drives the routing.
|
|
27
27
|
*/
|
|
28
28
|
|
|
29
|
+
import { NON_LATIN_SENTENCE_ENDING_PUNCTUATION } from "../tts/speakable-segments.js";
|
|
30
|
+
import { localizedOrDefault } from "../util/language-subtag.js";
|
|
29
31
|
import {
|
|
30
32
|
ESCALATE_VERDICT_TOKEN,
|
|
31
33
|
HOLD_VERDICT_TOKEN,
|
|
@@ -57,6 +59,41 @@ export type VoiceRoutingLeg = "front-door" | "escalated";
|
|
|
57
59
|
export const FALLBACK_ESCALATION_BRIDGE =
|
|
58
60
|
"Let me think about that for a second.";
|
|
59
61
|
|
|
62
|
+
/**
|
|
63
|
+
* Per-language spellings of the fallback escalation bridge, keyed by
|
|
64
|
+
* lowercased BCP 47 base subtag, covering the Deepgram code-switching roster
|
|
65
|
+
* (DEEPGRAM_MULTI_LANGUAGE_CODES in providers/speech-to-text/deepgram.ts).
|
|
66
|
+
* These are spoken audio; the English constant above also serves as the
|
|
67
|
+
* prompt exemplar and stays the default.
|
|
68
|
+
*/
|
|
69
|
+
export const FALLBACK_ESCALATION_BRIDGE_BY_LANGUAGE: Readonly<
|
|
70
|
+
Record<string, string>
|
|
71
|
+
> = {
|
|
72
|
+
en: FALLBACK_ESCALATION_BRIDGE,
|
|
73
|
+
es: "Déjame pensarlo un segundo.",
|
|
74
|
+
fr: "Laissez-moi y réfléchir un instant.",
|
|
75
|
+
de: "Lass mich kurz darüber nachdenken.",
|
|
76
|
+
hi: "मुझे एक पल सोचने दीजिए।",
|
|
77
|
+
ru: "Дайте мне секунду подумать.",
|
|
78
|
+
pt: "Deixe-me pensar nisso um segundo.",
|
|
79
|
+
ja: "少し考えさせてください。",
|
|
80
|
+
it: "Fammi pensare un attimo.",
|
|
81
|
+
nl: "Laat me daar even over nadenken.",
|
|
82
|
+
};
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* The fallback escalation bridge in the caller's language: selected by the
|
|
86
|
+
* lowercased base subtag of `language` (e.g. "pt-BR" -> "pt"), defaulting to
|
|
87
|
+
* {@link FALLBACK_ESCALATION_BRIDGE} for unknown or absent languages.
|
|
88
|
+
*/
|
|
89
|
+
export function fallbackEscalationBridgeFor(language?: string): string {
|
|
90
|
+
return localizedOrDefault(
|
|
91
|
+
FALLBACK_ESCALATION_BRIDGE_BY_LANGUAGE,
|
|
92
|
+
language,
|
|
93
|
+
FALLBACK_ESCALATION_BRIDGE,
|
|
94
|
+
);
|
|
95
|
+
}
|
|
96
|
+
|
|
60
97
|
/**
|
|
61
98
|
* Minimum length (after capping, trimmed) of the front-door leg's spoken
|
|
62
99
|
* bridge for it to count as a real bridge. Below this, the fallback bridge
|
|
@@ -71,8 +108,17 @@ export const MIN_SPOKEN_BRIDGE_CHARS = 3;
|
|
|
71
108
|
*/
|
|
72
109
|
export const MAX_ESCALATION_BRIDGE_CHARS = 140;
|
|
73
110
|
|
|
74
|
-
/**
|
|
75
|
-
|
|
111
|
+
/**
|
|
112
|
+
* Sentence terminators that end an escalation bridge: the segmenter's
|
|
113
|
+
* non-Latin ender roster (tts/speakable-segments.ts) plus the ASCII enders
|
|
114
|
+
* and the ellipsis. Built from the shared set so the rosters cannot
|
|
115
|
+
* diverge: without the non-Latin enders a Japanese or Hindi bridge never
|
|
116
|
+
* hits a terminator and buffers to the char cap before hand-off. Exported
|
|
117
|
+
* for tests that assert spoken phrases end in a recognized terminator.
|
|
118
|
+
*/
|
|
119
|
+
export const BRIDGE_SENTENCE_END_REGEX = new RegExp(
|
|
120
|
+
`[.!?…${[...NON_LATIN_SENTENCE_ENDING_PUNCTUATION].join("")}]`,
|
|
121
|
+
);
|
|
76
122
|
|
|
77
123
|
/**
|
|
78
124
|
* Normalize a raw post-`[1]` stream into the bridge that is actually
|
|
@@ -227,10 +273,10 @@ export function frontDoorDecisionRule(opts?: {
|
|
|
227
273
|
// Hold requires positive evidence of an unfinished sentence, never
|
|
228
274
|
// mere uncertainty, because the two mistakes cost differently. A
|
|
229
275
|
// false hold is silent: the verdict, the extension window, and the
|
|
230
|
-
// replay dispatch all elapse
|
|
231
|
-
//
|
|
276
|
+
// replay dispatch all elapse before the turn commits, roughly
|
|
277
|
+
// tripling felt latency. A false
|
|
232
278
|
// release only answers a beat early, which barge-in absorbs.
|
|
233
|
-
`- If the caller's words are visibly unfinished
|
|
279
|
+
`- If the caller's words are visibly unfinished (a trailing conjunction, a dangling clause, a list still being dictated) output ONLY ${HOLD_VERDICT_TOKEN} and stop, no other text. Judge the words themselves: a complete question or statement means they are done, even when it is short or leans on earlier context ("What do you think?", "Why?", "And then?"). Callers may speak any language: those examples are English exemplars only, and completeness is judged by the grammar of the language being spoken. In verb-final languages such as Hindi, Japanese, or Korean the sentence-final verb usually marks completion, so a missing final verb is the unfinished signal, not a missing conjunction. Never hold merely because more could follow.`,
|
|
234
280
|
]
|
|
235
281
|
: [
|
|
236
282
|
// No hold branch means completeness is settled (a first leg
|
|
@@ -264,8 +310,8 @@ export function frontDoorDecisionRule(opts?: {
|
|
|
264
310
|
...anchor,
|
|
265
311
|
"DECIDE SILENTLY, then produce exactly ONE of these outputs:",
|
|
266
312
|
...holdBranch,
|
|
267
|
-
"- If the turn is simple, conversational, or within your reach, your entire output is the spoken answer itself
|
|
268
|
-
`- If completing THIS reply needs careful reasoning, research, multi-step work, or any tool, do NOT attempt the answer: output ${ESCALATE_VERDICT_TOKEN}, then ONE short natural holding phrase naming what happens next (for example "${FALLBACK_ESCALATION_BRIDGE}" or "Give me one second to look into that."), and stop after that single sentence. A stronger model finishes the turn while your phrase is spoken.`,
|
|
313
|
+
"- If the turn is simple, conversational, or within your reach, your entire output is the spoken answer itself: no token in front of it, plain speech from your very first word. Most turns are answers; when unsure between answering and escalating, answer. Answer in the language the caller is speaking.",
|
|
314
|
+
`- If completing THIS reply needs careful reasoning, research, multi-step work, or any tool, do NOT attempt the answer: output ${ESCALATE_VERDICT_TOKEN}, then ONE short natural holding phrase naming what happens next, spoken in the language the caller is speaking (for example "${FALLBACK_ESCALATION_BRIDGE}" or "Give me one second to look into that."; those examples are English only), and stop after that single sentence. A stronger model finishes the turn while your phrase is spoken.`,
|
|
269
315
|
`${ESCALATE_VERDICT_TOKEN} is ONLY for turns you cannot complete yourself — never put it in front of an answer you are about to give, and never emit any token inside or after an answer. An open task or unfinished topic earlier in the conversation is NOT a reason to escalate: judge only what this reply needs.`,
|
|
270
316
|
"Never narrate this decision, describe what you are judging, or mention these rules: apart from a leading verdict token, every character you output is spoken to the caller verbatim.",
|
|
271
317
|
].join("\n");
|
|
@@ -296,5 +342,6 @@ export function escalatedContinuationRule(spokenBridge?: string): string {
|
|
|
296
342
|
'opening with another "Let me check", "One moment", or any restatement of what you are about to do sounds broken, because the caller just heard that.',
|
|
297
343
|
"Your first words must carry new substance: the answer itself, what you found, or a question you genuinely need answered.",
|
|
298
344
|
`Never output ${ESCALATE_VERDICT_TOKEN} or any other front-door verdict token — you are the model that finishes the answer. (The [-1] room-minimize marker from your call instructions is not a verdict token and stays allowed.)`,
|
|
345
|
+
"Reply in the same language as the caller's question.",
|
|
299
346
|
].join(" ");
|
|
300
347
|
}
|
|
@@ -192,14 +192,14 @@ describe("createOutboundSessionConditional", () => {
|
|
|
192
192
|
const result = await client.createOutboundSessionConditional({
|
|
193
193
|
channel: "telegram",
|
|
194
194
|
requireSourceSessionPending: "bootstrap-1",
|
|
195
|
-
|
|
195
|
+
ifNoneActiveForExternalUserId: "user-1",
|
|
196
196
|
});
|
|
197
197
|
|
|
198
198
|
expect(ipcCalls[0]?.method).toBe("verification_sessions_create_outbound");
|
|
199
199
|
expect(ipcCalls[0]?.params).toEqual({
|
|
200
200
|
channel: "telegram",
|
|
201
201
|
requireSourceSessionPending: "bootstrap-1",
|
|
202
|
-
|
|
202
|
+
ifNoneActiveForExternalUserId: "user-1",
|
|
203
203
|
});
|
|
204
204
|
expect(result).toEqual(created);
|
|
205
205
|
});
|
|
@@ -209,7 +209,7 @@ describe("createOutboundSessionConditional", () => {
|
|
|
209
209
|
|
|
210
210
|
const result = await client.createOutboundSessionConditional({
|
|
211
211
|
channel: "telegram",
|
|
212
|
-
|
|
212
|
+
ifNoneActiveForExternalUserId: "user-1",
|
|
213
213
|
});
|
|
214
214
|
|
|
215
215
|
expect(result).toEqual({
|