@vellumai/assistant 0.11.2 → 0.11.3-dev.202608080427.92f668a
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +69 -9
- package/docs/flux-turn-detection-spike.md +243 -0
- package/docs/stt-provider-onboarding.md +3 -1
- package/docs/trusted-contact-access.md +9 -5
- package/docs/vellum-doctor.md +195 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/channels.ts +50 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/ingress.ts +10 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/remote-web-pairing.ts +6 -0
- package/node_modules/@vellumai/ces-client/src/index.ts +1 -0
- package/node_modules/@vellumai/ces-client/src/rpc-client.ts +19 -2
- package/node_modules/@vellumai/environments/package.json +2 -1
- package/node_modules/@vellumai/environments/src/__tests__/cloud-assistant-hub-url.test.ts +38 -0
- package/node_modules/@vellumai/environments/src/__tests__/install-layout.test.ts +98 -0
- package/node_modules/@vellumai/environments/src/__tests__/package-boundary.test.ts +5 -5
- package/node_modules/@vellumai/environments/src/index.ts +17 -6
- package/node_modules/@vellumai/environments/src/install-layout.ts +49 -0
- package/node_modules/@vellumai/environments/src/seeds.ts +29 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/channels.ts +50 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/ingress.ts +10 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/remote-web-pairing.ts +6 -0
- package/node_modules/@vellumai/gateway-client/src/index.ts +4 -0
- package/node_modules/@vellumai/gateway-client/src/verification-session-contract.ts +91 -6
- package/node_modules/@vellumai/ipc-server-utils/src/endpoint.test.ts +36 -0
- package/node_modules/@vellumai/ipc-server-utils/src/endpoint.ts +142 -0
- package/node_modules/@vellumai/ipc-server-utils/src/index.ts +2 -0
- package/node_modules/@vellumai/ipc-server-utils/src/listen-options.ts +3 -0
- package/node_modules/@vellumai/ipc-server-utils/src/socket-watchdog.test.ts +15 -1
- package/node_modules/@vellumai/ipc-server-utils/src/socket-watchdog.ts +17 -2
- package/node_modules/@vellumai/service-contracts/src/channels.ts +50 -0
- package/node_modules/@vellumai/service-contracts/src/ingress.ts +10 -0
- package/node_modules/@vellumai/service-contracts/src/remote-web-pairing.ts +6 -0
- package/node_modules/@vellumai/slack-text/src/index.ts +16 -2
- package/openapi.yaml +619 -47
- package/package.json +1 -1
- package/scripts/voice-ttft-spike.ts +3 -3
- package/src/__tests__/agent-image-optimize.test.ts +4 -4
- package/src/__tests__/agent-loop-mutable-latest-user-message.test.ts +113 -66
- package/src/__tests__/agent-loop-override-profile.test.ts +38 -48
- package/src/__tests__/agent-loop-subagent-billing-headers.test.ts +166 -0
- package/src/__tests__/agent-wake-disk-pressure-callsite.test.ts +1 -0
- package/src/__tests__/agent-wake-override-profile.test.ts +1 -0
- package/src/__tests__/answered-question-persistence.test.ts +240 -0
- package/src/__tests__/anthropic-provider.test.ts +165 -30
- package/src/__tests__/app-control-flow.test.ts +9 -8
- package/src/__tests__/assistant-event-hub.test.ts +48 -0
- package/src/__tests__/attachments-store-heic-normalize.test.ts +16 -12
- package/src/__tests__/attachments-store.test.ts +130 -68
- package/src/__tests__/attachments.test.ts +36 -36
- package/src/__tests__/background-workers-disk-pressure.test.ts +1 -1
- package/src/__tests__/call-controller.test.ts +120 -0
- package/src/__tests__/call-start-guardian-guard.test.ts +18 -4
- package/src/__tests__/canned-reply-release.test.ts +86 -0
- package/src/__tests__/channel-availability-routes.test.ts +14 -1
- package/src/__tests__/channel-guardian.test.ts +59 -29
- package/src/__tests__/channel-readiness-telegram-remote.test.ts +154 -0
- package/src/__tests__/channel-setup-panel-ack.test.ts +6 -5
- package/src/__tests__/client-os-metadata-persistence.test.ts +103 -4
- package/src/__tests__/compactor-image-manifest-trust.test.ts +1 -1
- package/src/__tests__/config-loader-backfill.test.ts +19 -0
- package/src/__tests__/config-schema.test.ts +150 -7
- package/src/__tests__/consult-deadline.test.ts +65 -0
- package/src/__tests__/conversation-agent-loop.test.ts +438 -2
- package/src/__tests__/conversation-disk-view-integration.test.ts +5 -1
- package/src/__tests__/conversation-disk-view.test.ts +4 -4
- package/src/__tests__/conversation-error.test.ts +22 -3
- package/src/__tests__/conversation-fork-crud.test.ts +5 -1
- package/src/__tests__/conversation-fork-referential.test.ts +352 -0
- package/src/__tests__/conversation-fork-retrospective.test.ts +5 -1
- package/src/__tests__/conversation-group-tools.test.ts +32 -2
- package/src/__tests__/conversation-inference-profile-route.test.ts +27 -1
- package/src/__tests__/conversation-initial-prompt.test.ts +51 -22
- package/src/__tests__/conversation-key-store-bootstrap-cleanup.test.ts +119 -0
- package/src/__tests__/conversation-key-store-origin.test.ts +105 -0
- package/src/__tests__/conversation-list-recency-ordering.test.ts +104 -0
- package/src/__tests__/conversation-origin-at-creation.test.ts +153 -0
- package/src/__tests__/conversation-origin-channel-filter.test.ts +95 -0
- package/src/__tests__/conversation-placement-promotion.test.ts +172 -0
- package/src/__tests__/conversation-process-app-control-preactivation.test.ts +123 -0
- package/src/__tests__/conversation-queue.test.ts +433 -1
- package/src/__tests__/conversation-retry-route.test.ts +3 -1
- package/src/__tests__/conversation-routes-hidden-queue.test.ts +64 -1
- package/src/__tests__/conversation-store-ephemeral.test.ts +4 -0
- package/src/__tests__/conversation-store.test.ts +26 -10
- package/src/__tests__/conversation-summarize-route.test.ts +1 -1
- package/src/__tests__/conversation-surfaces-action-delivery.test.ts +15 -6
- package/src/__tests__/conversation-surfaces-activation-emit.test.ts +6 -5
- package/src/__tests__/conversation-surfaces-app-control.test.ts +12 -11
- package/src/__tests__/conversation-surfaces-app-open.test.ts +11 -5
- package/src/__tests__/conversation-surfaces-data-persist.test.ts +5 -4
- package/src/__tests__/conversation-surfaces-history-restored-completion.test.ts +967 -0
- package/src/__tests__/conversation-surfaces-persisted-info.test.ts +312 -0
- package/src/__tests__/conversation-surfaces-queued-emit.test.ts +6 -5
- package/src/__tests__/conversation-surfaces-standalone-payloads.test.ts +12 -8
- package/src/__tests__/conversation-surfaces-standalone.test.ts +21 -11
- package/src/__tests__/conversation-surfaces-state-update.test.ts +7 -11
- package/src/__tests__/conversation-surfaces-table-action.test.ts +7 -6
- package/src/__tests__/conversation-surfaces-task-progress.test.ts +11 -6
- package/src/__tests__/conversation-tool-setup-app-refresh.test.ts +9 -6
- package/src/__tests__/conversation-tool-setup-attribution.test.ts +70 -7
- package/src/__tests__/conversation-wait-for-idle.test.ts +6 -11
- package/src/__tests__/credential-prompt-route.test.ts +54 -7
- package/src/__tests__/credential-security-invariants.test.ts +1 -0
- package/src/__tests__/cu-unified-flow.test.ts +89 -28
- package/src/__tests__/custom-group-listing-visibility.test.ts +1 -1
- package/src/__tests__/db-schedule-syntax-migration.test.ts +4 -1
- package/src/__tests__/discord-access-request-privacy.test.ts +366 -0
- package/src/__tests__/discord-requester-notice-privacy.test.ts +87 -0
- package/src/__tests__/disk-pressure-guard.test.ts +55 -55
- package/src/__tests__/disk-pressure-routes.test.ts +10 -10
- package/src/__tests__/disk-pressure-tools.test.ts +2 -2
- package/src/__tests__/disk-usage.test.ts +35 -20
- package/src/__tests__/document-sync-tags.test.ts +379 -0
- package/src/__tests__/document-update-default-surface.test.ts +28 -0
- package/src/__tests__/dynamic-skill-background-guard.test.ts +2 -2
- package/src/__tests__/emit-signal-routing-intent.test.ts +3 -4
- package/src/__tests__/ensure-conversation-exists.test.ts +8 -5
- package/src/__tests__/events-dev-bypass-actor.test.ts +294 -1
- package/src/__tests__/evict-conversations-for-reload.test.ts +90 -0
- package/src/__tests__/file-list-tool.test.ts +12 -10
- package/src/__tests__/file-ops-service.test.ts +151 -44
- package/src/__tests__/file-read-container-boundary.test.ts +137 -0
- package/src/__tests__/file-read-tool.test.ts +21 -9
- package/src/__tests__/file-url-path-guard.test.ts +83 -0
- package/src/__tests__/filesystem-tools.test.ts +82 -66
- package/src/__tests__/guardian-outbound-http.test.ts +8 -8
- package/src/__tests__/helpers/gateway-verification-sessions-stub.ts +14 -1
- package/src/__tests__/helpers/hub-clients.ts +42 -0
- package/src/__tests__/helpers/mock-conversation.ts +46 -0
- package/src/__tests__/helpers/verification-sessions-ipc-sim.ts +61 -57
- package/src/__tests__/history-repair.test.ts +112 -12
- package/src/__tests__/host-bash-proxy.test.ts +10 -5
- package/src/__tests__/host-bash-routes.test.ts +67 -0
- package/src/__tests__/host-browser-proxy.test.ts +2 -2
- package/src/__tests__/host-cu-proxy.test.ts +3 -7
- package/src/__tests__/host-file-proxy.test.ts +3 -7
- package/src/__tests__/host-file-read-tool.test.ts +5 -1
- package/src/__tests__/host-transfer-proxy-targeted.test.ts +6 -4
- package/src/__tests__/http-conversation-lineage.test.ts +2 -5
- package/src/__tests__/http-user-message-parity.test.ts +44 -0
- package/src/__tests__/identity-routes.test.ts +13 -13
- package/src/__tests__/image-conversion.test.ts +29 -26
- package/src/__tests__/image-source-path-reinject.test.ts +2 -2
- package/src/__tests__/inference-profile-session-handler.test.ts +95 -0
- package/src/__tests__/inference-profile-session-ipc.test.ts +25 -0
- package/src/__tests__/introduction-card-resolver.test.ts +6 -1
- package/src/__tests__/list-messages-attachments.test.ts +56 -32
- package/src/__tests__/list-messages-background-tool-completion.test.ts +4 -4
- package/src/__tests__/list-messages-client-message-id.test.ts +2 -2
- package/src/__tests__/list-messages-hidden-metadata.test.ts +16 -16
- package/src/__tests__/list-messages-page-latest.test.ts +61 -61
- package/src/__tests__/list-messages-provider-error.test.ts +6 -6
- package/src/__tests__/list-messages-queued.test.ts +49 -6
- package/src/__tests__/list-messages-system-card.test.ts +4 -4
- package/src/__tests__/list-messages-tool-merge.test.ts +9 -9
- package/src/__tests__/llm-resolver-override-or-default.test.ts +141 -0
- package/src/__tests__/llm-usage-store.test.ts +55 -0
- package/src/__tests__/manual-token-reconciliation.test.ts +45 -0
- package/src/__tests__/mcp-list-plugin-servers.test.ts +250 -0
- package/src/__tests__/mcp-tool-annotations-risk.test.ts +133 -0
- package/src/__tests__/media-stream-output.test.ts +175 -0
- package/src/__tests__/media-stream-server-integration.test.ts +36 -0
- package/src/__tests__/media-stream-stt-session.test.ts +67 -0
- package/src/__tests__/migration-export-http.test.ts +1 -1
- package/src/__tests__/migration-import-commit-http.test.ts +1 -1
- package/src/__tests__/migration-import-preflight-http.test.ts +1 -1
- package/src/__tests__/migration-validate-http.test.ts +1 -1
- package/src/__tests__/mtime-cache.test.ts +65 -3
- package/src/__tests__/notification-platform-adapter.test.ts +104 -0
- package/src/__tests__/notification-schedule-notify-dedup.test.ts +1 -1
- package/src/__tests__/oauth-provider-profiles.test.ts +4 -2
- package/src/__tests__/oauth-provider-seed-logos.test.ts +65 -0
- package/src/__tests__/oauth-provider-serializer.test.ts +17 -0
- package/src/__tests__/oauth-providers-routes.test.ts +1 -0
- package/src/__tests__/openai-responses-prompt-cache.test.ts +5 -2
- package/src/__tests__/path-policy.test.ts +164 -1
- package/src/__tests__/persist-media-references.test.ts +2 -2
- package/src/__tests__/persist-unsendable-image-downscale.test.ts +5 -5
- package/src/__tests__/persist-unsendable-image.test.ts +23 -23
- package/src/__tests__/persona-resolver.test.ts +154 -0
- package/src/__tests__/plugin-api-webhook-url.test.ts +170 -0
- package/src/__tests__/plugin-import-boundary-guard.test.ts +6 -0
- package/src/__tests__/processing-flag-persist-failure.test.ts +15 -10
- package/src/__tests__/profile-availability-incomplete.test.ts +113 -0
- package/src/__tests__/profiler-routes.test.ts +13 -13
- package/src/__tests__/prompt-cache-cross-turn-stability.test.ts +593 -0
- package/src/__tests__/proxy-approval-callback.test.ts +3 -3
- package/src/__tests__/queued-message-cancel-and-steer.test.ts +189 -0
- package/src/__tests__/queued-message-steer-actor-scoping.test.ts +164 -0
- package/src/__tests__/resolve-trust-class.test.ts +20 -0
- package/src/__tests__/run-conversation-turn-persistence.test.ts +1 -1
- package/src/__tests__/run-due-schedules.test.ts +174 -0
- package/src/__tests__/runtime-attachment-metadata.test.ts +15 -7
- package/src/__tests__/scaffold-managed-skill-tool.test.ts +166 -0
- package/src/__tests__/schedule-routes.test.ts +730 -8
- package/src/__tests__/schedule-store.test.ts +540 -0
- package/src/__tests__/schedule-tools.test.ts +103 -0
- package/src/__tests__/scheduler-wake.test.ts +5 -0
- package/src/__tests__/secret-routes-platform-proxy.test.ts +29 -1
- package/src/__tests__/secure-keys.test.ts +91 -0
- package/src/__tests__/server-history-render.test.ts +97 -7
- package/src/__tests__/size-guard.test.ts +8 -8
- package/src/__tests__/skill-docs-sync-guard.test.ts +3 -2
- package/src/__tests__/skill-tool-factory.test.ts +247 -0
- package/src/__tests__/skills-install-staging.test.ts +15 -6
- package/src/__tests__/skills.test.ts +32 -0
- package/src/__tests__/slack-edit-ordering-characterization.test.ts +103 -0
- package/src/__tests__/slack-inbound-verification.test.ts +4 -1
- package/src/__tests__/slack-mention-binding-invariance.test.ts +103 -0
- package/src/__tests__/slack-mention-provider-leak-audit.test.ts +135 -0
- package/src/__tests__/starter-task-flow.test.ts +5 -4
- package/src/__tests__/steer-on-enqueue-question.test.ts +3 -2
- package/src/__tests__/stt-language-catalog-parity.test.ts +38 -0
- package/src/__tests__/subagent-allowlist-validation.test.ts +26 -4
- package/src/__tests__/subagent-cron-run-attribution.test.ts +301 -0
- package/src/__tests__/subagent-disposal.test.ts +10 -0
- package/src/__tests__/subagent-fork-notifications.test.ts +10 -0
- package/src/__tests__/subagent-fork-prompt-role.test.ts +85 -11
- package/src/__tests__/subagent-fork-spawn.test.ts +15 -7
- package/src/__tests__/subagent-manager-notify.test.ts +148 -0
- package/src/__tests__/subagent-notify-parent.test.ts +1 -1
- package/src/__tests__/subagent-persistence.test.ts +141 -0
- package/src/__tests__/subagent-role-registry.test.ts +172 -63
- package/src/__tests__/subagent-role-resolution.test.ts +88 -0
- package/src/__tests__/subagent-spawn-and-await.test.ts +48 -0
- package/src/__tests__/subagent-spawn-tool-fork.test.ts +111 -10
- package/src/__tests__/subagent-terminal-message.test.ts +88 -1
- package/src/__tests__/subagent-tool-gate-mode.test.ts +225 -10
- package/src/__tests__/subagent-tools.test.ts +1901 -108
- package/src/__tests__/surface-completion-compaction-boundary.test.ts +124 -0
- package/src/__tests__/surface-completion-in-flight-snapshot.test.ts +298 -0
- package/src/__tests__/system-prompt.test.ts +82 -7
- package/src/__tests__/tool-grant-request-escalation.test.ts +40 -1
- package/src/__tests__/tool-result-spool.test.ts +74 -18
- package/src/__tests__/tool-side-effects-documents.test.ts +174 -0
- package/src/__tests__/tool-side-effects-hook-failures.test.ts +129 -0
- package/src/__tests__/tools-audio-read.test.ts +10 -10
- package/src/__tests__/tools-get-route.test.ts +85 -12
- package/src/__tests__/tools-list-cli.test.ts +171 -0
- package/src/__tests__/top-level-renderer.test.ts +54 -1
- package/src/__tests__/trusted-contact-inline-approval-integration.test.ts +68 -1
- package/src/__tests__/trusted-contact-multichannel.test.ts +4 -1
- package/src/__tests__/turn-tail-chain.test.ts +135 -0
- package/src/__tests__/ui-choice-copy-surfaces.test.ts +6 -4
- package/src/__tests__/ui-visual-surface.test.ts +5 -4
- package/src/__tests__/ui-work-result-surface.test.ts +5 -4
- package/src/__tests__/unified-turn-context-visible-app.test.ts +7 -8
- package/src/__tests__/usage-routes.test.ts +62 -0
- package/src/__tests__/verification-outbound-delivery.test.ts +176 -0
- package/src/__tests__/visible-app-context.test.ts +20 -4
- package/src/__tests__/web-search-history.test.ts +96 -0
- package/src/__tests__/workspace-migration-139-clear-renamed-cost-profile-label.test.ts +6 -6
- package/src/__tests__/workspace-migration-140-repair-seed-pinned-memory-v3-live.test.ts +330 -0
- package/src/__tests__/workspace-migration-142-consolidate-voice-front-door.test.ts +150 -0
- package/src/acp/failure-error.ts +3 -5
- package/src/agent/attachments.ts +26 -24
- package/src/agent/history-repair/history-repair.ts +27 -26
- package/src/agent/image-optimize.ts +6 -6
- package/src/agent/loop.ts +40 -22
- package/src/agent/message-types.ts +3 -3
- package/src/api/constants/document-tools.ts +55 -0
- package/src/api/events/question-answered.ts +53 -0
- package/src/api/events/tool-result.ts +7 -0
- package/src/api/index.ts +13 -0
- package/src/api/responses/conversation-message.ts +11 -0
- package/src/api/surfaces-oauth-connect.test.ts +96 -0
- package/src/api/surfaces.ts +36 -8
- package/src/approvals/guardian-channel-delivery.ts +83 -0
- package/src/approvals/guardian-expiry-notifier.ts +13 -11
- package/src/approvals/guardian-request-resolvers.ts +222 -135
- package/src/calls/__tests__/tts-text-sanitizer.test.ts +13 -0
- package/src/calls/__tests__/voice-session-bridge.test.ts +131 -12
- package/src/calls/__tests__/voice-triage-escalate.test.ts +106 -0
- package/src/calls/call-controller.ts +30 -2
- package/src/calls/call-speech-output.ts +12 -4
- package/src/calls/call-transport.ts +18 -1
- package/src/calls/media-stream-output.ts +53 -5
- package/src/calls/media-stream-server.ts +21 -1
- package/src/calls/media-stream-stt-session.ts +49 -0
- package/src/calls/telephony-synthesis-language.ts +84 -0
- package/src/calls/tts-text-sanitizer.ts +12 -5
- package/src/calls/voice-session-bridge.ts +86 -17
- package/src/calls/voice-triage-escalate.ts +54 -7
- package/src/channels/__tests__/gateway-verification-sessions.test.ts +3 -3
- package/src/channels/__tests__/plugin-channel-declarations.test.ts +161 -0
- package/src/channels/config.ts +20 -2
- package/src/channels/gateway-verification-sessions.ts +7 -2
- package/src/channels/plugin-channel-declarations.ts +108 -0
- package/src/channels/types.ts +37 -1
- package/src/cli/AGENTS.md +5 -2
- package/src/cli/commands/__tests__/cache.test.ts +7 -4
- package/src/cli/commands/__tests__/cli-test-harness.ts +82 -0
- package/src/cli/commands/__tests__/conversations-wake.test.ts +51 -6
- package/src/cli/commands/__tests__/inference-models.test.ts +14 -0
- package/src/cli/commands/__tests__/inference-profiles.test.ts +44 -1
- package/src/cli/commands/__tests__/inference-providers.test.ts +7 -0
- package/src/cli/commands/__tests__/plugins.test.ts +692 -0
- package/src/cli/commands/__tests__/schedules.test.ts +136 -37
- package/src/cli/commands/conversations.ts +7 -3
- package/src/cli/commands/credentials.help.ts +7 -0
- package/src/cli/commands/inference-profiles.ts +9 -3
- package/src/cli/commands/inference.help.ts +16 -2
- package/src/cli/commands/mcp.help.ts +17 -6
- package/src/cli/commands/mcp.ts +9 -0
- package/src/cli/commands/memory/memory-retrospective.ts +18 -0
- package/src/cli/commands/notifications.ts +3 -6
- package/src/cli/commands/oauth/connect-surface-guidance.test.ts +16 -0
- package/src/cli/commands/oauth/connect-surface-guidance.ts +25 -7
- package/src/cli/commands/oauth/connect.test.ts +26 -1
- package/src/cli/commands/oauth/connect.ts +4 -1
- package/src/cli/commands/oauth/index.help.ts +3 -1
- package/src/cli/commands/oauth/providers.ts +3 -0
- package/src/cli/commands/plugins.help.ts +25 -2
- package/src/cli/commands/plugins.ts +289 -10
- package/src/cli/commands/schedules.help.ts +14 -12
- package/src/cli/commands/schedules.ts +19 -5
- package/src/cli/commands/stt.help.ts +27 -2
- package/src/cli/commands/tools.help.ts +4 -2
- package/src/cli/commands/tools.ts +38 -0
- package/src/cli/lib/__tests__/inspect-plugin.test.ts +1 -0
- package/src/cli/lib/__tests__/install-from-github.test.ts +79 -0
- package/src/cli/lib/__tests__/install-from-platform.test.ts +47 -1
- package/src/cli/lib/__tests__/plugin-surfaces.test.ts +86 -7
- package/src/cli/lib/__tests__/upgrade-plugin.test.ts +161 -0
- package/src/cli/lib/install-from-github.ts +61 -0
- package/src/cli/lib/install-from-platform.ts +13 -2
- package/src/cli/lib/plugin-surfaces.ts +124 -4
- package/src/cli/lib/upgrade-plugin.ts +60 -0
- package/src/cli.ts +1 -1
- package/src/config/__tests__/default-profile-catalog.test.ts +11 -1
- package/src/config/__tests__/deployment-context-defaults.test.ts +73 -0
- package/src/config/__tests__/memory-retrospective-schema.test.ts +40 -0
- package/src/config/__tests__/plugin-updates-schema.test.ts +43 -0
- package/src/config/__tests__/profile-tool-support.test.ts +131 -0
- package/src/config/bundled-skills/AGENTS.md +1 -1
- package/src/config/bundled-skills/acp/SKILL.md +6 -7
- package/src/config/bundled-skills/app-builder/SKILL.md +1 -1
- package/src/config/bundled-skills/app-builder/references/RESPONSIVE.md +3 -3
- package/src/config/bundled-skills/media-processing/services/preprocess.ts +14 -4
- package/src/config/bundled-skills/phone-calls/references/CONFIG.md +15 -15
- package/src/config/bundled-skills/schedule/SKILL.md +5 -1
- package/src/config/bundled-skills/schedule/TOOLS.json +2 -2
- package/src/config/bundled-skills/settings/TOOLS.json +3 -3
- package/src/config/bundled-skills/settings/tools/voice-config-update.ts +1 -0
- package/src/config/bundled-skills/subagent/SKILL.md +53 -22
- package/src/config/bundled-skills/subagent/TOOLS.json +14 -13
- package/src/config/bundled-skills/transcribe/tools/transcribe-media.test.ts +22 -1
- package/src/config/bundled-skills/transcribe/tools/transcribe-media.ts +9 -2
- package/src/config/bundled-skills/visualize/SKILL.md +1 -1
- package/src/config/call-site-defaults.ts +4 -5
- package/src/config/default-profile-catalog.ts +9 -5
- package/src/config/feature-flag-registry.json +25 -17
- package/src/config/llm-resolver.ts +45 -15
- package/src/config/loader.ts +75 -1
- package/src/config/profile-tool-support.ts +110 -0
- package/src/config/sanitize-for-transfer.ts +11 -0
- package/src/config/schema.ts +4 -0
- package/src/config/schemas/__tests__/live-voice.test.ts +142 -4
- package/src/config/schemas/__tests__/stt.test.ts +20 -5
- package/src/config/schemas/call-site-catalog.ts +4 -4
- package/src/config/schemas/calls.ts +0 -4
- package/src/config/schemas/channels.ts +13 -0
- package/src/config/schemas/live-voice.ts +82 -23
- package/src/config/schemas/llm.ts +1 -1
- package/src/config/schemas/memory-retrospective.ts +25 -0
- package/src/config/schemas/plugin-updates.ts +59 -0
- package/src/config/schemas/services.ts +8 -4
- package/src/config/schemas/stt.ts +18 -5
- package/src/config/schemas/timeouts.ts +1 -1
- package/src/config/schemas/tts.ts +63 -0
- package/src/contacts/__tests__/guardian-delivery-reader.test.ts +16 -0
- package/src/contacts/guardian-delivery-reader.ts +29 -2
- package/src/context/compactor.ts +4 -4
- package/src/context/post-turn-tool-result-truncation.ts +32 -16
- package/src/context/strip-injections.ts +42 -0
- package/src/context/tool-result-spool.ts +32 -18
- package/src/credential-execution/ces-runtime.ts +3 -0
- package/src/credential-execution/client.ts +22 -1
- package/src/credential-execution/process-manager.test.ts +99 -0
- package/src/credential-execution/process-manager.ts +62 -16
- package/src/daemon/__tests__/abort-null-controller.test.ts +141 -5
- package/src/daemon/__tests__/conversation-surfaces-launch.test.ts +8 -12
- package/src/daemon/__tests__/install-assistant-command.test.ts +234 -0
- package/src/daemon/__tests__/turn-tail-assistant-reply-notify.test.ts +2 -4
- package/src/daemon/__tests__/turn-tail-deleted-conversation.test.ts +20 -29
- package/src/daemon/channel-ui-capability.ts +4 -4
- package/src/daemon/conversation-agent-loop-handlers.ts +117 -61
- package/src/daemon/conversation-agent-loop.ts +318 -148
- package/src/daemon/conversation-attachments.ts +1 -1
- package/src/daemon/conversation-error.ts +8 -1
- package/src/daemon/conversation-initial-prompt.ts +16 -38
- package/src/daemon/conversation-lifecycle.ts +154 -32
- package/src/daemon/conversation-messaging.ts +106 -43
- package/src/daemon/conversation-process.ts +131 -33
- package/src/daemon/conversation-queue-manager.ts +10 -0
- package/src/daemon/conversation-runtime-assembly.ts +9 -4
- package/src/daemon/conversation-store.ts +52 -3
- package/src/daemon/conversation-surface-state.ts +9 -0
- package/src/daemon/conversation-surfaces.ts +403 -210
- package/src/daemon/conversation-tool-setup.ts +56 -7
- package/src/daemon/conversation-turn-finalize.ts +134 -89
- package/src/daemon/conversation.ts +111 -27
- package/src/daemon/disk-pressure-guard-lifecycle.ts +2 -2
- package/src/daemon/disk-pressure-guard.ts +3 -3
- package/src/daemon/doordash-steps.ts +5 -5
- package/src/daemon/external-plugins-bootstrap.ts +7 -1
- package/src/daemon/handlers/config-channels.ts +48 -5
- package/src/daemon/handlers/conversation-history.ts +1 -1
- package/src/daemon/handlers/conversations.ts +62 -17
- package/src/daemon/handlers/shared.ts +12 -0
- package/src/daemon/handlers/skills.ts +11 -1
- package/src/daemon/identity-helpers.ts +38 -10
- package/src/daemon/install-assistant-command.ts +322 -0
- package/src/daemon/lifecycle.ts +14 -3
- package/src/daemon/message-types/conversations.ts +6 -7
- package/src/daemon/message-types/sync.ts +1 -0
- package/src/daemon/persist-media-references.ts +23 -14
- package/src/daemon/process-message.ts +6 -6
- package/src/daemon/tool-setup-types.ts +19 -122
- package/src/daemon/tool-side-effects.ts +17 -94
- package/src/daemon/trust-context-types.ts +39 -0
- package/src/daemon/trust-context.ts +14 -6
- package/src/daemon/turn-tail-chain.ts +73 -0
- package/src/daemon/web-search-history.ts +47 -33
- package/src/heartbeat/heartbeat-service.ts +6 -6
- package/src/inbound/__tests__/oauth-callback-url.test.ts +83 -0
- package/src/inbound/oauth-callback-url.ts +61 -0
- package/src/ipc/__tests__/socket-path.test.ts +21 -0
- package/src/ipc/assistant-server.ts +17 -10
- package/src/ipc/cli-client.ts +10 -1
- package/src/ipc/routes/__tests__/documents-sync-ipc-routes.test.ts +56 -0
- package/src/ipc/routes/documents-sync-ipc-routes.ts +42 -0
- package/src/ipc/socket-cleanup.ts +9 -7
- package/src/ipc/socket-path.ts +4 -96
- package/src/live-voice/__tests__/live-voice-agent-turn.test.ts +1 -104
- package/src/live-voice/__tests__/live-voice-archive.test.ts +16 -16
- package/src/live-voice/__tests__/live-voice-attach-image.test.ts +209 -0
- package/src/live-voice/__tests__/live-voice-connection.test.ts +20 -0
- package/src/live-voice/__tests__/live-voice-events.test.ts +7 -8
- package/src/live-voice/__tests__/live-voice-flux-turn-end.test.ts +932 -0
- package/src/live-voice/__tests__/live-voice-integration.test.ts +12 -5
- package/src/live-voice/__tests__/live-voice-metrics.test.ts +115 -8
- package/src/live-voice/__tests__/live-voice-photo.test.ts +100 -0
- package/src/live-voice/__tests__/live-voice-progress.test.ts +179 -146
- package/src/live-voice/__tests__/live-voice-session-telemetry.test.ts +265 -0
- package/src/live-voice/__tests__/live-voice-stt.test.ts +324 -2
- package/src/live-voice/__tests__/live-voice-triage-escalate.test.ts +131 -2
- package/src/live-voice/__tests__/live-voice-tts-session.test.ts +0 -483
- package/src/live-voice/__tests__/live-voice-tts.test.ts +148 -2
- package/src/live-voice/__tests__/live-voice-vad.test.ts +313 -32
- package/src/live-voice/__tests__/progress-narration.test.ts +214 -0
- package/src/live-voice/__tests__/progress-phrases.test.ts +167 -0
- package/src/live-voice/__tests__/protocol.test.ts +52 -0
- package/src/live-voice/__tests__/runtime-websocket-shell.test.ts +3 -0
- package/src/live-voice/live-voice-archive.ts +13 -13
- package/src/live-voice/live-voice-connection.ts +7 -1
- package/src/live-voice/live-voice-metrics.ts +57 -32
- package/src/live-voice/live-voice-photo.ts +168 -0
- package/src/live-voice/live-voice-session.ts +1137 -351
- package/src/live-voice/live-voice-tts.ts +18 -2
- package/src/live-voice/progress-narration.ts +277 -0
- package/src/live-voice/progress-phrases.ts +105 -2
- package/src/live-voice/protocol.ts +110 -2
- package/src/mcp/__tests__/mcp-auth-orchestrator.test.ts +0 -1
- package/src/mcp/__tests__/mcp-oauth-client-registration.test.ts +196 -0
- package/src/mcp/__tests__/mcp-oauth-provider.test.ts +9 -9
- package/src/mcp/client.ts +14 -7
- package/src/mcp/mcp-auth-orchestrator.ts +13 -22
- package/src/mcp/mcp-oauth-provider.ts +195 -240
- package/src/messaging/providers/__tests__/callback-routing.test.ts +5 -1
- package/src/messaging/providers/__tests__/transport-dispatch.test.ts +66 -4
- package/src/messaging/providers/callback-routing.ts +1 -0
- package/src/messaging/providers/discord/api.ts +312 -0
- package/src/messaging/providers/discord/dm-delivery.test.ts +204 -0
- package/src/messaging/providers/discord/render.test.ts +204 -0
- package/src/messaging/providers/discord/render.ts +195 -0
- package/src/messaging/providers/discord/send.test.ts +179 -0
- package/src/messaging/providers/discord/send.ts +212 -0
- package/src/messaging/providers/discord/transport.ts +81 -0
- package/src/messaging/providers/index.ts +3 -1
- package/src/messaging/providers/retry-policy.test.ts +247 -0
- package/src/messaging/providers/retry-policy.ts +206 -0
- package/src/messaging/providers/slack/mention-source.test.ts +285 -0
- package/src/messaging/providers/slack/mention-source.ts +287 -0
- package/src/messaging/providers/telegram-bot/api.ts +47 -130
- package/src/messaging/providers/whatsapp/api.ts +47 -136
- package/src/monitoring/__tests__/db-integrity-sample.test.ts +4 -1
- package/src/monitoring/__tests__/plugin-auto-update.test.ts +495 -0
- package/src/monitoring/db-integrity-sample.ts +6 -1
- package/src/monitoring/plugin-auto-update.ts +463 -0
- package/src/monitoring/resource-sampler.ts +21 -5
- package/src/monitoring/worker.ts +13 -0
- package/src/notifications/__tests__/assistant-reply-producer.test.ts +375 -2
- package/src/notifications/__tests__/broadcaster.test.ts +37 -1
- package/src/notifications/__tests__/copy-composer.test.ts +165 -0
- package/src/notifications/__tests__/emit-signal-pipeline-failure.test.ts +205 -0
- package/src/notifications/__tests__/notification-utils.test.ts +249 -0
- package/src/notifications/adapters/platform.ts +31 -2
- package/src/notifications/assistant-reply-producer.ts +79 -32
- package/src/notifications/broadcaster.ts +30 -10
- package/src/notifications/copy-composer.ts +69 -0
- package/src/notifications/emit-signal.ts +82 -15
- package/src/notifications/events-store.ts +10 -2
- package/src/notifications/home-feed-side-effect.ts +6 -3
- package/src/notifications/notification-utils.ts +173 -5
- package/src/notifications/signal.ts +13 -0
- package/src/oauth/AGENTS.md +4 -2
- package/src/oauth/byo-connection.ts +6 -1
- package/src/oauth/credential-token-resolver.ts +3 -10
- package/src/oauth/manual-token-connection.ts +29 -64
- package/src/oauth/manual-token-providers.ts +62 -0
- package/src/oauth/provider-serializer.ts +13 -0
- package/src/oauth/seed-providers.ts +62 -11
- package/src/onboarding/onboarding-events-store.ts +91 -1
- package/src/permissions/question-prompter.test.ts +4 -0
- package/src/permissions/question-prompter.ts +21 -4
- package/src/persistence/__tests__/conversation-lineage.test.ts +248 -0
- package/src/persistence/attachments-store.ts +15 -15
- package/src/persistence/conversation-attention-store.ts +30 -10
- package/src/persistence/conversation-bootstrap.ts +17 -0
- package/src/persistence/conversation-crud.ts +363 -97
- package/src/persistence/conversation-group-migration.ts +1 -1
- package/src/persistence/conversation-key-store.ts +38 -10
- package/src/persistence/conversation-lineage.ts +226 -0
- package/src/persistence/conversation-plugin-facade.ts +28 -3
- package/src/persistence/conversation-queries.ts +207 -33
- package/src/persistence/conversation-types.ts +107 -1
- package/src/persistence/jobs-store.ts +32 -8
- package/src/persistence/llm-usage-store.ts +22 -0
- package/src/persistence/migrations/361-normalize-managed-connection-rows.test.ts +168 -0
- package/src/persistence/migrations/361-normalize-managed-connection-rows.ts +76 -0
- package/src/persistence/migrations/362-add-conversation-subagent-kind.test.ts +187 -0
- package/src/persistence/migrations/362-add-conversation-subagent-kind.ts +93 -0
- package/src/persistence/migrations/363-backfill-schedule-inference-profile.test.ts +229 -0
- package/src/persistence/migrations/363-backfill-schedule-inference-profile.ts +127 -0
- package/src/persistence/migrations/364-add-schedule-source-key.test.ts +120 -0
- package/src/persistence/migrations/364-add-schedule-source-key.ts +43 -0
- package/src/persistence/migrations/365-add-conversation-fork-strategy.ts +27 -0
- package/src/persistence/schema/conversations.ts +32 -0
- package/src/persistence/schema/infrastructure.ts +3 -0
- package/src/persistence/steps.ts +30 -0
- package/src/persistence/subagent-store.ts +148 -6
- package/src/platform/managed-speech.test.ts +20 -0
- package/src/platform/managed-speech.ts +9 -0
- package/src/plugin-api/__tests__/oauth-callback-url-export.test.ts +29 -0
- package/src/plugin-api/index.ts +28 -5
- package/src/plugin-api/webhook-url.ts +151 -0
- package/src/plugins/__tests__/installed-plugin-dirs.test.ts +82 -0
- package/src/plugins/__tests__/mcp-servers.test.ts +370 -0
- package/src/plugins/__tests__/source-fingerprint.test.ts +5 -1
- package/src/plugins/collect-source-versions.ts +4 -27
- package/src/plugins/defaults/exploration-drift/hooks/post-tool-use.ts +5 -5
- package/src/plugins/defaults/exploration-drift/package.json +1 -1
- package/src/plugins/defaults/image-recovery/hooks/post-model-call.ts +4 -2
- package/src/plugins/defaults/image-recovery/recover.ts +67 -57
- package/src/plugins/defaults/index.ts +1 -1
- package/src/plugins/defaults/memory/__tests__/conversation-queries.test.ts +64 -18
- package/src/plugins/defaults/memory/__tests__/jobs-worker-outcome-contract.test.ts +242 -0
- package/src/plugins/defaults/memory/__tests__/jobs-worker-retrospective-sweep.test.ts +24 -1
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-enqueue.test.ts +48 -3
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-job.test.ts +511 -19
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-prompt.test.ts +39 -3
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-provider-path.test.ts +493 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-wake-chain.test.ts +5 -4
- package/src/plugins/defaults/memory/__tests__/memory-tier-boundary-guard.test.ts +5 -1
- package/src/plugins/defaults/memory/job-handlers.ts +87 -5
- package/src/plugins/defaults/memory/jobs-worker.ts +60 -6
- package/src/plugins/defaults/memory/memory-retrospective-constants.ts +11 -0
- package/src/plugins/defaults/memory/memory-retrospective-enqueue.ts +29 -0
- package/src/plugins/defaults/memory/memory-retrospective-job.ts +342 -60
- package/src/plugins/defaults/memory/memory-retrospective-prompt.ts +23 -3
- package/src/plugins/defaults/memory/substrate/consolidation-lock.ts +9 -4
- package/src/plugins/defaults/memory/substrate/skill-content.ts +8 -1
- package/src/plugins/defaults/memory/substrate/sweep-job.ts +7 -0
- package/src/plugins/defaults/memory/v1/graph/injection.ts +1 -1
- package/src/plugins/defaults/memory/v2/router.ts +8 -0
- package/src/plugins/defaults/turn-context/unified-turn-context.ts +1 -2
- package/src/plugins/external-plugin-loader.ts +64 -8
- package/src/plugins/installed-plugin-dirs.ts +101 -0
- package/src/plugins/mcp-servers.ts +343 -0
- package/src/plugins/mtime-cache.ts +34 -22
- package/src/plugins/types.ts +69 -8
- package/src/prompts/persona-resolver.ts +6 -2
- package/src/prompts/system-prompt.ts +72 -17
- package/src/providers/__tests__/inference.test.ts +40 -0
- package/src/providers/__tests__/preflight-resolved-config.test.ts +58 -1
- package/src/providers/__tests__/retry-callsite.test.ts +72 -2
- package/src/providers/__tests__/vellum-connection-routing.test.ts +42 -11
- package/src/providers/anthropic/client.ts +112 -95
- package/src/providers/connection-resolution.ts +35 -1
- package/src/providers/gemini/client.ts +5 -5
- package/src/providers/inference/adapter-factory.ts +16 -10
- package/src/providers/inference/auth.ts +16 -0
- package/src/providers/inference/connection-availability.ts +164 -12
- package/src/providers/inference/connections.ts +25 -1
- package/src/providers/media-resolve.ts +21 -11
- package/src/providers/model-catalog.ts +15 -0
- package/src/providers/openai/chat-completions-provider.ts +4 -4
- package/src/providers/openai/responses-provider.ts +12 -25
- package/src/providers/registry.ts +9 -4
- package/src/providers/retry.ts +25 -0
- package/src/providers/server-tool-pairing.ts +134 -0
- package/src/providers/speech-to-text/__tests__/deepgram-flux-frames.test.ts +433 -0
- package/src/providers/speech-to-text/__tests__/deepgram-flux-realtime.test.ts +620 -0
- package/src/providers/speech-to-text/__tests__/provider-catalog.test.ts +34 -0
- package/src/providers/speech-to-text/__tests__/resolve.test.ts +472 -12
- package/src/providers/speech-to-text/deepgram-flux-frames.ts +395 -0
- package/src/providers/speech-to-text/deepgram-flux-realtime.ts +719 -0
- package/src/providers/speech-to-text/deepgram-realtime.test.ts +283 -1
- package/src/providers/speech-to-text/deepgram-realtime.ts +117 -3
- package/src/providers/speech-to-text/deepgram.ts +1 -0
- package/src/providers/speech-to-text/provider-catalog.ts +137 -8
- package/src/providers/speech-to-text/resolve.ts +105 -12
- package/src/providers/speech-to-text/vellum-managed.ts +7 -1
- package/src/providers/speech-to-text/vellum-speech-relay-connection.ts +3 -1
- package/src/providers/types.ts +26 -6
- package/src/providers/vellum-model-routing.ts +6 -7
- package/src/runtime/AGENTS.md +2 -0
- package/src/runtime/__tests__/agent-wake.test.ts +16 -16
- package/src/runtime/__tests__/background-job-runner.test.ts +0 -144
- package/src/runtime/__tests__/desktop-presence.test.ts +162 -0
- package/src/runtime/__tests__/local-actor-identity-force-refresh.test.ts +104 -0
- package/src/runtime/agent-wake.ts +63 -24
- package/src/runtime/assistant-event-hub.ts +110 -12
- package/src/runtime/auth/__tests__/guard-tests.test.ts +2 -0
- package/src/runtime/auth/__tests__/scopes.test.ts +20 -0
- package/src/runtime/auth/same-actor.ts +56 -8
- package/src/runtime/auth/scopes.ts +12 -0
- package/src/runtime/auth/types.ts +2 -0
- package/src/runtime/background-job-runner.ts +5 -27
- package/src/runtime/channel-readiness-service.ts +59 -3
- package/src/runtime/channel-readiness-types.ts +18 -0
- package/src/runtime/desktop-presence.ts +69 -0
- package/src/runtime/http-server.ts +12 -0
- package/src/runtime/local-actor-identity.ts +29 -14
- package/src/runtime/migrations/__tests__/vbundle-symlink-streaming-importer.test.ts +0 -0
- package/src/runtime/routes/__tests__/channel-verification-routes.test.ts +1 -1
- package/src/runtime/routes/__tests__/client-routes.test.ts +186 -14
- package/src/runtime/routes/__tests__/conversation-list-routes.test.ts +547 -1
- package/src/runtime/routes/__tests__/conversation-management-routes.test.ts +162 -7
- package/src/runtime/routes/__tests__/conversation-query-routes.test.ts +14 -10
- package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +340 -3
- package/src/runtime/routes/__tests__/llm-call-sites-routes.test.ts +28 -1
- package/src/runtime/routes/__tests__/plugins-routes.test.ts +217 -1
- package/src/runtime/routes/__tests__/question-routes.test.ts +64 -0
- package/src/runtime/routes/__tests__/retrospective-routes.test.ts +20 -0
- package/src/runtime/routes/__tests__/skill-history-route.test.ts +116 -0
- package/src/runtime/routes/__tests__/sse-actor-principal-heal.test.ts +165 -0
- package/src/runtime/routes/__tests__/stt-routes.test.ts +25 -0
- package/src/runtime/routes/__tests__/surface-action-routes.test.ts +2 -0
- package/src/runtime/routes/__tests__/wake-conversation-routes.test.ts +60 -2
- package/src/runtime/routes/archive-utils.ts +16 -12
- package/src/runtime/routes/attachment-routes.ts +13 -6
- package/src/runtime/routes/canned-reply-release.ts +54 -0
- package/src/runtime/routes/channel-availability-routes.ts +32 -14
- package/src/runtime/routes/channel-verification-routes.ts +2 -27
- package/src/runtime/routes/chatgpt-subscription-auth-routes.ts +5 -0
- package/src/runtime/routes/client-routes.ts +114 -2
- package/src/runtime/routes/conversation-cli-routes.ts +10 -12
- package/src/runtime/routes/conversation-list-routes.ts +59 -10
- package/src/runtime/routes/conversation-management-routes.ts +47 -18
- package/src/runtime/routes/conversation-query-routes.ts +32 -53
- package/src/runtime/routes/conversation-routes.ts +360 -293
- package/src/runtime/routes/credential-prompt-routes.ts +20 -2
- package/src/runtime/routes/debug-routes.ts +4 -1
- package/src/runtime/routes/default-provider-routes.ts +7 -11
- package/src/runtime/routes/documents-routes.ts +44 -1
- package/src/runtime/routes/events-routes.ts +29 -1
- package/src/runtime/routes/host-app-control-routes.ts +2 -0
- package/src/runtime/routes/host-bash-routes.ts +2 -0
- package/src/runtime/routes/host-browser-routes.ts +2 -0
- package/src/runtime/routes/host-cu-routes.ts +2 -0
- package/src/runtime/routes/host-file-routes.ts +2 -0
- package/src/runtime/routes/host-transfer-routes.ts +4 -0
- package/src/runtime/routes/identity-routes.ts +6 -6
- package/src/runtime/routes/inbound-message-handler.ts +5 -5
- package/src/runtime/routes/inbound-stages/acl-enforcement.ts +7 -3
- package/src/runtime/routes/inbound-stages/background-dispatch.ts +14 -0
- package/src/runtime/routes/inbound-stages/bootstrap-intercept.ts +2 -2
- package/src/runtime/routes/inbound-stages/guardian-activation-intercept.test.ts +14 -11
- package/src/runtime/routes/inbound-stages/guardian-activation-intercept.ts +14 -17
- package/src/runtime/routes/inference-profile-availability-guard.ts +171 -0
- package/src/runtime/routes/inference-profile-session-handler.ts +26 -2
- package/src/runtime/routes/inference-profile-session-routes.ts +2 -1
- package/src/runtime/routes/inference-profiles-routes.ts +152 -73
- package/src/runtime/routes/inference-provider-connection-routes.ts +0 -16
- package/src/runtime/routes/llm-call-sites-routes.ts +13 -0
- package/src/runtime/routes/log-export-routes.ts +1 -1
- package/src/runtime/routes/mcp-auth-routes.ts +141 -57
- package/src/runtime/routes/notification-routes.ts +0 -14
- package/src/runtime/routes/oauth-providers.ts +1 -0
- package/src/runtime/routes/plugins-routes.ts +127 -4
- package/src/runtime/routes/profiler-routes.ts +4 -2
- package/src/runtime/routes/question-routes.ts +28 -10
- package/src/runtime/routes/retrospective-routes.ts +9 -3
- package/src/runtime/routes/schedule-routes.ts +210 -12
- package/src/runtime/routes/secret-routes.ts +40 -23
- package/src/runtime/routes/settings-routes.ts +41 -13
- package/src/runtime/routes/skills-routes.ts +79 -0
- package/src/runtime/routes/sse-actor-principal-heal.ts +113 -0
- package/src/runtime/routes/stt-routes.ts +31 -25
- package/src/runtime/routes/ui-snapshot-routes.ts +1 -0
- package/src/runtime/routes/wake-conversation-routes.ts +23 -0
- package/src/runtime/services/__tests__/conversation-serializer.test.ts +1 -0
- package/src/runtime/services/conversation-serializer.ts +24 -9
- package/src/runtime/sync/documents-sidecar-publish.test.ts +101 -0
- package/src/runtime/sync/resource-sync-events.ts +79 -1
- package/src/runtime/sync/worker-daemon-notify.test.ts +34 -0
- package/src/runtime/sync/worker-daemon-notify.ts +44 -6
- package/src/runtime/verification-outbound-actions.ts +250 -222
- package/src/runtime/verification-templates.ts +41 -30
- package/src/schedule/__tests__/plugin-schedule-declarations.test.ts +614 -0
- package/src/schedule/__tests__/plugin-schedule-reconciler.test.ts +1066 -0
- package/src/schedule/inference-profile.ts +64 -3
- package/src/schedule/plugin-schedule-declarations.ts +518 -0
- package/src/schedule/plugin-schedule-reconciler.ts +605 -0
- package/src/schedule/plugin-schedules-gate.ts +23 -0
- package/src/schedule/schedule-store.ts +520 -26
- package/src/schedule/scheduler.ts +38 -0
- package/src/security/credential-backend.ts +51 -4
- package/src/security/encrypted-store.ts +64 -42
- package/src/security/oauth2.ts +6 -22
- package/src/security/secure-keys.ts +55 -7
- package/src/skills/catalog-install.ts +20 -7
- package/src/skills/inline-command-runner.ts +2 -7
- package/src/skills/skill-history.test.ts +269 -0
- package/src/skills/skill-history.ts +250 -0
- package/src/skills/skillssh-registry.ts +1 -1
- package/src/stt/__tests__/daemon-batch-transcriber.test.ts +22 -0
- package/src/stt/__tests__/language-metadata.test.ts +85 -0
- package/src/stt/__tests__/speech-energy.test.ts +79 -0
- package/src/stt/__tests__/types.test.ts +94 -0
- package/src/stt/daemon-batch-transcriber.ts +18 -6
- package/src/stt/language-metadata.ts +65 -0
- package/src/stt/speech-energy.ts +115 -12
- package/src/stt/stt-stream-session.ts +8 -4
- package/src/stt/types.ts +119 -0
- package/src/subagent/__tests__/consult-prompt.test.ts +25 -0
- package/src/subagent/consult-prompt.ts +3 -1
- package/src/subagent/consult-transcript.ts +1 -1
- package/src/subagent/manager.ts +304 -44
- package/src/subagent/progress-events.ts +35 -0
- package/src/subagent/role-resolution.ts +88 -0
- package/src/subagent/types.ts +303 -67
- package/src/subagent/validate-allowlists.ts +3 -4
- package/src/telegram/__tests__/webhook-health.test.ts +73 -1
- package/src/telegram/webhook-health.ts +101 -3
- package/src/telemetry/__tests__/telemetry-event-fixtures.ts +2 -0
- package/src/telemetry/activation-funnel.ts +6 -1
- package/src/telemetry/live-voice-funnel.ts +75 -0
- package/src/telemetry/telemetry-event-sources.ts +6 -0
- package/src/telemetry/telemetry-wire-source.json +1 -1
- package/src/telemetry/telemetry-wire.generated.ts +2 -0
- package/src/telemetry/types.ts +23 -4
- package/src/telemetry/usage-telemetry-reporter.test.ts +4 -0
- package/src/tools/__tests__/subagent-workflow-tool-input-schemas.test.ts +1 -1
- package/src/tools/ask-question/ask-question-tool.test.ts +118 -3
- package/src/tools/ask-question/ask-question-tool.ts +56 -2
- package/src/tools/browser/browser-execution.ts +1 -1
- package/src/tools/calls/call-start.ts +8 -5
- package/src/tools/conversation-groups/move_to_group.ts +10 -12
- package/src/tools/document/document-tool.ts +11 -4
- package/src/tools/execution-timeout.ts +9 -2
- package/src/tools/executor.ts +7 -6
- package/src/tools/filesystem/edit.ts +1 -1
- package/src/tools/filesystem/list.ts +1 -1
- package/src/tools/filesystem/read.ts +7 -13
- package/src/tools/filesystem/search.ts +23 -21
- package/src/tools/filesystem/write.ts +4 -1
- package/src/tools/host-filesystem/edit.ts +7 -2
- package/src/tools/host-filesystem/read.ts +21 -8
- package/src/tools/host-filesystem/write.ts +10 -2
- package/src/tools/host-terminal/host-shell.ts +2 -2
- package/src/tools/mcp/mcp-tool-factory.ts +25 -1
- package/src/tools/permission-checker.ts +8 -3
- package/src/tools/schedule/delete.ts +13 -1
- package/src/tools/schedule/list.ts +14 -3
- package/src/tools/schedule/update.ts +56 -33
- package/src/tools/shared/filesystem/audio-read.ts +6 -4
- package/src/tools/shared/filesystem/file-ops-service.ts +176 -116
- package/src/tools/shared/filesystem/image-read.ts +12 -12
- package/src/tools/shared/filesystem/path-policy.ts +70 -5
- package/src/tools/shared/filesystem/size-guard.ts +6 -6
- package/src/tools/shared/input-misuse.ts +72 -0
- package/src/tools/skills/scaffold-managed.ts +82 -0
- package/src/tools/skills/skill-tool-factory.ts +14 -1
- package/src/tools/subagent/consult-deadline.ts +12 -7
- package/src/tools/subagent/message.ts +3 -1
- package/src/tools/subagent/read.ts +66 -4
- package/src/tools/subagent/spawn.ts +500 -78
- package/src/tools/tool-approval-handler.ts +45 -4
- package/src/tools/types.ts +22 -3
- package/src/tools/ui-surface/surface-shape-docs.ts +1 -1
- package/src/tools/ui-surface/visual-validation.ts +2 -0
- package/src/tts/__tests__/provider-adapters.test.ts +147 -0
- package/src/tts/__tests__/speakable-segments.test.ts +470 -0
- package/src/tts/language-voices.ts +23 -0
- package/src/tts/providers/deepgram-provider.ts +3 -1
- package/src/tts/providers/elevenlabs-provider.ts +73 -1
- package/src/tts/providers/xai-provider.ts +28 -2
- package/src/tts/speakable-segments.ts +293 -23
- package/src/tts/synthesis-stream.ts +7 -0
- package/src/tts/types.ts +7 -0
- package/src/usage/__tests__/subagent-attribution.test.ts +109 -0
- package/src/usage/subagent-attribution.ts +108 -0
- package/src/util/__tests__/language-subtag.test.ts +54 -0
- package/src/util/__tests__/sqlite-retry.test.ts +112 -0
- package/src/util/abort-reasons.ts +16 -0
- package/src/util/ansi.ts +44 -0
- package/src/util/disk-usage.ts +17 -13
- package/src/util/image-conversion.ts +20 -13
- package/src/util/language-subtag.ts +43 -0
- package/src/util/schedule-source-key.ts +23 -0
- package/src/util/unicode.ts +1 -1
- package/src/util/worker-process.ts +7 -1
- package/src/workspace/git-service.ts +33 -9
- package/src/workspace/migrations/140-repair-seed-pinned-memory-v3-live.ts +260 -0
- package/src/workspace/migrations/141-stt-english-default-to-multilingual.ts +111 -0
- package/src/workspace/migrations/142-consolidate-voice-front-door.ts +70 -0
- package/src/workspace/migrations/__tests__/141-stt-english-default-to-multilingual.test.ts +116 -0
- package/src/workspace/migrations/registry.ts +6 -0
- package/src/workspace/top-level-renderer.ts +15 -2
- package/src/__tests__/tool-side-effects-slack-dm.test.ts +0 -357
- package/src/daemon/install-symlink.ts +0 -207
- package/src/live-voice/__tests__/front-decision.test.ts +0 -525
- package/src/live-voice/front-decision.ts +0 -429
- package/src/notifications/deferred-emit.ts +0 -147
|
@@ -76,6 +76,7 @@ import { assistantEventHub } from "../../runtime/assistant-event-hub.js";
|
|
|
76
76
|
import { CALL_OPENING_MARKER } from "../voice-control-protocol.js";
|
|
77
77
|
import {
|
|
78
78
|
cutFrontDoorContentAtVerdict,
|
|
79
|
+
preSpeechLanguageRuleFragment,
|
|
79
80
|
startVoiceTurn,
|
|
80
81
|
TOOL_RESULT_PREVIEW_MAX_CHARS,
|
|
81
82
|
type VoiceTurnOptions,
|
|
@@ -380,6 +381,56 @@ describe("startVoiceTurn escalation-continuation persistence", () => {
|
|
|
380
381
|
voiceSessionTurn: true,
|
|
381
382
|
});
|
|
382
383
|
});
|
|
384
|
+
|
|
385
|
+
test("a live-voice turn persists its session attribution under metadata.client", async () => {
|
|
386
|
+
const fake = makeFakeConversation({ processing: false });
|
|
387
|
+
fakeConversation = fake.conversation;
|
|
388
|
+
|
|
389
|
+
await startVoiceTurn({
|
|
390
|
+
...makeTurnOptions(),
|
|
391
|
+
voiceTelemetry: { sessionId: "session-123", client: "ios" },
|
|
392
|
+
});
|
|
393
|
+
|
|
394
|
+
// `turn-events-store` projects `$.client` onto `TurnTelemetryEvent.client`,
|
|
395
|
+
// so this bag is what makes a voice turn countable per client and joinable
|
|
396
|
+
// to its session's funnel rows. The platform goes in `os`, the same key
|
|
397
|
+
// the HTTP send path fills, so voice turns sit in the column existing turn
|
|
398
|
+
// analytics already read.
|
|
399
|
+
expect(fake.lastPersistOpts()?.metadata).toEqual({
|
|
400
|
+
voiceSessionTurn: true,
|
|
401
|
+
client: {
|
|
402
|
+
voice: true,
|
|
403
|
+
voice_session_id: "session-123",
|
|
404
|
+
os: "ios",
|
|
405
|
+
},
|
|
406
|
+
});
|
|
407
|
+
});
|
|
408
|
+
|
|
409
|
+
test("omits the client key when the session reported no originating client", async () => {
|
|
410
|
+
const fake = makeFakeConversation({ processing: false });
|
|
411
|
+
fakeConversation = fake.conversation;
|
|
412
|
+
|
|
413
|
+
await startVoiceTurn({
|
|
414
|
+
...makeTurnOptions(),
|
|
415
|
+
voiceTelemetry: { sessionId: "session-123" },
|
|
416
|
+
});
|
|
417
|
+
|
|
418
|
+
expect(fake.lastPersistOpts()?.metadata).toEqual({
|
|
419
|
+
voiceSessionTurn: true,
|
|
420
|
+
client: { voice: true, voice_session_id: "session-123" },
|
|
421
|
+
});
|
|
422
|
+
});
|
|
423
|
+
|
|
424
|
+
test("a phone turn carries no client bag", async () => {
|
|
425
|
+
const fake = makeFakeConversation({ processing: false });
|
|
426
|
+
fakeConversation = fake.conversation;
|
|
427
|
+
|
|
428
|
+
await startVoiceTurn(makeTurnOptions());
|
|
429
|
+
|
|
430
|
+
// Only live-voice sessions pass `voiceTelemetry`; a phone call has no
|
|
431
|
+
// live-voice session id to attribute a turn to.
|
|
432
|
+
expect(fake.lastPersistOpts()?.metadata).not.toHaveProperty("client");
|
|
433
|
+
});
|
|
383
434
|
});
|
|
384
435
|
|
|
385
436
|
describe("startVoiceTurn hiddenSyntheticPrompt", () => {
|
|
@@ -448,6 +499,18 @@ describe("startVoiceTurn hiddenSyntheticPrompt", () => {
|
|
|
448
499
|
});
|
|
449
500
|
});
|
|
450
501
|
|
|
502
|
+
// The turn installs its resolved control prompt, then cleanup resets it to
|
|
503
|
+
// null, so capture every applied value and read the installed (non-null) one.
|
|
504
|
+
function captureInstalledPrompt(): () => string | undefined {
|
|
505
|
+
const fake = makeFakeConversation({ processing: false });
|
|
506
|
+
fakeConversation = fake.conversation;
|
|
507
|
+
const applied: Array<string | null> = [];
|
|
508
|
+
fake.conversation.setVoiceCallControlPrompt = (prompt) => {
|
|
509
|
+
applied.push(prompt);
|
|
510
|
+
};
|
|
511
|
+
return () => applied.find((p): p is string => typeof p === "string");
|
|
512
|
+
}
|
|
513
|
+
|
|
451
514
|
describe("startVoiceTurn triage-and-escalate control prompt", () => {
|
|
452
515
|
// Live-voice supplies its own voiceControlPrompt, bypassing
|
|
453
516
|
// buildVoiceCallControlPrompt where the routing-leg rule is normally injected.
|
|
@@ -455,18 +518,6 @@ describe("startVoiceTurn triage-and-escalate control prompt", () => {
|
|
|
455
518
|
// verdict protocol and can't hold or hand off.
|
|
456
519
|
const LIVE_VOICE_PROMPT = "You are speaking in a local live voice session.";
|
|
457
520
|
|
|
458
|
-
// The turn installs its resolved control prompt, then cleanup resets it to
|
|
459
|
-
// null — so capture every applied value and read the installed (non-null) one.
|
|
460
|
-
function captureInstalledPrompt(): () => string | undefined {
|
|
461
|
-
const fake = makeFakeConversation({ processing: false });
|
|
462
|
-
fakeConversation = fake.conversation;
|
|
463
|
-
const applied: Array<string | null> = [];
|
|
464
|
-
fake.conversation.setVoiceCallControlPrompt = (prompt) => {
|
|
465
|
-
applied.push(prompt);
|
|
466
|
-
};
|
|
467
|
-
return () => applied.find((p): p is string => typeof p === "string");
|
|
468
|
-
}
|
|
469
|
-
|
|
470
521
|
test("appends the front-door decision rule to a caller-supplied prompt", async () => {
|
|
471
522
|
const installed = captureInstalledPrompt();
|
|
472
523
|
await startVoiceTurn({
|
|
@@ -510,6 +561,74 @@ describe("startVoiceTurn triage-and-escalate control prompt", () => {
|
|
|
510
561
|
});
|
|
511
562
|
});
|
|
512
563
|
|
|
564
|
+
describe("default call protocol numbered rules", () => {
|
|
565
|
+
// With no caller-supplied voiceControlPrompt the bridge builds the numbered
|
|
566
|
+
// CALL PROTOCOL RULES itself. Pin the speak-the-caller's-language rule and
|
|
567
|
+
// keep the numbering gapless so no rule silently shadows another.
|
|
568
|
+
test("teaches speaking the caller's language as its own numbered rule", async () => {
|
|
569
|
+
const installed = captureInstalledPrompt();
|
|
570
|
+
await startVoiceTurn(makeTurnOptions());
|
|
571
|
+
expect(installed()).toContain(
|
|
572
|
+
"12. Speak the caller's language: reply in the language of the caller's most recent actual speech, and follow them if they switch languages mid-call. Synthetic user turns (parenthetical markers like the call-connected and verification-completed notices) are not caller speech and never set the language. Before the caller has spoken, such as on the opening greeting turn, use the language the Task context implies, if any; otherwise default to English.",
|
|
573
|
+
);
|
|
574
|
+
});
|
|
575
|
+
|
|
576
|
+
test("the language rule excludes synthetic turns and covers pre-speech turns", async () => {
|
|
577
|
+
// Outbound calls open with the English "(call connected ...)" sentinel as
|
|
578
|
+
// the latest user-role turn, and the verification-complete sentinel does
|
|
579
|
+
// the same mid-call. Neither is caller speech, so neither may pull a
|
|
580
|
+
// Spanish or Japanese Task into an English opener.
|
|
581
|
+
const installed = captureInstalledPrompt();
|
|
582
|
+
await startVoiceTurn(makeTurnOptions());
|
|
583
|
+
const prompt = installed()!;
|
|
584
|
+
expect(prompt).toContain("most recent actual speech");
|
|
585
|
+
expect(prompt).toContain("not caller speech and never set the language");
|
|
586
|
+
expect(prompt).toContain(
|
|
587
|
+
"use the language the Task context implies, if any; otherwise default to English",
|
|
588
|
+
);
|
|
589
|
+
});
|
|
590
|
+
|
|
591
|
+
test("a monolingual listening language becomes the pre-speech fallback", () => {
|
|
592
|
+
// An assistant pinned to services.stt.language = "es" on a provider that
|
|
593
|
+
// honors the pin (deepgram, vellum) is already transcribing Spanish, so
|
|
594
|
+
// the opener must not default to English. The default test config runs
|
|
595
|
+
// the auto-detect branch ("multi"), so the pinned branch is covered at
|
|
596
|
+
// the fragment level.
|
|
597
|
+
expect(preSpeechLanguageRuleFragment("es", "deepgram")).toContain(
|
|
598
|
+
'configured listening language ("es")',
|
|
599
|
+
);
|
|
600
|
+
expect(preSpeechLanguageRuleFragment("es", "vellum")).toContain(
|
|
601
|
+
"default to English only when neither gives a language",
|
|
602
|
+
);
|
|
603
|
+
for (const autoDetect of ["multi", "", " ", undefined]) {
|
|
604
|
+
expect(preSpeechLanguageRuleFragment(autoDetect, "deepgram")).toBe(
|
|
605
|
+
"use the language the Task context implies, if any; otherwise default to English",
|
|
606
|
+
);
|
|
607
|
+
}
|
|
608
|
+
});
|
|
609
|
+
|
|
610
|
+
test("a language pin on an auto-detecting provider keeps the English fallback", () => {
|
|
611
|
+
// google-gemini and openai-whisper ignore services.stt.language entirely
|
|
612
|
+
// (languageSelection: "auto"), so a persisted "es" pin must not force a
|
|
613
|
+
// Spanish greeting the transcriber will not honor.
|
|
614
|
+
for (const provider of ["google-gemini", "openai-whisper", undefined]) {
|
|
615
|
+
expect(preSpeechLanguageRuleFragment("es", provider)).toBe(
|
|
616
|
+
"use the language the Task context implies, if any; otherwise default to English",
|
|
617
|
+
);
|
|
618
|
+
}
|
|
619
|
+
});
|
|
620
|
+
|
|
621
|
+
test("rule numbers stay sequential from 0, including the routing rule", async () => {
|
|
622
|
+
const installed = captureInstalledPrompt();
|
|
623
|
+
await startVoiceTurn({ ...makeTurnOptions(), routingLeg: "escalated" });
|
|
624
|
+
const numbers = [...installed()!.matchAll(/^(\d+)\. /gm)].map((match) =>
|
|
625
|
+
Number(match[1]),
|
|
626
|
+
);
|
|
627
|
+
expect(numbers.length).toBeGreaterThan(12);
|
|
628
|
+
expect(numbers).toEqual(numbers.map((_, index) => index));
|
|
629
|
+
});
|
|
630
|
+
});
|
|
631
|
+
|
|
513
632
|
describe("startVoiceTurn channel capabilities", () => {
|
|
514
633
|
// Whether a call can show a surface is a property of its channel, not of
|
|
515
634
|
// calls in general, so the bridge applies no voice-specific override: a
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { describe, expect, test } from "bun:test";
|
|
2
2
|
|
|
3
|
+
import { DEEPGRAM_MULTI_LANGUAGE_CODES } from "../../providers/speech-to-text/deepgram.js";
|
|
3
4
|
import {
|
|
4
5
|
capEscalationBridge,
|
|
5
6
|
classifyFrontDoorLeading,
|
|
@@ -7,6 +8,8 @@ import {
|
|
|
7
8
|
escalatedContinuationRule,
|
|
8
9
|
ESCALATION_CONTINUATION_CONTENT,
|
|
9
10
|
FALLBACK_ESCALATION_BRIDGE,
|
|
11
|
+
FALLBACK_ESCALATION_BRIDGE_BY_LANGUAGE,
|
|
12
|
+
fallbackEscalationBridgeFor,
|
|
10
13
|
frontDoorCapabilityDigest,
|
|
11
14
|
frontDoorDecisionRule,
|
|
12
15
|
HOLD_VERDICT_TOKEN,
|
|
@@ -162,6 +165,33 @@ describe("front-door decision rule", () => {
|
|
|
162
165
|
expect(rule.toLowerCase()).toContain("one short natural holding phrase");
|
|
163
166
|
expect(rule.toLowerCase()).toContain("stop after that single sentence");
|
|
164
167
|
});
|
|
168
|
+
|
|
169
|
+
test("hold completeness is judged in the caller's language", () => {
|
|
170
|
+
// Callers are not English-only: the hold branch's exemplars are English,
|
|
171
|
+
// so the rule must say completeness follows the grammar of the language
|
|
172
|
+
// being spoken, with the verb-final case called out (a missing final
|
|
173
|
+
// verb, not a missing conjunction, is the unfinished signal there).
|
|
174
|
+
const withHold = frontDoorDecisionRule({ includeHold: true });
|
|
175
|
+
expect(withHold.toLowerCase()).toContain("may speak any language");
|
|
176
|
+
expect(withHold.toLowerCase()).toContain(
|
|
177
|
+
"grammar of the language being spoken",
|
|
178
|
+
);
|
|
179
|
+
expect(withHold.toLowerCase()).toContain("verb-final");
|
|
180
|
+
expect(withHold.toLowerCase()).toContain("missing final verb");
|
|
181
|
+
});
|
|
182
|
+
|
|
183
|
+
test("the answer branch demands the caller's language", () => {
|
|
184
|
+
expect(rule).toContain("Answer in the language the caller is speaking.");
|
|
185
|
+
});
|
|
186
|
+
|
|
187
|
+
test("the escalation holding phrase demands the caller's language", () => {
|
|
188
|
+
// The bridge examples are English; without an explicit requirement a
|
|
189
|
+
// Spanish turn that needs a tool gets an English holding phrase before
|
|
190
|
+
// the localized answer. The examples stay, labeled as English only.
|
|
191
|
+
expect(rule).toContain("spoken in the language the caller is speaking");
|
|
192
|
+
expect(rule).toContain("those examples are English only");
|
|
193
|
+
expect(rule).toContain(`"${FALLBACK_ESCALATION_BRIDGE}"`);
|
|
194
|
+
});
|
|
165
195
|
});
|
|
166
196
|
|
|
167
197
|
describe("escalated continuation rule", () => {
|
|
@@ -190,6 +220,12 @@ describe("escalated continuation rule", () => {
|
|
|
190
220
|
);
|
|
191
221
|
});
|
|
192
222
|
|
|
223
|
+
test("demands the reply match the caller's language", () => {
|
|
224
|
+
expect(rule).toContain(
|
|
225
|
+
"Reply in the same language as the caller's question.",
|
|
226
|
+
);
|
|
227
|
+
});
|
|
228
|
+
|
|
193
229
|
test("bans re-announcing the holding phrase (bridge-echo regression)", () => {
|
|
194
230
|
// Regression: after the bridge "Let me check your calendar", the quality
|
|
195
231
|
// model opened with "Let me check what calendar connections…" — a
|
|
@@ -201,6 +237,41 @@ describe("escalated continuation rule", () => {
|
|
|
201
237
|
});
|
|
202
238
|
});
|
|
203
239
|
|
|
240
|
+
describe("fallbackEscalationBridgeFor", () => {
|
|
241
|
+
test("covers every Deepgram code-switching language with a non-empty bridge", () => {
|
|
242
|
+
for (const code of DEEPGRAM_MULTI_LANGUAGE_CODES) {
|
|
243
|
+
const bridge = FALLBACK_ESCALATION_BRIDGE_BY_LANGUAGE[code];
|
|
244
|
+
expect(bridge).toBeDefined();
|
|
245
|
+
expect(bridge!.trim().length).toBeGreaterThan(0);
|
|
246
|
+
expect(fallbackEscalationBridgeFor(code)).toBe(bridge!);
|
|
247
|
+
}
|
|
248
|
+
});
|
|
249
|
+
|
|
250
|
+
test("every bridge fits the session-side cap", () => {
|
|
251
|
+
for (const bridge of Object.values(
|
|
252
|
+
FALLBACK_ESCALATION_BRIDGE_BY_LANGUAGE,
|
|
253
|
+
)) {
|
|
254
|
+
expect(bridge.length).toBeLessThanOrEqual(MAX_ESCALATION_BRIDGE_CHARS);
|
|
255
|
+
}
|
|
256
|
+
});
|
|
257
|
+
|
|
258
|
+
test("selects by lowercased base subtag", () => {
|
|
259
|
+
expect(fallbackEscalationBridgeFor("pt-BR")).toBe(
|
|
260
|
+
FALLBACK_ESCALATION_BRIDGE_BY_LANGUAGE.pt!,
|
|
261
|
+
);
|
|
262
|
+
expect(fallbackEscalationBridgeFor("JA")).toBe(
|
|
263
|
+
FALLBACK_ESCALATION_BRIDGE_BY_LANGUAGE.ja!,
|
|
264
|
+
);
|
|
265
|
+
});
|
|
266
|
+
|
|
267
|
+
test("falls back to English for unknown or absent languages", () => {
|
|
268
|
+
expect(fallbackEscalationBridgeFor()).toBe(FALLBACK_ESCALATION_BRIDGE);
|
|
269
|
+
expect(fallbackEscalationBridgeFor("ko")).toBe(FALLBACK_ESCALATION_BRIDGE);
|
|
270
|
+
expect(fallbackEscalationBridgeFor("")).toBe(FALLBACK_ESCALATION_BRIDGE);
|
|
271
|
+
expect(fallbackEscalationBridgeFor("en")).toBe(FALLBACK_ESCALATION_BRIDGE);
|
|
272
|
+
});
|
|
273
|
+
});
|
|
274
|
+
|
|
204
275
|
describe("capEscalationBridge", () => {
|
|
205
276
|
test("cuts just after the first sentence terminator", () => {
|
|
206
277
|
expect(
|
|
@@ -218,6 +289,24 @@ describe("capEscalationBridge", () => {
|
|
|
218
289
|
test("strips internal markers before capping", () => {
|
|
219
290
|
expect(capEscalationBridge("[END_CALL] One moment.")).toBe("One moment.");
|
|
220
291
|
});
|
|
292
|
+
|
|
293
|
+
test("the Unicode ellipsis still terminates a bridge", () => {
|
|
294
|
+
// Regression pin: widening the terminator class for non-Latin enders
|
|
295
|
+
// must not drop the ellipsis the original regex recognized.
|
|
296
|
+
expect(capEscalationBridge("One moment… and some rambling")).toBe(
|
|
297
|
+
"One moment…",
|
|
298
|
+
);
|
|
299
|
+
expect(isEscalationBridgeComplete("One moment…")).toBe(true);
|
|
300
|
+
});
|
|
301
|
+
|
|
302
|
+
test("cuts just after a non-Latin sentence terminator", () => {
|
|
303
|
+
expect(capEscalationBridge("少し考えさせてください。その間の余談")).toBe(
|
|
304
|
+
"少し考えさせてください。",
|
|
305
|
+
);
|
|
306
|
+
expect(capEscalationBridge("मुझे एक पल सोचने दीजिए। और कुछ बातें")).toBe(
|
|
307
|
+
"मुझे एक पल सोचने दीजिए।",
|
|
308
|
+
);
|
|
309
|
+
});
|
|
221
310
|
});
|
|
222
311
|
|
|
223
312
|
describe("isEscalationBridgeComplete", () => {
|
|
@@ -233,6 +322,23 @@ describe("isEscalationBridgeComplete", () => {
|
|
|
233
322
|
isEscalationBridgeComplete("a".repeat(MAX_ESCALATION_BRIDGE_CHARS)),
|
|
234
323
|
).toBe(true);
|
|
235
324
|
});
|
|
325
|
+
|
|
326
|
+
test("a Japanese bridge ending in 。 completes without waiting for the cap", () => {
|
|
327
|
+
expect(isEscalationBridgeComplete("少し考えさせてください")).toBe(false);
|
|
328
|
+
expect(isEscalationBridgeComplete("少し考えさせてください。")).toBe(true);
|
|
329
|
+
});
|
|
330
|
+
|
|
331
|
+
test("every localized fallback bridge ends in a recognized terminator", () => {
|
|
332
|
+
// A model-spoken bridge in any roster language must hand off at its
|
|
333
|
+
// terminator, never by buffering to the char cap; the canned bridges are
|
|
334
|
+
// the canonical sample of each language's ender.
|
|
335
|
+
for (const bridge of Object.values(
|
|
336
|
+
FALLBACK_ESCALATION_BRIDGE_BY_LANGUAGE,
|
|
337
|
+
)) {
|
|
338
|
+
expect(isEscalationBridgeComplete(bridge)).toBe(true);
|
|
339
|
+
expect(capEscalationBridge(bridge)).toBe(bridge);
|
|
340
|
+
}
|
|
341
|
+
});
|
|
236
342
|
});
|
|
237
343
|
|
|
238
344
|
describe("spokenBridgeText", () => {
|
|
@@ -65,6 +65,10 @@ import {
|
|
|
65
65
|
resolveSynthesisFormats,
|
|
66
66
|
} from "./resolve-call-tts-provider.js";
|
|
67
67
|
import type { PromptSpeakerContext } from "./speaker-identification.js";
|
|
68
|
+
import {
|
|
69
|
+
resolveTelephonyLanguageVoice,
|
|
70
|
+
resolveTelephonySynthesisLanguage,
|
|
71
|
+
} from "./telephony-synthesis-language.js";
|
|
68
72
|
import { sanitizeForTts } from "./tts-text-sanitizer.js";
|
|
69
73
|
import {
|
|
70
74
|
ASK_GUARDIAN_CAPTURE_REGEX,
|
|
@@ -189,6 +193,13 @@ export class CallController {
|
|
|
189
193
|
private guardianUnavailableForCall = false;
|
|
190
194
|
/** Active synthesized-TTS session — tracked so interrupt handling can close it. */
|
|
191
195
|
private activeSynthesisAbort: AbortController | null = null;
|
|
196
|
+
/**
|
|
197
|
+
* Resolves the language hint for synthesized speech. The media-stream
|
|
198
|
+
* server supplies a resolver backed by the STT session's detected
|
|
199
|
+
* dominant language; the default falls back to the pin-based
|
|
200
|
+
* resolution only.
|
|
201
|
+
*/
|
|
202
|
+
private resolveSynthesisLanguage: () => string | undefined;
|
|
192
203
|
|
|
193
204
|
constructor(
|
|
194
205
|
callSessionId: string,
|
|
@@ -198,6 +209,7 @@ export class CallController {
|
|
|
198
209
|
broadcast?: (msg: AssistantEvent) => void;
|
|
199
210
|
assistantId?: string;
|
|
200
211
|
trustContext?: TrustContext;
|
|
212
|
+
resolveSynthesisLanguage?: () => string | undefined;
|
|
201
213
|
},
|
|
202
214
|
) {
|
|
203
215
|
this.callSessionId = callSessionId;
|
|
@@ -207,6 +219,9 @@ export class CallController {
|
|
|
207
219
|
this.broadcast = opts?.broadcast;
|
|
208
220
|
this.assistantId = opts?.assistantId ?? DAEMON_INTERNAL_ASSISTANT_ID;
|
|
209
221
|
this.trustContext = opts?.trustContext ?? null;
|
|
222
|
+
this.resolveSynthesisLanguage =
|
|
223
|
+
opts?.resolveSynthesisLanguage ??
|
|
224
|
+
(() => resolveTelephonySynthesisLanguage());
|
|
210
225
|
|
|
211
226
|
// Resolve the conversation ID and skipDisclosure from the call session
|
|
212
227
|
const session = getCallSession(callSessionId);
|
|
@@ -663,7 +678,9 @@ export class CallController {
|
|
|
663
678
|
// lock-hold wait budget, so surface a brief natural re-prompt (never a
|
|
664
679
|
// technical-error message) and re-arm listening. last=true doubles as
|
|
665
680
|
// the end-of-turn marker.
|
|
666
|
-
this.transport.sendTextToken("Sorry, could you say that again?", true
|
|
681
|
+
this.transport.sendTextToken("Sorry, could you say that again?", true, {
|
|
682
|
+
systemCopy: true,
|
|
683
|
+
});
|
|
667
684
|
this.state = "idle";
|
|
668
685
|
this.resetSilenceTimer();
|
|
669
686
|
this.flushPendingInstructions();
|
|
@@ -673,6 +690,7 @@ export class CallController {
|
|
|
673
690
|
this.transport.sendTextToken(
|
|
674
691
|
"I'm sorry, I encountered a technical issue. Could you repeat that?",
|
|
675
692
|
true,
|
|
693
|
+
{ systemCopy: true },
|
|
676
694
|
);
|
|
677
695
|
this.state = "idle";
|
|
678
696
|
this.resetSilenceTimer();
|
|
@@ -1080,11 +1098,17 @@ export class CallController {
|
|
|
1080
1098
|
|
|
1081
1099
|
this.activeSynthesisAbort = abortController;
|
|
1082
1100
|
|
|
1101
|
+
const language = this.resolveSynthesisLanguage();
|
|
1102
|
+
// A language-known segment may select the synthesizing provider's
|
|
1103
|
+
// configured per-language voice; no entry keeps the provider default.
|
|
1104
|
+
const voiceId = resolveTelephonyLanguageVoice(provider.id, language);
|
|
1083
1105
|
await synthesizeAndEmit({
|
|
1084
1106
|
provider,
|
|
1085
1107
|
text,
|
|
1086
1108
|
useCase: "phone-call",
|
|
1087
1109
|
outputFormat,
|
|
1110
|
+
...(voiceId !== undefined ? { voiceId } : {}),
|
|
1111
|
+
...(language !== undefined ? { language } : {}),
|
|
1088
1112
|
signal: abortController.signal,
|
|
1089
1113
|
isCurrent: () => this.isCurrentRun(runVersion),
|
|
1090
1114
|
onChunk: sink.onChunk,
|
|
@@ -1804,6 +1828,7 @@ export class CallController {
|
|
|
1804
1828
|
this.transport.sendTextToken(
|
|
1805
1829
|
"Just to let you know, we're running low on time for this call.",
|
|
1806
1830
|
true,
|
|
1831
|
+
{ systemCopy: true },
|
|
1807
1832
|
);
|
|
1808
1833
|
}, warningMs);
|
|
1809
1834
|
}
|
|
@@ -1816,6 +1841,7 @@ export class CallController {
|
|
|
1816
1841
|
this.transport.sendTextToken(
|
|
1817
1842
|
"I'm sorry, but we've reached the maximum time for this call. Thank you for your time. Goodbye!",
|
|
1818
1843
|
true,
|
|
1844
|
+
{ systemCopy: true },
|
|
1819
1845
|
);
|
|
1820
1846
|
// Give TTS a moment to play, then end
|
|
1821
1847
|
this.durationEndTimer = setTimeout(() => {
|
|
@@ -1878,7 +1904,9 @@ export class CallController {
|
|
|
1878
1904
|
{ callSessionId: this.callSessionId },
|
|
1879
1905
|
"Silence timeout triggered",
|
|
1880
1906
|
);
|
|
1881
|
-
this.transport.sendTextToken("Are you still there?", true
|
|
1907
|
+
this.transport.sendTextToken("Are you still there?", true, {
|
|
1908
|
+
systemCopy: true,
|
|
1909
|
+
});
|
|
1882
1910
|
}, getSilenceTimeoutMs());
|
|
1883
1911
|
}
|
|
1884
1912
|
}
|
|
@@ -68,8 +68,10 @@ export async function speakSystemPrompt(
|
|
|
68
68
|
});
|
|
69
69
|
|
|
70
70
|
if (!useSynthesizedPath || !provider) {
|
|
71
|
-
// Native path
|
|
72
|
-
|
|
71
|
+
// Native path: send text tokens through the transport. Marked as
|
|
72
|
+
// system copy so transports that re-synthesize tokens (media-stream)
|
|
73
|
+
// never attach the caller-language hint to these fixed English prompts.
|
|
74
|
+
relay.sendTextToken(text, true, { systemCopy: true });
|
|
73
75
|
return;
|
|
74
76
|
}
|
|
75
77
|
|
|
@@ -128,6 +130,11 @@ async function synthesizeAndPlay(
|
|
|
128
130
|
},
|
|
129
131
|
});
|
|
130
132
|
|
|
133
|
+
// No language hint: this path speaks fixed English copy (verification
|
|
134
|
+
// codes, guardian prompts), so a pin-based hint would tell an
|
|
135
|
+
// enforcing provider to render English text in the pinned language.
|
|
136
|
+
// Conversational synthesis (call-controller, media-stream-output)
|
|
137
|
+
// carries the hint instead.
|
|
131
138
|
const result = await synthesizeAndEmit({
|
|
132
139
|
provider,
|
|
133
140
|
text,
|
|
@@ -258,8 +265,9 @@ async function synthesizeAndPlay(
|
|
|
258
265
|
// Fallback: send text via native TTS so the caller still hears the message.
|
|
259
266
|
// sendTextToken with last:true includes the end-of-turn signal inherently.
|
|
260
267
|
// This fallback is only used for providers whose catalog entry allows
|
|
261
|
-
// native fallback.
|
|
262
|
-
|
|
268
|
+
// native fallback. Marked as system copy for the same reason as the
|
|
269
|
+
// native path above.
|
|
270
|
+
relay.sendTextToken(text, true, { systemCopy: true });
|
|
263
271
|
} finally {
|
|
264
272
|
sink?.finalize();
|
|
265
273
|
}
|
|
@@ -9,6 +9,19 @@
|
|
|
9
9
|
|
|
10
10
|
// ── Transport interface ──────────────────────────────────────────────
|
|
11
11
|
|
|
12
|
+
/** Options for {@link CallTransport.sendTextToken}. */
|
|
13
|
+
export interface SendTextTokenOptions {
|
|
14
|
+
/**
|
|
15
|
+
* The token is fixed, known-English system copy (error recovery, silence
|
|
16
|
+
* checks, duration warnings, deterministic prompts) rather than
|
|
17
|
+
* model-generated turn text. Transports that synthesize text themselves
|
|
18
|
+
* must not attach the caller-language hint to these tokens: a provider
|
|
19
|
+
* that enforces the hint would render the English words as though they
|
|
20
|
+
* were the caller's language. Model text keeps the hint.
|
|
21
|
+
*/
|
|
22
|
+
systemCopy?: boolean;
|
|
23
|
+
}
|
|
24
|
+
|
|
12
25
|
/**
|
|
13
26
|
* Minimal output surface that CallController uses to send speech,
|
|
14
27
|
* audio, and lifecycle signals to the caller.
|
|
@@ -18,7 +31,11 @@ export interface CallTransport {
|
|
|
18
31
|
* Send a text token for TTS playback. When `last` is true the
|
|
19
32
|
* transport should signal end-of-turn to the caller.
|
|
20
33
|
*/
|
|
21
|
-
sendTextToken(
|
|
34
|
+
sendTextToken(
|
|
35
|
+
token: string,
|
|
36
|
+
last: boolean,
|
|
37
|
+
opts?: SendTextTokenOptions,
|
|
38
|
+
): void;
|
|
22
39
|
|
|
23
40
|
/**
|
|
24
41
|
* Send a pre-synthesized audio URL for playback.
|
|
@@ -41,7 +41,7 @@ import { extractSpeakableSegments } from "../tts/speakable-segments.js";
|
|
|
41
41
|
import { synthesizeAndEmit } from "../tts/synthesis-stream.js";
|
|
42
42
|
import { getLogger } from "../util/logger.js";
|
|
43
43
|
import type { CallAudioFormat } from "./audio-store.js";
|
|
44
|
-
import type { CallTransport } from "./call-transport.js";
|
|
44
|
+
import type { CallTransport, SendTextTokenOptions } from "./call-transport.js";
|
|
45
45
|
import {
|
|
46
46
|
chunkMulawToBase64Frames,
|
|
47
47
|
MULAW_FRAME_SIZE,
|
|
@@ -54,6 +54,10 @@ import type {
|
|
|
54
54
|
MediaStreamSendMediaCommand,
|
|
55
55
|
} from "./media-stream-protocol.js";
|
|
56
56
|
import { resolveCallTtsProvider } from "./resolve-call-tts-provider.js";
|
|
57
|
+
import {
|
|
58
|
+
resolveTelephonyLanguageVoice,
|
|
59
|
+
resolveTelephonySynthesisLanguage,
|
|
60
|
+
} from "./telephony-synthesis-language.js";
|
|
57
61
|
|
|
58
62
|
const log = getLogger("media-stream-output");
|
|
59
63
|
|
|
@@ -289,7 +293,7 @@ export type MediaStreamOutputState = "connected" | "closed";
|
|
|
289
293
|
*/
|
|
290
294
|
type PlaybackItem =
|
|
291
295
|
| { type: "frames"; frames: string[] }
|
|
292
|
-
| { type: "synthesize"; text: string }
|
|
296
|
+
| { type: "synthesize"; text: string; systemCopy: boolean }
|
|
293
297
|
| { type: "fetch-url"; url: string }
|
|
294
298
|
| { type: "mark"; name: string };
|
|
295
299
|
|
|
@@ -338,6 +342,15 @@ export class MediaStreamOutput implements CallTransport {
|
|
|
338
342
|
*/
|
|
339
343
|
private audioStartCallback: (() => void) | null = null;
|
|
340
344
|
|
|
345
|
+
/**
|
|
346
|
+
* Resolves the language hint passed on synthesis requests. Defaults to
|
|
347
|
+
* the pin-based resolution; the media-stream server overrides it with
|
|
348
|
+
* a resolver that consults the STT session's detected dominant
|
|
349
|
+
* language.
|
|
350
|
+
*/
|
|
351
|
+
private resolveSynthesisLanguage: () => string | undefined = () =>
|
|
352
|
+
resolveTelephonySynthesisLanguage();
|
|
353
|
+
|
|
341
354
|
/** Incremented per end-of-turn mark enqueued. */
|
|
342
355
|
private enqueuedEndOfTurnSeq = 0;
|
|
343
356
|
|
|
@@ -373,8 +386,16 @@ export class MediaStreamOutput implements CallTransport {
|
|
|
373
386
|
* An empty token with `last: true` signals end-of-turn without TTS:
|
|
374
387
|
* a mark is sent so the session transitions from "assistant speaking"
|
|
375
388
|
* to "caller speaking".
|
|
389
|
+
*
|
|
390
|
+
* `opts.systemCopy` marks fixed English system copy: its segments
|
|
391
|
+
* synthesize without the caller-language hint (see
|
|
392
|
+
* {@link processSynthesizeItem}).
|
|
376
393
|
*/
|
|
377
|
-
sendTextToken(
|
|
394
|
+
sendTextToken(
|
|
395
|
+
token: string,
|
|
396
|
+
last: boolean,
|
|
397
|
+
opts?: SendTextTokenOptions,
|
|
398
|
+
): void {
|
|
378
399
|
if (this.state === "closed") {
|
|
379
400
|
return;
|
|
380
401
|
}
|
|
@@ -388,7 +409,11 @@ export class MediaStreamOutput implements CallTransport {
|
|
|
388
409
|
);
|
|
389
410
|
this.textBuffer = remainder;
|
|
390
411
|
for (const segment of segments) {
|
|
391
|
-
this.enqueuePlayback({
|
|
412
|
+
this.enqueuePlayback({
|
|
413
|
+
type: "synthesize",
|
|
414
|
+
text: segment,
|
|
415
|
+
systemCopy: opts?.systemCopy === true,
|
|
416
|
+
});
|
|
392
417
|
this.turnSegmentEnqueued = true;
|
|
393
418
|
}
|
|
394
419
|
|
|
@@ -427,6 +452,14 @@ export class MediaStreamOutput implements CallTransport {
|
|
|
427
452
|
this.audioStartCallback = cb;
|
|
428
453
|
}
|
|
429
454
|
|
|
455
|
+
/**
|
|
456
|
+
* Override the synthesis-language resolver (see
|
|
457
|
+
* {@link resolveSynthesisLanguage}).
|
|
458
|
+
*/
|
|
459
|
+
setSynthesisLanguageResolver(resolver: () => string | undefined): void {
|
|
460
|
+
this.resolveSynthesisLanguage = resolver;
|
|
461
|
+
}
|
|
462
|
+
|
|
430
463
|
/**
|
|
431
464
|
* Discard accumulated text that has not yet been queued for synthesis.
|
|
432
465
|
* The call controller invokes this when it aborts an in-flight turn so
|
|
@@ -778,7 +811,9 @@ export class MediaStreamOutput implements CallTransport {
|
|
|
778
811
|
break;
|
|
779
812
|
|
|
780
813
|
case "synthesize":
|
|
781
|
-
await this.processSynthesizeItem(item.text, version
|
|
814
|
+
await this.processSynthesizeItem(item.text, version, {
|
|
815
|
+
systemCopy: item.systemCopy,
|
|
816
|
+
});
|
|
782
817
|
break;
|
|
783
818
|
|
|
784
819
|
case "fetch-url":
|
|
@@ -827,10 +862,17 @@ export class MediaStreamOutput implements CallTransport {
|
|
|
827
862
|
* each streamed chunk becomes frames as it arrives — while other
|
|
828
863
|
* providers accumulate into the whole-buffer conversion path. Falls
|
|
829
864
|
* back to a silent frame if synthesis fails.
|
|
865
|
+
*
|
|
866
|
+
* `systemCopy` items are fixed English copy: they synthesize without a
|
|
867
|
+
* language hint (and therefore without a per-language voice override),
|
|
868
|
+
* so an enforcing provider never renders English text in the caller's
|
|
869
|
+
* language (same exemption as call-speech-output's synthesized path).
|
|
870
|
+
* Model text keeps the resolver's hint.
|
|
830
871
|
*/
|
|
831
872
|
private async processSynthesizeItem(
|
|
832
873
|
text: string,
|
|
833
874
|
version: number,
|
|
875
|
+
{ systemCopy }: { systemCopy: boolean },
|
|
834
876
|
): Promise<void> {
|
|
835
877
|
const abortController = new AbortController();
|
|
836
878
|
this.activePlaybackAbort = abortController;
|
|
@@ -876,12 +918,18 @@ export class MediaStreamOutput implements CallTransport {
|
|
|
876
918
|
// Providers that support it (e.g. ElevenLabs pcm_16000) will
|
|
877
919
|
// return raw PCM; others fall back to their default format and
|
|
878
920
|
// the content-type sniffing below handles the mismatch.
|
|
921
|
+
const language = systemCopy ? undefined : this.resolveSynthesisLanguage();
|
|
922
|
+
// A language-known segment may select the synthesizing provider's
|
|
923
|
+
// configured per-language voice; no entry keeps the provider default.
|
|
924
|
+
const voiceId = resolveTelephonyLanguageVoice(provider.id, language);
|
|
879
925
|
const result = await synthesizeAndEmit({
|
|
880
926
|
provider,
|
|
881
927
|
text,
|
|
882
928
|
useCase: "phone-call",
|
|
883
929
|
outputFormat: "pcm",
|
|
884
930
|
sampleRateHz: STREAMING_PCM_SAMPLE_RATE_HZ,
|
|
931
|
+
...(voiceId !== undefined ? { voiceId } : {}),
|
|
932
|
+
...(language !== undefined ? { language } : {}),
|
|
885
933
|
signal: abortController.signal,
|
|
886
934
|
isCurrent,
|
|
887
935
|
onChunk: (chunk) => {
|