@vellumai/assistant 0.11.2 → 0.11.3-dev.202608080427.92f668a
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +69 -9
- package/docs/flux-turn-detection-spike.md +243 -0
- package/docs/stt-provider-onboarding.md +3 -1
- package/docs/trusted-contact-access.md +9 -5
- package/docs/vellum-doctor.md +195 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/channels.ts +50 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/ingress.ts +10 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/remote-web-pairing.ts +6 -0
- package/node_modules/@vellumai/ces-client/src/index.ts +1 -0
- package/node_modules/@vellumai/ces-client/src/rpc-client.ts +19 -2
- package/node_modules/@vellumai/environments/package.json +2 -1
- package/node_modules/@vellumai/environments/src/__tests__/cloud-assistant-hub-url.test.ts +38 -0
- package/node_modules/@vellumai/environments/src/__tests__/install-layout.test.ts +98 -0
- package/node_modules/@vellumai/environments/src/__tests__/package-boundary.test.ts +5 -5
- package/node_modules/@vellumai/environments/src/index.ts +17 -6
- package/node_modules/@vellumai/environments/src/install-layout.ts +49 -0
- package/node_modules/@vellumai/environments/src/seeds.ts +29 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/channels.ts +50 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/ingress.ts +10 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/remote-web-pairing.ts +6 -0
- package/node_modules/@vellumai/gateway-client/src/index.ts +4 -0
- package/node_modules/@vellumai/gateway-client/src/verification-session-contract.ts +91 -6
- package/node_modules/@vellumai/ipc-server-utils/src/endpoint.test.ts +36 -0
- package/node_modules/@vellumai/ipc-server-utils/src/endpoint.ts +142 -0
- package/node_modules/@vellumai/ipc-server-utils/src/index.ts +2 -0
- package/node_modules/@vellumai/ipc-server-utils/src/listen-options.ts +3 -0
- package/node_modules/@vellumai/ipc-server-utils/src/socket-watchdog.test.ts +15 -1
- package/node_modules/@vellumai/ipc-server-utils/src/socket-watchdog.ts +17 -2
- package/node_modules/@vellumai/service-contracts/src/channels.ts +50 -0
- package/node_modules/@vellumai/service-contracts/src/ingress.ts +10 -0
- package/node_modules/@vellumai/service-contracts/src/remote-web-pairing.ts +6 -0
- package/node_modules/@vellumai/slack-text/src/index.ts +16 -2
- package/openapi.yaml +619 -47
- package/package.json +1 -1
- package/scripts/voice-ttft-spike.ts +3 -3
- package/src/__tests__/agent-image-optimize.test.ts +4 -4
- package/src/__tests__/agent-loop-mutable-latest-user-message.test.ts +113 -66
- package/src/__tests__/agent-loop-override-profile.test.ts +38 -48
- package/src/__tests__/agent-loop-subagent-billing-headers.test.ts +166 -0
- package/src/__tests__/agent-wake-disk-pressure-callsite.test.ts +1 -0
- package/src/__tests__/agent-wake-override-profile.test.ts +1 -0
- package/src/__tests__/answered-question-persistence.test.ts +240 -0
- package/src/__tests__/anthropic-provider.test.ts +165 -30
- package/src/__tests__/app-control-flow.test.ts +9 -8
- package/src/__tests__/assistant-event-hub.test.ts +48 -0
- package/src/__tests__/attachments-store-heic-normalize.test.ts +16 -12
- package/src/__tests__/attachments-store.test.ts +130 -68
- package/src/__tests__/attachments.test.ts +36 -36
- package/src/__tests__/background-workers-disk-pressure.test.ts +1 -1
- package/src/__tests__/call-controller.test.ts +120 -0
- package/src/__tests__/call-start-guardian-guard.test.ts +18 -4
- package/src/__tests__/canned-reply-release.test.ts +86 -0
- package/src/__tests__/channel-availability-routes.test.ts +14 -1
- package/src/__tests__/channel-guardian.test.ts +59 -29
- package/src/__tests__/channel-readiness-telegram-remote.test.ts +154 -0
- package/src/__tests__/channel-setup-panel-ack.test.ts +6 -5
- package/src/__tests__/client-os-metadata-persistence.test.ts +103 -4
- package/src/__tests__/compactor-image-manifest-trust.test.ts +1 -1
- package/src/__tests__/config-loader-backfill.test.ts +19 -0
- package/src/__tests__/config-schema.test.ts +150 -7
- package/src/__tests__/consult-deadline.test.ts +65 -0
- package/src/__tests__/conversation-agent-loop.test.ts +438 -2
- package/src/__tests__/conversation-disk-view-integration.test.ts +5 -1
- package/src/__tests__/conversation-disk-view.test.ts +4 -4
- package/src/__tests__/conversation-error.test.ts +22 -3
- package/src/__tests__/conversation-fork-crud.test.ts +5 -1
- package/src/__tests__/conversation-fork-referential.test.ts +352 -0
- package/src/__tests__/conversation-fork-retrospective.test.ts +5 -1
- package/src/__tests__/conversation-group-tools.test.ts +32 -2
- package/src/__tests__/conversation-inference-profile-route.test.ts +27 -1
- package/src/__tests__/conversation-initial-prompt.test.ts +51 -22
- package/src/__tests__/conversation-key-store-bootstrap-cleanup.test.ts +119 -0
- package/src/__tests__/conversation-key-store-origin.test.ts +105 -0
- package/src/__tests__/conversation-list-recency-ordering.test.ts +104 -0
- package/src/__tests__/conversation-origin-at-creation.test.ts +153 -0
- package/src/__tests__/conversation-origin-channel-filter.test.ts +95 -0
- package/src/__tests__/conversation-placement-promotion.test.ts +172 -0
- package/src/__tests__/conversation-process-app-control-preactivation.test.ts +123 -0
- package/src/__tests__/conversation-queue.test.ts +433 -1
- package/src/__tests__/conversation-retry-route.test.ts +3 -1
- package/src/__tests__/conversation-routes-hidden-queue.test.ts +64 -1
- package/src/__tests__/conversation-store-ephemeral.test.ts +4 -0
- package/src/__tests__/conversation-store.test.ts +26 -10
- package/src/__tests__/conversation-summarize-route.test.ts +1 -1
- package/src/__tests__/conversation-surfaces-action-delivery.test.ts +15 -6
- package/src/__tests__/conversation-surfaces-activation-emit.test.ts +6 -5
- package/src/__tests__/conversation-surfaces-app-control.test.ts +12 -11
- package/src/__tests__/conversation-surfaces-app-open.test.ts +11 -5
- package/src/__tests__/conversation-surfaces-data-persist.test.ts +5 -4
- package/src/__tests__/conversation-surfaces-history-restored-completion.test.ts +967 -0
- package/src/__tests__/conversation-surfaces-persisted-info.test.ts +312 -0
- package/src/__tests__/conversation-surfaces-queued-emit.test.ts +6 -5
- package/src/__tests__/conversation-surfaces-standalone-payloads.test.ts +12 -8
- package/src/__tests__/conversation-surfaces-standalone.test.ts +21 -11
- package/src/__tests__/conversation-surfaces-state-update.test.ts +7 -11
- package/src/__tests__/conversation-surfaces-table-action.test.ts +7 -6
- package/src/__tests__/conversation-surfaces-task-progress.test.ts +11 -6
- package/src/__tests__/conversation-tool-setup-app-refresh.test.ts +9 -6
- package/src/__tests__/conversation-tool-setup-attribution.test.ts +70 -7
- package/src/__tests__/conversation-wait-for-idle.test.ts +6 -11
- package/src/__tests__/credential-prompt-route.test.ts +54 -7
- package/src/__tests__/credential-security-invariants.test.ts +1 -0
- package/src/__tests__/cu-unified-flow.test.ts +89 -28
- package/src/__tests__/custom-group-listing-visibility.test.ts +1 -1
- package/src/__tests__/db-schedule-syntax-migration.test.ts +4 -1
- package/src/__tests__/discord-access-request-privacy.test.ts +366 -0
- package/src/__tests__/discord-requester-notice-privacy.test.ts +87 -0
- package/src/__tests__/disk-pressure-guard.test.ts +55 -55
- package/src/__tests__/disk-pressure-routes.test.ts +10 -10
- package/src/__tests__/disk-pressure-tools.test.ts +2 -2
- package/src/__tests__/disk-usage.test.ts +35 -20
- package/src/__tests__/document-sync-tags.test.ts +379 -0
- package/src/__tests__/document-update-default-surface.test.ts +28 -0
- package/src/__tests__/dynamic-skill-background-guard.test.ts +2 -2
- package/src/__tests__/emit-signal-routing-intent.test.ts +3 -4
- package/src/__tests__/ensure-conversation-exists.test.ts +8 -5
- package/src/__tests__/events-dev-bypass-actor.test.ts +294 -1
- package/src/__tests__/evict-conversations-for-reload.test.ts +90 -0
- package/src/__tests__/file-list-tool.test.ts +12 -10
- package/src/__tests__/file-ops-service.test.ts +151 -44
- package/src/__tests__/file-read-container-boundary.test.ts +137 -0
- package/src/__tests__/file-read-tool.test.ts +21 -9
- package/src/__tests__/file-url-path-guard.test.ts +83 -0
- package/src/__tests__/filesystem-tools.test.ts +82 -66
- package/src/__tests__/guardian-outbound-http.test.ts +8 -8
- package/src/__tests__/helpers/gateway-verification-sessions-stub.ts +14 -1
- package/src/__tests__/helpers/hub-clients.ts +42 -0
- package/src/__tests__/helpers/mock-conversation.ts +46 -0
- package/src/__tests__/helpers/verification-sessions-ipc-sim.ts +61 -57
- package/src/__tests__/history-repair.test.ts +112 -12
- package/src/__tests__/host-bash-proxy.test.ts +10 -5
- package/src/__tests__/host-bash-routes.test.ts +67 -0
- package/src/__tests__/host-browser-proxy.test.ts +2 -2
- package/src/__tests__/host-cu-proxy.test.ts +3 -7
- package/src/__tests__/host-file-proxy.test.ts +3 -7
- package/src/__tests__/host-file-read-tool.test.ts +5 -1
- package/src/__tests__/host-transfer-proxy-targeted.test.ts +6 -4
- package/src/__tests__/http-conversation-lineage.test.ts +2 -5
- package/src/__tests__/http-user-message-parity.test.ts +44 -0
- package/src/__tests__/identity-routes.test.ts +13 -13
- package/src/__tests__/image-conversion.test.ts +29 -26
- package/src/__tests__/image-source-path-reinject.test.ts +2 -2
- package/src/__tests__/inference-profile-session-handler.test.ts +95 -0
- package/src/__tests__/inference-profile-session-ipc.test.ts +25 -0
- package/src/__tests__/introduction-card-resolver.test.ts +6 -1
- package/src/__tests__/list-messages-attachments.test.ts +56 -32
- package/src/__tests__/list-messages-background-tool-completion.test.ts +4 -4
- package/src/__tests__/list-messages-client-message-id.test.ts +2 -2
- package/src/__tests__/list-messages-hidden-metadata.test.ts +16 -16
- package/src/__tests__/list-messages-page-latest.test.ts +61 -61
- package/src/__tests__/list-messages-provider-error.test.ts +6 -6
- package/src/__tests__/list-messages-queued.test.ts +49 -6
- package/src/__tests__/list-messages-system-card.test.ts +4 -4
- package/src/__tests__/list-messages-tool-merge.test.ts +9 -9
- package/src/__tests__/llm-resolver-override-or-default.test.ts +141 -0
- package/src/__tests__/llm-usage-store.test.ts +55 -0
- package/src/__tests__/manual-token-reconciliation.test.ts +45 -0
- package/src/__tests__/mcp-list-plugin-servers.test.ts +250 -0
- package/src/__tests__/mcp-tool-annotations-risk.test.ts +133 -0
- package/src/__tests__/media-stream-output.test.ts +175 -0
- package/src/__tests__/media-stream-server-integration.test.ts +36 -0
- package/src/__tests__/media-stream-stt-session.test.ts +67 -0
- package/src/__tests__/migration-export-http.test.ts +1 -1
- package/src/__tests__/migration-import-commit-http.test.ts +1 -1
- package/src/__tests__/migration-import-preflight-http.test.ts +1 -1
- package/src/__tests__/migration-validate-http.test.ts +1 -1
- package/src/__tests__/mtime-cache.test.ts +65 -3
- package/src/__tests__/notification-platform-adapter.test.ts +104 -0
- package/src/__tests__/notification-schedule-notify-dedup.test.ts +1 -1
- package/src/__tests__/oauth-provider-profiles.test.ts +4 -2
- package/src/__tests__/oauth-provider-seed-logos.test.ts +65 -0
- package/src/__tests__/oauth-provider-serializer.test.ts +17 -0
- package/src/__tests__/oauth-providers-routes.test.ts +1 -0
- package/src/__tests__/openai-responses-prompt-cache.test.ts +5 -2
- package/src/__tests__/path-policy.test.ts +164 -1
- package/src/__tests__/persist-media-references.test.ts +2 -2
- package/src/__tests__/persist-unsendable-image-downscale.test.ts +5 -5
- package/src/__tests__/persist-unsendable-image.test.ts +23 -23
- package/src/__tests__/persona-resolver.test.ts +154 -0
- package/src/__tests__/plugin-api-webhook-url.test.ts +170 -0
- package/src/__tests__/plugin-import-boundary-guard.test.ts +6 -0
- package/src/__tests__/processing-flag-persist-failure.test.ts +15 -10
- package/src/__tests__/profile-availability-incomplete.test.ts +113 -0
- package/src/__tests__/profiler-routes.test.ts +13 -13
- package/src/__tests__/prompt-cache-cross-turn-stability.test.ts +593 -0
- package/src/__tests__/proxy-approval-callback.test.ts +3 -3
- package/src/__tests__/queued-message-cancel-and-steer.test.ts +189 -0
- package/src/__tests__/queued-message-steer-actor-scoping.test.ts +164 -0
- package/src/__tests__/resolve-trust-class.test.ts +20 -0
- package/src/__tests__/run-conversation-turn-persistence.test.ts +1 -1
- package/src/__tests__/run-due-schedules.test.ts +174 -0
- package/src/__tests__/runtime-attachment-metadata.test.ts +15 -7
- package/src/__tests__/scaffold-managed-skill-tool.test.ts +166 -0
- package/src/__tests__/schedule-routes.test.ts +730 -8
- package/src/__tests__/schedule-store.test.ts +540 -0
- package/src/__tests__/schedule-tools.test.ts +103 -0
- package/src/__tests__/scheduler-wake.test.ts +5 -0
- package/src/__tests__/secret-routes-platform-proxy.test.ts +29 -1
- package/src/__tests__/secure-keys.test.ts +91 -0
- package/src/__tests__/server-history-render.test.ts +97 -7
- package/src/__tests__/size-guard.test.ts +8 -8
- package/src/__tests__/skill-docs-sync-guard.test.ts +3 -2
- package/src/__tests__/skill-tool-factory.test.ts +247 -0
- package/src/__tests__/skills-install-staging.test.ts +15 -6
- package/src/__tests__/skills.test.ts +32 -0
- package/src/__tests__/slack-edit-ordering-characterization.test.ts +103 -0
- package/src/__tests__/slack-inbound-verification.test.ts +4 -1
- package/src/__tests__/slack-mention-binding-invariance.test.ts +103 -0
- package/src/__tests__/slack-mention-provider-leak-audit.test.ts +135 -0
- package/src/__tests__/starter-task-flow.test.ts +5 -4
- package/src/__tests__/steer-on-enqueue-question.test.ts +3 -2
- package/src/__tests__/stt-language-catalog-parity.test.ts +38 -0
- package/src/__tests__/subagent-allowlist-validation.test.ts +26 -4
- package/src/__tests__/subagent-cron-run-attribution.test.ts +301 -0
- package/src/__tests__/subagent-disposal.test.ts +10 -0
- package/src/__tests__/subagent-fork-notifications.test.ts +10 -0
- package/src/__tests__/subagent-fork-prompt-role.test.ts +85 -11
- package/src/__tests__/subagent-fork-spawn.test.ts +15 -7
- package/src/__tests__/subagent-manager-notify.test.ts +148 -0
- package/src/__tests__/subagent-notify-parent.test.ts +1 -1
- package/src/__tests__/subagent-persistence.test.ts +141 -0
- package/src/__tests__/subagent-role-registry.test.ts +172 -63
- package/src/__tests__/subagent-role-resolution.test.ts +88 -0
- package/src/__tests__/subagent-spawn-and-await.test.ts +48 -0
- package/src/__tests__/subagent-spawn-tool-fork.test.ts +111 -10
- package/src/__tests__/subagent-terminal-message.test.ts +88 -1
- package/src/__tests__/subagent-tool-gate-mode.test.ts +225 -10
- package/src/__tests__/subagent-tools.test.ts +1901 -108
- package/src/__tests__/surface-completion-compaction-boundary.test.ts +124 -0
- package/src/__tests__/surface-completion-in-flight-snapshot.test.ts +298 -0
- package/src/__tests__/system-prompt.test.ts +82 -7
- package/src/__tests__/tool-grant-request-escalation.test.ts +40 -1
- package/src/__tests__/tool-result-spool.test.ts +74 -18
- package/src/__tests__/tool-side-effects-documents.test.ts +174 -0
- package/src/__tests__/tool-side-effects-hook-failures.test.ts +129 -0
- package/src/__tests__/tools-audio-read.test.ts +10 -10
- package/src/__tests__/tools-get-route.test.ts +85 -12
- package/src/__tests__/tools-list-cli.test.ts +171 -0
- package/src/__tests__/top-level-renderer.test.ts +54 -1
- package/src/__tests__/trusted-contact-inline-approval-integration.test.ts +68 -1
- package/src/__tests__/trusted-contact-multichannel.test.ts +4 -1
- package/src/__tests__/turn-tail-chain.test.ts +135 -0
- package/src/__tests__/ui-choice-copy-surfaces.test.ts +6 -4
- package/src/__tests__/ui-visual-surface.test.ts +5 -4
- package/src/__tests__/ui-work-result-surface.test.ts +5 -4
- package/src/__tests__/unified-turn-context-visible-app.test.ts +7 -8
- package/src/__tests__/usage-routes.test.ts +62 -0
- package/src/__tests__/verification-outbound-delivery.test.ts +176 -0
- package/src/__tests__/visible-app-context.test.ts +20 -4
- package/src/__tests__/web-search-history.test.ts +96 -0
- package/src/__tests__/workspace-migration-139-clear-renamed-cost-profile-label.test.ts +6 -6
- package/src/__tests__/workspace-migration-140-repair-seed-pinned-memory-v3-live.test.ts +330 -0
- package/src/__tests__/workspace-migration-142-consolidate-voice-front-door.test.ts +150 -0
- package/src/acp/failure-error.ts +3 -5
- package/src/agent/attachments.ts +26 -24
- package/src/agent/history-repair/history-repair.ts +27 -26
- package/src/agent/image-optimize.ts +6 -6
- package/src/agent/loop.ts +40 -22
- package/src/agent/message-types.ts +3 -3
- package/src/api/constants/document-tools.ts +55 -0
- package/src/api/events/question-answered.ts +53 -0
- package/src/api/events/tool-result.ts +7 -0
- package/src/api/index.ts +13 -0
- package/src/api/responses/conversation-message.ts +11 -0
- package/src/api/surfaces-oauth-connect.test.ts +96 -0
- package/src/api/surfaces.ts +36 -8
- package/src/approvals/guardian-channel-delivery.ts +83 -0
- package/src/approvals/guardian-expiry-notifier.ts +13 -11
- package/src/approvals/guardian-request-resolvers.ts +222 -135
- package/src/calls/__tests__/tts-text-sanitizer.test.ts +13 -0
- package/src/calls/__tests__/voice-session-bridge.test.ts +131 -12
- package/src/calls/__tests__/voice-triage-escalate.test.ts +106 -0
- package/src/calls/call-controller.ts +30 -2
- package/src/calls/call-speech-output.ts +12 -4
- package/src/calls/call-transport.ts +18 -1
- package/src/calls/media-stream-output.ts +53 -5
- package/src/calls/media-stream-server.ts +21 -1
- package/src/calls/media-stream-stt-session.ts +49 -0
- package/src/calls/telephony-synthesis-language.ts +84 -0
- package/src/calls/tts-text-sanitizer.ts +12 -5
- package/src/calls/voice-session-bridge.ts +86 -17
- package/src/calls/voice-triage-escalate.ts +54 -7
- package/src/channels/__tests__/gateway-verification-sessions.test.ts +3 -3
- package/src/channels/__tests__/plugin-channel-declarations.test.ts +161 -0
- package/src/channels/config.ts +20 -2
- package/src/channels/gateway-verification-sessions.ts +7 -2
- package/src/channels/plugin-channel-declarations.ts +108 -0
- package/src/channels/types.ts +37 -1
- package/src/cli/AGENTS.md +5 -2
- package/src/cli/commands/__tests__/cache.test.ts +7 -4
- package/src/cli/commands/__tests__/cli-test-harness.ts +82 -0
- package/src/cli/commands/__tests__/conversations-wake.test.ts +51 -6
- package/src/cli/commands/__tests__/inference-models.test.ts +14 -0
- package/src/cli/commands/__tests__/inference-profiles.test.ts +44 -1
- package/src/cli/commands/__tests__/inference-providers.test.ts +7 -0
- package/src/cli/commands/__tests__/plugins.test.ts +692 -0
- package/src/cli/commands/__tests__/schedules.test.ts +136 -37
- package/src/cli/commands/conversations.ts +7 -3
- package/src/cli/commands/credentials.help.ts +7 -0
- package/src/cli/commands/inference-profiles.ts +9 -3
- package/src/cli/commands/inference.help.ts +16 -2
- package/src/cli/commands/mcp.help.ts +17 -6
- package/src/cli/commands/mcp.ts +9 -0
- package/src/cli/commands/memory/memory-retrospective.ts +18 -0
- package/src/cli/commands/notifications.ts +3 -6
- package/src/cli/commands/oauth/connect-surface-guidance.test.ts +16 -0
- package/src/cli/commands/oauth/connect-surface-guidance.ts +25 -7
- package/src/cli/commands/oauth/connect.test.ts +26 -1
- package/src/cli/commands/oauth/connect.ts +4 -1
- package/src/cli/commands/oauth/index.help.ts +3 -1
- package/src/cli/commands/oauth/providers.ts +3 -0
- package/src/cli/commands/plugins.help.ts +25 -2
- package/src/cli/commands/plugins.ts +289 -10
- package/src/cli/commands/schedules.help.ts +14 -12
- package/src/cli/commands/schedules.ts +19 -5
- package/src/cli/commands/stt.help.ts +27 -2
- package/src/cli/commands/tools.help.ts +4 -2
- package/src/cli/commands/tools.ts +38 -0
- package/src/cli/lib/__tests__/inspect-plugin.test.ts +1 -0
- package/src/cli/lib/__tests__/install-from-github.test.ts +79 -0
- package/src/cli/lib/__tests__/install-from-platform.test.ts +47 -1
- package/src/cli/lib/__tests__/plugin-surfaces.test.ts +86 -7
- package/src/cli/lib/__tests__/upgrade-plugin.test.ts +161 -0
- package/src/cli/lib/install-from-github.ts +61 -0
- package/src/cli/lib/install-from-platform.ts +13 -2
- package/src/cli/lib/plugin-surfaces.ts +124 -4
- package/src/cli/lib/upgrade-plugin.ts +60 -0
- package/src/cli.ts +1 -1
- package/src/config/__tests__/default-profile-catalog.test.ts +11 -1
- package/src/config/__tests__/deployment-context-defaults.test.ts +73 -0
- package/src/config/__tests__/memory-retrospective-schema.test.ts +40 -0
- package/src/config/__tests__/plugin-updates-schema.test.ts +43 -0
- package/src/config/__tests__/profile-tool-support.test.ts +131 -0
- package/src/config/bundled-skills/AGENTS.md +1 -1
- package/src/config/bundled-skills/acp/SKILL.md +6 -7
- package/src/config/bundled-skills/app-builder/SKILL.md +1 -1
- package/src/config/bundled-skills/app-builder/references/RESPONSIVE.md +3 -3
- package/src/config/bundled-skills/media-processing/services/preprocess.ts +14 -4
- package/src/config/bundled-skills/phone-calls/references/CONFIG.md +15 -15
- package/src/config/bundled-skills/schedule/SKILL.md +5 -1
- package/src/config/bundled-skills/schedule/TOOLS.json +2 -2
- package/src/config/bundled-skills/settings/TOOLS.json +3 -3
- package/src/config/bundled-skills/settings/tools/voice-config-update.ts +1 -0
- package/src/config/bundled-skills/subagent/SKILL.md +53 -22
- package/src/config/bundled-skills/subagent/TOOLS.json +14 -13
- package/src/config/bundled-skills/transcribe/tools/transcribe-media.test.ts +22 -1
- package/src/config/bundled-skills/transcribe/tools/transcribe-media.ts +9 -2
- package/src/config/bundled-skills/visualize/SKILL.md +1 -1
- package/src/config/call-site-defaults.ts +4 -5
- package/src/config/default-profile-catalog.ts +9 -5
- package/src/config/feature-flag-registry.json +25 -17
- package/src/config/llm-resolver.ts +45 -15
- package/src/config/loader.ts +75 -1
- package/src/config/profile-tool-support.ts +110 -0
- package/src/config/sanitize-for-transfer.ts +11 -0
- package/src/config/schema.ts +4 -0
- package/src/config/schemas/__tests__/live-voice.test.ts +142 -4
- package/src/config/schemas/__tests__/stt.test.ts +20 -5
- package/src/config/schemas/call-site-catalog.ts +4 -4
- package/src/config/schemas/calls.ts +0 -4
- package/src/config/schemas/channels.ts +13 -0
- package/src/config/schemas/live-voice.ts +82 -23
- package/src/config/schemas/llm.ts +1 -1
- package/src/config/schemas/memory-retrospective.ts +25 -0
- package/src/config/schemas/plugin-updates.ts +59 -0
- package/src/config/schemas/services.ts +8 -4
- package/src/config/schemas/stt.ts +18 -5
- package/src/config/schemas/timeouts.ts +1 -1
- package/src/config/schemas/tts.ts +63 -0
- package/src/contacts/__tests__/guardian-delivery-reader.test.ts +16 -0
- package/src/contacts/guardian-delivery-reader.ts +29 -2
- package/src/context/compactor.ts +4 -4
- package/src/context/post-turn-tool-result-truncation.ts +32 -16
- package/src/context/strip-injections.ts +42 -0
- package/src/context/tool-result-spool.ts +32 -18
- package/src/credential-execution/ces-runtime.ts +3 -0
- package/src/credential-execution/client.ts +22 -1
- package/src/credential-execution/process-manager.test.ts +99 -0
- package/src/credential-execution/process-manager.ts +62 -16
- package/src/daemon/__tests__/abort-null-controller.test.ts +141 -5
- package/src/daemon/__tests__/conversation-surfaces-launch.test.ts +8 -12
- package/src/daemon/__tests__/install-assistant-command.test.ts +234 -0
- package/src/daemon/__tests__/turn-tail-assistant-reply-notify.test.ts +2 -4
- package/src/daemon/__tests__/turn-tail-deleted-conversation.test.ts +20 -29
- package/src/daemon/channel-ui-capability.ts +4 -4
- package/src/daemon/conversation-agent-loop-handlers.ts +117 -61
- package/src/daemon/conversation-agent-loop.ts +318 -148
- package/src/daemon/conversation-attachments.ts +1 -1
- package/src/daemon/conversation-error.ts +8 -1
- package/src/daemon/conversation-initial-prompt.ts +16 -38
- package/src/daemon/conversation-lifecycle.ts +154 -32
- package/src/daemon/conversation-messaging.ts +106 -43
- package/src/daemon/conversation-process.ts +131 -33
- package/src/daemon/conversation-queue-manager.ts +10 -0
- package/src/daemon/conversation-runtime-assembly.ts +9 -4
- package/src/daemon/conversation-store.ts +52 -3
- package/src/daemon/conversation-surface-state.ts +9 -0
- package/src/daemon/conversation-surfaces.ts +403 -210
- package/src/daemon/conversation-tool-setup.ts +56 -7
- package/src/daemon/conversation-turn-finalize.ts +134 -89
- package/src/daemon/conversation.ts +111 -27
- package/src/daemon/disk-pressure-guard-lifecycle.ts +2 -2
- package/src/daemon/disk-pressure-guard.ts +3 -3
- package/src/daemon/doordash-steps.ts +5 -5
- package/src/daemon/external-plugins-bootstrap.ts +7 -1
- package/src/daemon/handlers/config-channels.ts +48 -5
- package/src/daemon/handlers/conversation-history.ts +1 -1
- package/src/daemon/handlers/conversations.ts +62 -17
- package/src/daemon/handlers/shared.ts +12 -0
- package/src/daemon/handlers/skills.ts +11 -1
- package/src/daemon/identity-helpers.ts +38 -10
- package/src/daemon/install-assistant-command.ts +322 -0
- package/src/daemon/lifecycle.ts +14 -3
- package/src/daemon/message-types/conversations.ts +6 -7
- package/src/daemon/message-types/sync.ts +1 -0
- package/src/daemon/persist-media-references.ts +23 -14
- package/src/daemon/process-message.ts +6 -6
- package/src/daemon/tool-setup-types.ts +19 -122
- package/src/daemon/tool-side-effects.ts +17 -94
- package/src/daemon/trust-context-types.ts +39 -0
- package/src/daemon/trust-context.ts +14 -6
- package/src/daemon/turn-tail-chain.ts +73 -0
- package/src/daemon/web-search-history.ts +47 -33
- package/src/heartbeat/heartbeat-service.ts +6 -6
- package/src/inbound/__tests__/oauth-callback-url.test.ts +83 -0
- package/src/inbound/oauth-callback-url.ts +61 -0
- package/src/ipc/__tests__/socket-path.test.ts +21 -0
- package/src/ipc/assistant-server.ts +17 -10
- package/src/ipc/cli-client.ts +10 -1
- package/src/ipc/routes/__tests__/documents-sync-ipc-routes.test.ts +56 -0
- package/src/ipc/routes/documents-sync-ipc-routes.ts +42 -0
- package/src/ipc/socket-cleanup.ts +9 -7
- package/src/ipc/socket-path.ts +4 -96
- package/src/live-voice/__tests__/live-voice-agent-turn.test.ts +1 -104
- package/src/live-voice/__tests__/live-voice-archive.test.ts +16 -16
- package/src/live-voice/__tests__/live-voice-attach-image.test.ts +209 -0
- package/src/live-voice/__tests__/live-voice-connection.test.ts +20 -0
- package/src/live-voice/__tests__/live-voice-events.test.ts +7 -8
- package/src/live-voice/__tests__/live-voice-flux-turn-end.test.ts +932 -0
- package/src/live-voice/__tests__/live-voice-integration.test.ts +12 -5
- package/src/live-voice/__tests__/live-voice-metrics.test.ts +115 -8
- package/src/live-voice/__tests__/live-voice-photo.test.ts +100 -0
- package/src/live-voice/__tests__/live-voice-progress.test.ts +179 -146
- package/src/live-voice/__tests__/live-voice-session-telemetry.test.ts +265 -0
- package/src/live-voice/__tests__/live-voice-stt.test.ts +324 -2
- package/src/live-voice/__tests__/live-voice-triage-escalate.test.ts +131 -2
- package/src/live-voice/__tests__/live-voice-tts-session.test.ts +0 -483
- package/src/live-voice/__tests__/live-voice-tts.test.ts +148 -2
- package/src/live-voice/__tests__/live-voice-vad.test.ts +313 -32
- package/src/live-voice/__tests__/progress-narration.test.ts +214 -0
- package/src/live-voice/__tests__/progress-phrases.test.ts +167 -0
- package/src/live-voice/__tests__/protocol.test.ts +52 -0
- package/src/live-voice/__tests__/runtime-websocket-shell.test.ts +3 -0
- package/src/live-voice/live-voice-archive.ts +13 -13
- package/src/live-voice/live-voice-connection.ts +7 -1
- package/src/live-voice/live-voice-metrics.ts +57 -32
- package/src/live-voice/live-voice-photo.ts +168 -0
- package/src/live-voice/live-voice-session.ts +1137 -351
- package/src/live-voice/live-voice-tts.ts +18 -2
- package/src/live-voice/progress-narration.ts +277 -0
- package/src/live-voice/progress-phrases.ts +105 -2
- package/src/live-voice/protocol.ts +110 -2
- package/src/mcp/__tests__/mcp-auth-orchestrator.test.ts +0 -1
- package/src/mcp/__tests__/mcp-oauth-client-registration.test.ts +196 -0
- package/src/mcp/__tests__/mcp-oauth-provider.test.ts +9 -9
- package/src/mcp/client.ts +14 -7
- package/src/mcp/mcp-auth-orchestrator.ts +13 -22
- package/src/mcp/mcp-oauth-provider.ts +195 -240
- package/src/messaging/providers/__tests__/callback-routing.test.ts +5 -1
- package/src/messaging/providers/__tests__/transport-dispatch.test.ts +66 -4
- package/src/messaging/providers/callback-routing.ts +1 -0
- package/src/messaging/providers/discord/api.ts +312 -0
- package/src/messaging/providers/discord/dm-delivery.test.ts +204 -0
- package/src/messaging/providers/discord/render.test.ts +204 -0
- package/src/messaging/providers/discord/render.ts +195 -0
- package/src/messaging/providers/discord/send.test.ts +179 -0
- package/src/messaging/providers/discord/send.ts +212 -0
- package/src/messaging/providers/discord/transport.ts +81 -0
- package/src/messaging/providers/index.ts +3 -1
- package/src/messaging/providers/retry-policy.test.ts +247 -0
- package/src/messaging/providers/retry-policy.ts +206 -0
- package/src/messaging/providers/slack/mention-source.test.ts +285 -0
- package/src/messaging/providers/slack/mention-source.ts +287 -0
- package/src/messaging/providers/telegram-bot/api.ts +47 -130
- package/src/messaging/providers/whatsapp/api.ts +47 -136
- package/src/monitoring/__tests__/db-integrity-sample.test.ts +4 -1
- package/src/monitoring/__tests__/plugin-auto-update.test.ts +495 -0
- package/src/monitoring/db-integrity-sample.ts +6 -1
- package/src/monitoring/plugin-auto-update.ts +463 -0
- package/src/monitoring/resource-sampler.ts +21 -5
- package/src/monitoring/worker.ts +13 -0
- package/src/notifications/__tests__/assistant-reply-producer.test.ts +375 -2
- package/src/notifications/__tests__/broadcaster.test.ts +37 -1
- package/src/notifications/__tests__/copy-composer.test.ts +165 -0
- package/src/notifications/__tests__/emit-signal-pipeline-failure.test.ts +205 -0
- package/src/notifications/__tests__/notification-utils.test.ts +249 -0
- package/src/notifications/adapters/platform.ts +31 -2
- package/src/notifications/assistant-reply-producer.ts +79 -32
- package/src/notifications/broadcaster.ts +30 -10
- package/src/notifications/copy-composer.ts +69 -0
- package/src/notifications/emit-signal.ts +82 -15
- package/src/notifications/events-store.ts +10 -2
- package/src/notifications/home-feed-side-effect.ts +6 -3
- package/src/notifications/notification-utils.ts +173 -5
- package/src/notifications/signal.ts +13 -0
- package/src/oauth/AGENTS.md +4 -2
- package/src/oauth/byo-connection.ts +6 -1
- package/src/oauth/credential-token-resolver.ts +3 -10
- package/src/oauth/manual-token-connection.ts +29 -64
- package/src/oauth/manual-token-providers.ts +62 -0
- package/src/oauth/provider-serializer.ts +13 -0
- package/src/oauth/seed-providers.ts +62 -11
- package/src/onboarding/onboarding-events-store.ts +91 -1
- package/src/permissions/question-prompter.test.ts +4 -0
- package/src/permissions/question-prompter.ts +21 -4
- package/src/persistence/__tests__/conversation-lineage.test.ts +248 -0
- package/src/persistence/attachments-store.ts +15 -15
- package/src/persistence/conversation-attention-store.ts +30 -10
- package/src/persistence/conversation-bootstrap.ts +17 -0
- package/src/persistence/conversation-crud.ts +363 -97
- package/src/persistence/conversation-group-migration.ts +1 -1
- package/src/persistence/conversation-key-store.ts +38 -10
- package/src/persistence/conversation-lineage.ts +226 -0
- package/src/persistence/conversation-plugin-facade.ts +28 -3
- package/src/persistence/conversation-queries.ts +207 -33
- package/src/persistence/conversation-types.ts +107 -1
- package/src/persistence/jobs-store.ts +32 -8
- package/src/persistence/llm-usage-store.ts +22 -0
- package/src/persistence/migrations/361-normalize-managed-connection-rows.test.ts +168 -0
- package/src/persistence/migrations/361-normalize-managed-connection-rows.ts +76 -0
- package/src/persistence/migrations/362-add-conversation-subagent-kind.test.ts +187 -0
- package/src/persistence/migrations/362-add-conversation-subagent-kind.ts +93 -0
- package/src/persistence/migrations/363-backfill-schedule-inference-profile.test.ts +229 -0
- package/src/persistence/migrations/363-backfill-schedule-inference-profile.ts +127 -0
- package/src/persistence/migrations/364-add-schedule-source-key.test.ts +120 -0
- package/src/persistence/migrations/364-add-schedule-source-key.ts +43 -0
- package/src/persistence/migrations/365-add-conversation-fork-strategy.ts +27 -0
- package/src/persistence/schema/conversations.ts +32 -0
- package/src/persistence/schema/infrastructure.ts +3 -0
- package/src/persistence/steps.ts +30 -0
- package/src/persistence/subagent-store.ts +148 -6
- package/src/platform/managed-speech.test.ts +20 -0
- package/src/platform/managed-speech.ts +9 -0
- package/src/plugin-api/__tests__/oauth-callback-url-export.test.ts +29 -0
- package/src/plugin-api/index.ts +28 -5
- package/src/plugin-api/webhook-url.ts +151 -0
- package/src/plugins/__tests__/installed-plugin-dirs.test.ts +82 -0
- package/src/plugins/__tests__/mcp-servers.test.ts +370 -0
- package/src/plugins/__tests__/source-fingerprint.test.ts +5 -1
- package/src/plugins/collect-source-versions.ts +4 -27
- package/src/plugins/defaults/exploration-drift/hooks/post-tool-use.ts +5 -5
- package/src/plugins/defaults/exploration-drift/package.json +1 -1
- package/src/plugins/defaults/image-recovery/hooks/post-model-call.ts +4 -2
- package/src/plugins/defaults/image-recovery/recover.ts +67 -57
- package/src/plugins/defaults/index.ts +1 -1
- package/src/plugins/defaults/memory/__tests__/conversation-queries.test.ts +64 -18
- package/src/plugins/defaults/memory/__tests__/jobs-worker-outcome-contract.test.ts +242 -0
- package/src/plugins/defaults/memory/__tests__/jobs-worker-retrospective-sweep.test.ts +24 -1
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-enqueue.test.ts +48 -3
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-job.test.ts +511 -19
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-prompt.test.ts +39 -3
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-provider-path.test.ts +493 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-wake-chain.test.ts +5 -4
- package/src/plugins/defaults/memory/__tests__/memory-tier-boundary-guard.test.ts +5 -1
- package/src/plugins/defaults/memory/job-handlers.ts +87 -5
- package/src/plugins/defaults/memory/jobs-worker.ts +60 -6
- package/src/plugins/defaults/memory/memory-retrospective-constants.ts +11 -0
- package/src/plugins/defaults/memory/memory-retrospective-enqueue.ts +29 -0
- package/src/plugins/defaults/memory/memory-retrospective-job.ts +342 -60
- package/src/plugins/defaults/memory/memory-retrospective-prompt.ts +23 -3
- package/src/plugins/defaults/memory/substrate/consolidation-lock.ts +9 -4
- package/src/plugins/defaults/memory/substrate/skill-content.ts +8 -1
- package/src/plugins/defaults/memory/substrate/sweep-job.ts +7 -0
- package/src/plugins/defaults/memory/v1/graph/injection.ts +1 -1
- package/src/plugins/defaults/memory/v2/router.ts +8 -0
- package/src/plugins/defaults/turn-context/unified-turn-context.ts +1 -2
- package/src/plugins/external-plugin-loader.ts +64 -8
- package/src/plugins/installed-plugin-dirs.ts +101 -0
- package/src/plugins/mcp-servers.ts +343 -0
- package/src/plugins/mtime-cache.ts +34 -22
- package/src/plugins/types.ts +69 -8
- package/src/prompts/persona-resolver.ts +6 -2
- package/src/prompts/system-prompt.ts +72 -17
- package/src/providers/__tests__/inference.test.ts +40 -0
- package/src/providers/__tests__/preflight-resolved-config.test.ts +58 -1
- package/src/providers/__tests__/retry-callsite.test.ts +72 -2
- package/src/providers/__tests__/vellum-connection-routing.test.ts +42 -11
- package/src/providers/anthropic/client.ts +112 -95
- package/src/providers/connection-resolution.ts +35 -1
- package/src/providers/gemini/client.ts +5 -5
- package/src/providers/inference/adapter-factory.ts +16 -10
- package/src/providers/inference/auth.ts +16 -0
- package/src/providers/inference/connection-availability.ts +164 -12
- package/src/providers/inference/connections.ts +25 -1
- package/src/providers/media-resolve.ts +21 -11
- package/src/providers/model-catalog.ts +15 -0
- package/src/providers/openai/chat-completions-provider.ts +4 -4
- package/src/providers/openai/responses-provider.ts +12 -25
- package/src/providers/registry.ts +9 -4
- package/src/providers/retry.ts +25 -0
- package/src/providers/server-tool-pairing.ts +134 -0
- package/src/providers/speech-to-text/__tests__/deepgram-flux-frames.test.ts +433 -0
- package/src/providers/speech-to-text/__tests__/deepgram-flux-realtime.test.ts +620 -0
- package/src/providers/speech-to-text/__tests__/provider-catalog.test.ts +34 -0
- package/src/providers/speech-to-text/__tests__/resolve.test.ts +472 -12
- package/src/providers/speech-to-text/deepgram-flux-frames.ts +395 -0
- package/src/providers/speech-to-text/deepgram-flux-realtime.ts +719 -0
- package/src/providers/speech-to-text/deepgram-realtime.test.ts +283 -1
- package/src/providers/speech-to-text/deepgram-realtime.ts +117 -3
- package/src/providers/speech-to-text/deepgram.ts +1 -0
- package/src/providers/speech-to-text/provider-catalog.ts +137 -8
- package/src/providers/speech-to-text/resolve.ts +105 -12
- package/src/providers/speech-to-text/vellum-managed.ts +7 -1
- package/src/providers/speech-to-text/vellum-speech-relay-connection.ts +3 -1
- package/src/providers/types.ts +26 -6
- package/src/providers/vellum-model-routing.ts +6 -7
- package/src/runtime/AGENTS.md +2 -0
- package/src/runtime/__tests__/agent-wake.test.ts +16 -16
- package/src/runtime/__tests__/background-job-runner.test.ts +0 -144
- package/src/runtime/__tests__/desktop-presence.test.ts +162 -0
- package/src/runtime/__tests__/local-actor-identity-force-refresh.test.ts +104 -0
- package/src/runtime/agent-wake.ts +63 -24
- package/src/runtime/assistant-event-hub.ts +110 -12
- package/src/runtime/auth/__tests__/guard-tests.test.ts +2 -0
- package/src/runtime/auth/__tests__/scopes.test.ts +20 -0
- package/src/runtime/auth/same-actor.ts +56 -8
- package/src/runtime/auth/scopes.ts +12 -0
- package/src/runtime/auth/types.ts +2 -0
- package/src/runtime/background-job-runner.ts +5 -27
- package/src/runtime/channel-readiness-service.ts +59 -3
- package/src/runtime/channel-readiness-types.ts +18 -0
- package/src/runtime/desktop-presence.ts +69 -0
- package/src/runtime/http-server.ts +12 -0
- package/src/runtime/local-actor-identity.ts +29 -14
- package/src/runtime/migrations/__tests__/vbundle-symlink-streaming-importer.test.ts +0 -0
- package/src/runtime/routes/__tests__/channel-verification-routes.test.ts +1 -1
- package/src/runtime/routes/__tests__/client-routes.test.ts +186 -14
- package/src/runtime/routes/__tests__/conversation-list-routes.test.ts +547 -1
- package/src/runtime/routes/__tests__/conversation-management-routes.test.ts +162 -7
- package/src/runtime/routes/__tests__/conversation-query-routes.test.ts +14 -10
- package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +340 -3
- package/src/runtime/routes/__tests__/llm-call-sites-routes.test.ts +28 -1
- package/src/runtime/routes/__tests__/plugins-routes.test.ts +217 -1
- package/src/runtime/routes/__tests__/question-routes.test.ts +64 -0
- package/src/runtime/routes/__tests__/retrospective-routes.test.ts +20 -0
- package/src/runtime/routes/__tests__/skill-history-route.test.ts +116 -0
- package/src/runtime/routes/__tests__/sse-actor-principal-heal.test.ts +165 -0
- package/src/runtime/routes/__tests__/stt-routes.test.ts +25 -0
- package/src/runtime/routes/__tests__/surface-action-routes.test.ts +2 -0
- package/src/runtime/routes/__tests__/wake-conversation-routes.test.ts +60 -2
- package/src/runtime/routes/archive-utils.ts +16 -12
- package/src/runtime/routes/attachment-routes.ts +13 -6
- package/src/runtime/routes/canned-reply-release.ts +54 -0
- package/src/runtime/routes/channel-availability-routes.ts +32 -14
- package/src/runtime/routes/channel-verification-routes.ts +2 -27
- package/src/runtime/routes/chatgpt-subscription-auth-routes.ts +5 -0
- package/src/runtime/routes/client-routes.ts +114 -2
- package/src/runtime/routes/conversation-cli-routes.ts +10 -12
- package/src/runtime/routes/conversation-list-routes.ts +59 -10
- package/src/runtime/routes/conversation-management-routes.ts +47 -18
- package/src/runtime/routes/conversation-query-routes.ts +32 -53
- package/src/runtime/routes/conversation-routes.ts +360 -293
- package/src/runtime/routes/credential-prompt-routes.ts +20 -2
- package/src/runtime/routes/debug-routes.ts +4 -1
- package/src/runtime/routes/default-provider-routes.ts +7 -11
- package/src/runtime/routes/documents-routes.ts +44 -1
- package/src/runtime/routes/events-routes.ts +29 -1
- package/src/runtime/routes/host-app-control-routes.ts +2 -0
- package/src/runtime/routes/host-bash-routes.ts +2 -0
- package/src/runtime/routes/host-browser-routes.ts +2 -0
- package/src/runtime/routes/host-cu-routes.ts +2 -0
- package/src/runtime/routes/host-file-routes.ts +2 -0
- package/src/runtime/routes/host-transfer-routes.ts +4 -0
- package/src/runtime/routes/identity-routes.ts +6 -6
- package/src/runtime/routes/inbound-message-handler.ts +5 -5
- package/src/runtime/routes/inbound-stages/acl-enforcement.ts +7 -3
- package/src/runtime/routes/inbound-stages/background-dispatch.ts +14 -0
- package/src/runtime/routes/inbound-stages/bootstrap-intercept.ts +2 -2
- package/src/runtime/routes/inbound-stages/guardian-activation-intercept.test.ts +14 -11
- package/src/runtime/routes/inbound-stages/guardian-activation-intercept.ts +14 -17
- package/src/runtime/routes/inference-profile-availability-guard.ts +171 -0
- package/src/runtime/routes/inference-profile-session-handler.ts +26 -2
- package/src/runtime/routes/inference-profile-session-routes.ts +2 -1
- package/src/runtime/routes/inference-profiles-routes.ts +152 -73
- package/src/runtime/routes/inference-provider-connection-routes.ts +0 -16
- package/src/runtime/routes/llm-call-sites-routes.ts +13 -0
- package/src/runtime/routes/log-export-routes.ts +1 -1
- package/src/runtime/routes/mcp-auth-routes.ts +141 -57
- package/src/runtime/routes/notification-routes.ts +0 -14
- package/src/runtime/routes/oauth-providers.ts +1 -0
- package/src/runtime/routes/plugins-routes.ts +127 -4
- package/src/runtime/routes/profiler-routes.ts +4 -2
- package/src/runtime/routes/question-routes.ts +28 -10
- package/src/runtime/routes/retrospective-routes.ts +9 -3
- package/src/runtime/routes/schedule-routes.ts +210 -12
- package/src/runtime/routes/secret-routes.ts +40 -23
- package/src/runtime/routes/settings-routes.ts +41 -13
- package/src/runtime/routes/skills-routes.ts +79 -0
- package/src/runtime/routes/sse-actor-principal-heal.ts +113 -0
- package/src/runtime/routes/stt-routes.ts +31 -25
- package/src/runtime/routes/ui-snapshot-routes.ts +1 -0
- package/src/runtime/routes/wake-conversation-routes.ts +23 -0
- package/src/runtime/services/__tests__/conversation-serializer.test.ts +1 -0
- package/src/runtime/services/conversation-serializer.ts +24 -9
- package/src/runtime/sync/documents-sidecar-publish.test.ts +101 -0
- package/src/runtime/sync/resource-sync-events.ts +79 -1
- package/src/runtime/sync/worker-daemon-notify.test.ts +34 -0
- package/src/runtime/sync/worker-daemon-notify.ts +44 -6
- package/src/runtime/verification-outbound-actions.ts +250 -222
- package/src/runtime/verification-templates.ts +41 -30
- package/src/schedule/__tests__/plugin-schedule-declarations.test.ts +614 -0
- package/src/schedule/__tests__/plugin-schedule-reconciler.test.ts +1066 -0
- package/src/schedule/inference-profile.ts +64 -3
- package/src/schedule/plugin-schedule-declarations.ts +518 -0
- package/src/schedule/plugin-schedule-reconciler.ts +605 -0
- package/src/schedule/plugin-schedules-gate.ts +23 -0
- package/src/schedule/schedule-store.ts +520 -26
- package/src/schedule/scheduler.ts +38 -0
- package/src/security/credential-backend.ts +51 -4
- package/src/security/encrypted-store.ts +64 -42
- package/src/security/oauth2.ts +6 -22
- package/src/security/secure-keys.ts +55 -7
- package/src/skills/catalog-install.ts +20 -7
- package/src/skills/inline-command-runner.ts +2 -7
- package/src/skills/skill-history.test.ts +269 -0
- package/src/skills/skill-history.ts +250 -0
- package/src/skills/skillssh-registry.ts +1 -1
- package/src/stt/__tests__/daemon-batch-transcriber.test.ts +22 -0
- package/src/stt/__tests__/language-metadata.test.ts +85 -0
- package/src/stt/__tests__/speech-energy.test.ts +79 -0
- package/src/stt/__tests__/types.test.ts +94 -0
- package/src/stt/daemon-batch-transcriber.ts +18 -6
- package/src/stt/language-metadata.ts +65 -0
- package/src/stt/speech-energy.ts +115 -12
- package/src/stt/stt-stream-session.ts +8 -4
- package/src/stt/types.ts +119 -0
- package/src/subagent/__tests__/consult-prompt.test.ts +25 -0
- package/src/subagent/consult-prompt.ts +3 -1
- package/src/subagent/consult-transcript.ts +1 -1
- package/src/subagent/manager.ts +304 -44
- package/src/subagent/progress-events.ts +35 -0
- package/src/subagent/role-resolution.ts +88 -0
- package/src/subagent/types.ts +303 -67
- package/src/subagent/validate-allowlists.ts +3 -4
- package/src/telegram/__tests__/webhook-health.test.ts +73 -1
- package/src/telegram/webhook-health.ts +101 -3
- package/src/telemetry/__tests__/telemetry-event-fixtures.ts +2 -0
- package/src/telemetry/activation-funnel.ts +6 -1
- package/src/telemetry/live-voice-funnel.ts +75 -0
- package/src/telemetry/telemetry-event-sources.ts +6 -0
- package/src/telemetry/telemetry-wire-source.json +1 -1
- package/src/telemetry/telemetry-wire.generated.ts +2 -0
- package/src/telemetry/types.ts +23 -4
- package/src/telemetry/usage-telemetry-reporter.test.ts +4 -0
- package/src/tools/__tests__/subagent-workflow-tool-input-schemas.test.ts +1 -1
- package/src/tools/ask-question/ask-question-tool.test.ts +118 -3
- package/src/tools/ask-question/ask-question-tool.ts +56 -2
- package/src/tools/browser/browser-execution.ts +1 -1
- package/src/tools/calls/call-start.ts +8 -5
- package/src/tools/conversation-groups/move_to_group.ts +10 -12
- package/src/tools/document/document-tool.ts +11 -4
- package/src/tools/execution-timeout.ts +9 -2
- package/src/tools/executor.ts +7 -6
- package/src/tools/filesystem/edit.ts +1 -1
- package/src/tools/filesystem/list.ts +1 -1
- package/src/tools/filesystem/read.ts +7 -13
- package/src/tools/filesystem/search.ts +23 -21
- package/src/tools/filesystem/write.ts +4 -1
- package/src/tools/host-filesystem/edit.ts +7 -2
- package/src/tools/host-filesystem/read.ts +21 -8
- package/src/tools/host-filesystem/write.ts +10 -2
- package/src/tools/host-terminal/host-shell.ts +2 -2
- package/src/tools/mcp/mcp-tool-factory.ts +25 -1
- package/src/tools/permission-checker.ts +8 -3
- package/src/tools/schedule/delete.ts +13 -1
- package/src/tools/schedule/list.ts +14 -3
- package/src/tools/schedule/update.ts +56 -33
- package/src/tools/shared/filesystem/audio-read.ts +6 -4
- package/src/tools/shared/filesystem/file-ops-service.ts +176 -116
- package/src/tools/shared/filesystem/image-read.ts +12 -12
- package/src/tools/shared/filesystem/path-policy.ts +70 -5
- package/src/tools/shared/filesystem/size-guard.ts +6 -6
- package/src/tools/shared/input-misuse.ts +72 -0
- package/src/tools/skills/scaffold-managed.ts +82 -0
- package/src/tools/skills/skill-tool-factory.ts +14 -1
- package/src/tools/subagent/consult-deadline.ts +12 -7
- package/src/tools/subagent/message.ts +3 -1
- package/src/tools/subagent/read.ts +66 -4
- package/src/tools/subagent/spawn.ts +500 -78
- package/src/tools/tool-approval-handler.ts +45 -4
- package/src/tools/types.ts +22 -3
- package/src/tools/ui-surface/surface-shape-docs.ts +1 -1
- package/src/tools/ui-surface/visual-validation.ts +2 -0
- package/src/tts/__tests__/provider-adapters.test.ts +147 -0
- package/src/tts/__tests__/speakable-segments.test.ts +470 -0
- package/src/tts/language-voices.ts +23 -0
- package/src/tts/providers/deepgram-provider.ts +3 -1
- package/src/tts/providers/elevenlabs-provider.ts +73 -1
- package/src/tts/providers/xai-provider.ts +28 -2
- package/src/tts/speakable-segments.ts +293 -23
- package/src/tts/synthesis-stream.ts +7 -0
- package/src/tts/types.ts +7 -0
- package/src/usage/__tests__/subagent-attribution.test.ts +109 -0
- package/src/usage/subagent-attribution.ts +108 -0
- package/src/util/__tests__/language-subtag.test.ts +54 -0
- package/src/util/__tests__/sqlite-retry.test.ts +112 -0
- package/src/util/abort-reasons.ts +16 -0
- package/src/util/ansi.ts +44 -0
- package/src/util/disk-usage.ts +17 -13
- package/src/util/image-conversion.ts +20 -13
- package/src/util/language-subtag.ts +43 -0
- package/src/util/schedule-source-key.ts +23 -0
- package/src/util/unicode.ts +1 -1
- package/src/util/worker-process.ts +7 -1
- package/src/workspace/git-service.ts +33 -9
- package/src/workspace/migrations/140-repair-seed-pinned-memory-v3-live.ts +260 -0
- package/src/workspace/migrations/141-stt-english-default-to-multilingual.ts +111 -0
- package/src/workspace/migrations/142-consolidate-voice-front-door.ts +70 -0
- package/src/workspace/migrations/__tests__/141-stt-english-default-to-multilingual.test.ts +116 -0
- package/src/workspace/migrations/registry.ts +6 -0
- package/src/workspace/top-level-renderer.ts +15 -2
- package/src/__tests__/tool-side-effects-slack-dm.test.ts +0 -357
- package/src/daemon/install-symlink.ts +0 -207
- package/src/live-voice/__tests__/front-decision.test.ts +0 -525
- package/src/live-voice/front-decision.ts +0 -429
- package/src/notifications/deferred-emit.ts +0 -147
|
@@ -24,8 +24,11 @@ mock.module("../../security/secure-keys.js", () => ({
|
|
|
24
24
|
getProviderKeyAsync: async () => "test-api-key",
|
|
25
25
|
}));
|
|
26
26
|
|
|
27
|
-
const {
|
|
28
|
-
|
|
27
|
+
const {
|
|
28
|
+
LiveVoiceTtsError,
|
|
29
|
+
resolveLanguageVoiceOverride,
|
|
30
|
+
streamLiveVoiceTtsAudio,
|
|
31
|
+
} = await import("../live-voice-tts.js");
|
|
29
32
|
|
|
30
33
|
beforeEach(() => {
|
|
31
34
|
config = makeConfig();
|
|
@@ -494,6 +497,76 @@ describe("streamLiveVoiceTtsAudio", () => {
|
|
|
494
497
|
}
|
|
495
498
|
});
|
|
496
499
|
|
|
500
|
+
test("applies the configured per-language voice for a language-known turn", async () => {
|
|
501
|
+
config = makeConfig({
|
|
502
|
+
provider: "elevenlabs",
|
|
503
|
+
elevenlabsLanguageVoices: { hi: "voice-hindi", ja: "voice-japanese" },
|
|
504
|
+
});
|
|
505
|
+
const { requests, probeRequests } = installCapturingElevenLabsStub();
|
|
506
|
+
|
|
507
|
+
await streamLiveVoiceTtsAudio({
|
|
508
|
+
config,
|
|
509
|
+
text: "namaste",
|
|
510
|
+
language: "hi",
|
|
511
|
+
onAudioChunk: () => {},
|
|
512
|
+
});
|
|
513
|
+
|
|
514
|
+
expect(requests[0]?.voiceId).toBe("voice-hindi");
|
|
515
|
+
expect(probeRequests[0]?.voiceId).toBe("voice-hindi");
|
|
516
|
+
});
|
|
517
|
+
|
|
518
|
+
test("skips the override when the map has no entry for the turn language", async () => {
|
|
519
|
+
config = makeConfig({
|
|
520
|
+
provider: "elevenlabs",
|
|
521
|
+
elevenlabsLanguageVoices: { hi: "voice-hindi" },
|
|
522
|
+
});
|
|
523
|
+
const { requests } = installCapturingElevenLabsStub();
|
|
524
|
+
|
|
525
|
+
await streamLiveVoiceTtsAudio({
|
|
526
|
+
config,
|
|
527
|
+
text: "hola",
|
|
528
|
+
language: "es",
|
|
529
|
+
onAudioChunk: () => {},
|
|
530
|
+
});
|
|
531
|
+
|
|
532
|
+
expect(requests[0]?.voiceId).toBeUndefined();
|
|
533
|
+
});
|
|
534
|
+
|
|
535
|
+
test("an explicit request voiceId outranks the per-language map", async () => {
|
|
536
|
+
config = makeConfig({
|
|
537
|
+
provider: "elevenlabs",
|
|
538
|
+
elevenlabsLanguageVoices: { hi: "voice-hindi" },
|
|
539
|
+
});
|
|
540
|
+
const { requests, probeRequests } = installCapturingElevenLabsStub();
|
|
541
|
+
|
|
542
|
+
await streamLiveVoiceTtsAudio({
|
|
543
|
+
config,
|
|
544
|
+
text: "namaste",
|
|
545
|
+
language: "hi",
|
|
546
|
+
voiceId: "voice-explicit",
|
|
547
|
+
onAudioChunk: () => {},
|
|
548
|
+
});
|
|
549
|
+
|
|
550
|
+
expect(requests[0]?.voiceId).toBe("voice-explicit");
|
|
551
|
+
expect(probeRequests[0]?.voiceId).toBe("voice-explicit");
|
|
552
|
+
});
|
|
553
|
+
|
|
554
|
+
test("performs no lookup when the turn has no language", async () => {
|
|
555
|
+
config = makeConfig({
|
|
556
|
+
provider: "elevenlabs",
|
|
557
|
+
elevenlabsLanguageVoices: { hi: "voice-hindi" },
|
|
558
|
+
});
|
|
559
|
+
const { requests } = installCapturingElevenLabsStub();
|
|
560
|
+
|
|
561
|
+
await streamLiveVoiceTtsAudio({
|
|
562
|
+
config,
|
|
563
|
+
text: "hello",
|
|
564
|
+
onAudioChunk: () => {},
|
|
565
|
+
});
|
|
566
|
+
|
|
567
|
+
expect(requests[0]?.voiceId).toBeUndefined();
|
|
568
|
+
});
|
|
569
|
+
|
|
497
570
|
test("keeps live voice TTS behind the registry instead of direct provider SDKs", () => {
|
|
498
571
|
const source = readFileSync(
|
|
499
572
|
new URL("../live-voice-tts.ts", import.meta.url),
|
|
@@ -508,11 +581,83 @@ describe("streamLiveVoiceTtsAudio", () => {
|
|
|
508
581
|
});
|
|
509
582
|
});
|
|
510
583
|
|
|
584
|
+
describe("resolveLanguageVoiceOverride", () => {
|
|
585
|
+
test("keys the lookup by the language's lowercase base subtag", () => {
|
|
586
|
+
const map = { hi: "voice-hindi" };
|
|
587
|
+
expect(resolveLanguageVoiceOverride(map, "hi")).toBe("voice-hindi");
|
|
588
|
+
expect(resolveLanguageVoiceOverride(map, "hi-IN")).toBe("voice-hindi");
|
|
589
|
+
expect(resolveLanguageVoiceOverride(map, "HI")).toBe("voice-hindi");
|
|
590
|
+
});
|
|
591
|
+
|
|
592
|
+
test("returns undefined without a language", () => {
|
|
593
|
+
expect(
|
|
594
|
+
resolveLanguageVoiceOverride({ hi: "voice-hindi" }, undefined),
|
|
595
|
+
).toBeUndefined();
|
|
596
|
+
expect(
|
|
597
|
+
resolveLanguageVoiceOverride({ hi: "voice-hindi" }, ""),
|
|
598
|
+
).toBeUndefined();
|
|
599
|
+
});
|
|
600
|
+
|
|
601
|
+
test("returns undefined for a missing map, a missing entry, or a blank voice", () => {
|
|
602
|
+
expect(resolveLanguageVoiceOverride(undefined, "hi")).toBeUndefined();
|
|
603
|
+
expect(
|
|
604
|
+
resolveLanguageVoiceOverride({ ja: "voice-japanese" }, "hi"),
|
|
605
|
+
).toBeUndefined();
|
|
606
|
+
expect(resolveLanguageVoiceOverride({ hi: " " }, "hi")).toBeUndefined();
|
|
607
|
+
expect(resolveLanguageVoiceOverride({}, "constructor")).toBeUndefined();
|
|
608
|
+
});
|
|
609
|
+
|
|
610
|
+
test("trims the resolved voice identifier", () => {
|
|
611
|
+
expect(resolveLanguageVoiceOverride({ hi: " voice-hindi " }, "hi")).toBe(
|
|
612
|
+
"voice-hindi",
|
|
613
|
+
);
|
|
614
|
+
});
|
|
615
|
+
});
|
|
616
|
+
|
|
617
|
+
/**
|
|
618
|
+
* Install an ElevenLabs-id streaming stub that records the sample-rate probe
|
|
619
|
+
* and synthesis requests.
|
|
620
|
+
*/
|
|
621
|
+
function installCapturingElevenLabsStub(): {
|
|
622
|
+
requests: TtsSynthesisRequest[];
|
|
623
|
+
probeRequests: TtsSynthesisRequest[];
|
|
624
|
+
} {
|
|
625
|
+
const requests: TtsSynthesisRequest[] = [];
|
|
626
|
+
const probeRequests: TtsSynthesisRequest[] = [];
|
|
627
|
+
_setTtsProviderForTests({
|
|
628
|
+
id: "elevenlabs",
|
|
629
|
+
capabilities: {
|
|
630
|
+
supportsStreaming: true,
|
|
631
|
+
supportedFormats: ["mp3", "pcm"],
|
|
632
|
+
},
|
|
633
|
+
resolveOutputSampleRateHz: (request) => {
|
|
634
|
+
probeRequests.push(request);
|
|
635
|
+
return 24_000;
|
|
636
|
+
},
|
|
637
|
+
async synthesize(): Promise<TtsSynthesisResult> {
|
|
638
|
+
throw new Error("buffered synthesis should not be used");
|
|
639
|
+
},
|
|
640
|
+
async synthesizeStream(
|
|
641
|
+
request: TtsSynthesisRequest,
|
|
642
|
+
onChunk: (chunk: Uint8Array) => void,
|
|
643
|
+
): Promise<TtsSynthesisResult> {
|
|
644
|
+
requests.push(request);
|
|
645
|
+
onChunk(Buffer.from("pcm-one!"));
|
|
646
|
+
return {
|
|
647
|
+
audio: Buffer.from("pcm-one!"),
|
|
648
|
+
contentType: "audio/pcm",
|
|
649
|
+
};
|
|
650
|
+
},
|
|
651
|
+
});
|
|
652
|
+
return { requests, probeRequests };
|
|
653
|
+
}
|
|
654
|
+
|
|
511
655
|
function makeConfig(
|
|
512
656
|
overrides: {
|
|
513
657
|
provider?: string;
|
|
514
658
|
format?: "mp3" | "wav" | "opus";
|
|
515
659
|
sampleRate?: number;
|
|
660
|
+
elevenlabsLanguageVoices?: Record<string, string>;
|
|
516
661
|
} = {},
|
|
517
662
|
): LiveVoiceTtsConfig {
|
|
518
663
|
return {
|
|
@@ -535,6 +680,7 @@ function makeConfig(
|
|
|
535
680
|
stability: 0.5,
|
|
536
681
|
similarityBoost: 0.75,
|
|
537
682
|
conversationTimeoutSeconds: 30,
|
|
683
|
+
languageVoices: overrides.elevenlabsLanguageVoices,
|
|
538
684
|
},
|
|
539
685
|
deepgram: {
|
|
540
686
|
model: "aura-asteria-en",
|
|
@@ -19,7 +19,6 @@ import type {
|
|
|
19
19
|
} from "../../stt/types.js";
|
|
20
20
|
import { __resetRegistryForTesting } from "../../tools/registry.js";
|
|
21
21
|
import { getWorkspaceSkillsDir } from "../../util/platform.js";
|
|
22
|
-
import type { VoiceFrontDecider } from "../front-decision.js";
|
|
23
22
|
import type { LiveVoiceAudioArchiveResult } from "../live-voice-archive.js";
|
|
24
23
|
import {
|
|
25
24
|
CONTINUATION_DELIVERY_CONTENT,
|
|
@@ -83,6 +82,21 @@ function pcm(amplitude: number, sampleCount = 240): Uint8Array {
|
|
|
83
82
|
return new Uint8Array(buffer);
|
|
84
83
|
}
|
|
85
84
|
|
|
85
|
+
function tonePcm(
|
|
86
|
+
amplitude: number,
|
|
87
|
+
frequencyHz: number,
|
|
88
|
+
sampleCount = 240,
|
|
89
|
+
): Uint8Array {
|
|
90
|
+
const buffer = Buffer.alloc(sampleCount * 2);
|
|
91
|
+
for (let index = 0; index < sampleCount; index += 1) {
|
|
92
|
+
const sample = Math.round(
|
|
93
|
+
amplitude * Math.sin((2 * Math.PI * frequencyHz * index) / SAMPLE_RATE),
|
|
94
|
+
);
|
|
95
|
+
buffer.writeInt16LE(sample, index * 2);
|
|
96
|
+
}
|
|
97
|
+
return new Uint8Array(buffer);
|
|
98
|
+
}
|
|
99
|
+
|
|
86
100
|
// 10 ms of speech at 24 kHz.
|
|
87
101
|
const LOUD_CHUNK = pcm(8_000);
|
|
88
102
|
// 300 ms of speech at 24 kHz — comfortably exceeds the default sustained-speech
|
|
@@ -103,9 +117,13 @@ class MockStreamingTranscriber implements StreamingTranscriber {
|
|
|
103
117
|
readonly received: Buffer[] = [];
|
|
104
118
|
stopped = false;
|
|
105
119
|
private onEvent: ((event: SttStreamServerEvent) => void) | null = null;
|
|
120
|
+
private flushed = false;
|
|
121
|
+
// Claimed on the first audio chunk, so the scripted transcript is spent in
|
|
122
|
+
// the order cycles hear speech (see takeScriptedFinal).
|
|
123
|
+
private scriptedFinal: string | null = null;
|
|
106
124
|
|
|
107
125
|
constructor(
|
|
108
|
-
private readonly
|
|
126
|
+
private readonly takeScriptedFinal: () => string,
|
|
109
127
|
private readonly holdStopEvents = false,
|
|
110
128
|
) {}
|
|
111
129
|
|
|
@@ -114,6 +132,7 @@ class MockStreamingTranscriber implements StreamingTranscriber {
|
|
|
114
132
|
}
|
|
115
133
|
|
|
116
134
|
sendAudio(chunk: Buffer): void {
|
|
135
|
+
this.scriptedFinal ??= this.takeScriptedFinal();
|
|
117
136
|
this.received.push(Buffer.from(chunk));
|
|
118
137
|
}
|
|
119
138
|
|
|
@@ -128,19 +147,20 @@ class MockStreamingTranscriber implements StreamingTranscriber {
|
|
|
128
147
|
}
|
|
129
148
|
|
|
130
149
|
flushStopEvents(): void {
|
|
150
|
+
if (this.flushed) {
|
|
151
|
+
return;
|
|
152
|
+
}
|
|
153
|
+
this.flushed = true;
|
|
131
154
|
// A stream that was never sent audio has nothing to transcribe, so a real
|
|
132
|
-
// provider closes without a final. Scripting one anyway
|
|
133
|
-
//
|
|
134
|
-
// post-cancel re-arm
|
|
135
|
-
//
|
|
155
|
+
// provider closes without a final. Scripting one anyway lets a cycle that
|
|
156
|
+
// arms and tears down in the same breath (a client interrupt racing the
|
|
157
|
+
// post-cancel re-arm) hand the session a transcript it never heard, which
|
|
158
|
+
// launches a turn nothing will ever finalize and wedges the
|
|
136
159
|
// one-turn-at-a-time gate for every later utterance.
|
|
137
|
-
|
|
138
|
-
this.
|
|
139
|
-
? this.stopEvents
|
|
140
|
-
: this.stopEvents.filter((event) => event.type !== "final");
|
|
141
|
-
for (const event of events) {
|
|
142
|
-
this.onEvent?.(event);
|
|
160
|
+
if (this.scriptedFinal !== null) {
|
|
161
|
+
this.onEvent?.({ type: "final", text: this.scriptedFinal });
|
|
143
162
|
}
|
|
163
|
+
this.onEvent?.({ type: "closed" });
|
|
144
164
|
}
|
|
145
165
|
|
|
146
166
|
// Provider-initiated event (e.g. an idle-timeout close), no stop() needed.
|
|
@@ -158,7 +178,9 @@ function createHarness(options: {
|
|
|
158
178
|
turnDetectorConfig?: TurnDetectorConfig;
|
|
159
179
|
speechEnergyThreshold?: number;
|
|
160
180
|
bargeInMinSpeechMs?: number;
|
|
161
|
-
|
|
181
|
+
echoBargeInMargin?: number;
|
|
182
|
+
echoEmaHalfLifeMs?: number;
|
|
183
|
+
echoDrainSlackMs?: number;
|
|
162
184
|
frontModelConfig?: Partial<LiveVoiceFrontModelConfig>;
|
|
163
185
|
emitMetrics?: boolean;
|
|
164
186
|
metricsClock?: () => number;
|
|
@@ -192,12 +214,23 @@ function createHarness(options: {
|
|
|
192
214
|
};
|
|
193
215
|
|
|
194
216
|
const finals = options.finals ?? ["hello world"];
|
|
217
|
+
// Scripted finals are handed out in the order cycles first receive audio,
|
|
218
|
+
// not in the order they arm. Server VAD arms and discards audio-free cycles
|
|
219
|
+
// on wall-clock timers (the trailing-silence timer, the post-cancel re-arm),
|
|
220
|
+
// so how many cycles have armed by a given point depends on how fast the
|
|
221
|
+
// event loop is, while the order the test feeds audio in does not. Keying
|
|
222
|
+
// off audio keeps `finals` reading as "what the user says, in order".
|
|
223
|
+
let scriptedFinalIndex = 0;
|
|
224
|
+
const takeScriptedFinal = (): string => {
|
|
225
|
+
const text =
|
|
226
|
+
finals[scriptedFinalIndex] ?? `utterance ${scriptedFinalIndex + 1}`;
|
|
227
|
+
scriptedFinalIndex += 1;
|
|
228
|
+
return text;
|
|
229
|
+
};
|
|
195
230
|
const transcribers: MockStreamingTranscriber[] = [];
|
|
196
231
|
const resolveTranscriber = mock(async () => {
|
|
197
|
-
const text =
|
|
198
|
-
finals[transcribers.length] ?? `utterance ${transcribers.length + 1}`;
|
|
199
232
|
const transcriber = new MockStreamingTranscriber(
|
|
200
|
-
|
|
233
|
+
takeScriptedFinal,
|
|
201
234
|
options.holdStopEventsFor?.includes(transcribers.length) ?? false,
|
|
202
235
|
);
|
|
203
236
|
transcribers.push(transcriber);
|
|
@@ -225,9 +258,10 @@ function createHarness(options: {
|
|
|
225
258
|
(options.viaFactory ? undefined : { silenceThresholdMs: 40 }),
|
|
226
259
|
speechEnergyThreshold: options.speechEnergyThreshold,
|
|
227
260
|
bargeInMinSpeechMs: options.bargeInMinSpeechMs,
|
|
228
|
-
|
|
229
|
-
?
|
|
230
|
-
|
|
261
|
+
echoBargeInMargin:
|
|
262
|
+
options.echoBargeInMargin ?? (options.viaFactory ? undefined : 1),
|
|
263
|
+
echoEmaHalfLifeMs: options.echoEmaHalfLifeMs,
|
|
264
|
+
echoDrainSlackMs: options.echoDrainSlackMs,
|
|
231
265
|
...(options.frontModelConfig
|
|
232
266
|
? { frontModelConfig: options.frontModelConfig }
|
|
233
267
|
: {}),
|
|
@@ -274,6 +308,15 @@ function makeTtsChunk(text: string): LiveVoiceTtsAudioChunk {
|
|
|
274
308
|
};
|
|
275
309
|
}
|
|
276
310
|
|
|
311
|
+
function makePcmTtsChunk(audio: Uint8Array): LiveVoiceTtsAudioChunk {
|
|
312
|
+
return {
|
|
313
|
+
type: "tts_audio",
|
|
314
|
+
contentType: "audio/pcm",
|
|
315
|
+
sampleRate: SAMPLE_RATE,
|
|
316
|
+
dataBase64: Buffer.from(audio).toString("base64"),
|
|
317
|
+
};
|
|
318
|
+
}
|
|
319
|
+
|
|
277
320
|
function makeTtsResult(text: string): LiveVoiceTtsResult {
|
|
278
321
|
return {
|
|
279
322
|
provider: "fish-audio",
|
|
@@ -2929,8 +2972,11 @@ describe("LiveVoiceSession server VAD", () => {
|
|
|
2929
2972
|
|
|
2930
2973
|
test("an idle transcriber close before speech re-arms capture for the next utterance", async () => {
|
|
2931
2974
|
const { startVoiceTurn, calls } = makeAutoCompletingTurnStarter(["Hi."]);
|
|
2975
|
+
// The first transcriber closes before it is sent any audio, so it never
|
|
2976
|
+
// claims a scripted final: the one entry belongs to the cycle that speech
|
|
2977
|
+
// actually reaches.
|
|
2932
2978
|
const { frames, session, transcribers } = createHarness({
|
|
2933
|
-
finals: ["
|
|
2979
|
+
finals: ["hello after close"],
|
|
2934
2980
|
startVoiceTurn,
|
|
2935
2981
|
});
|
|
2936
2982
|
|
|
@@ -3189,6 +3235,9 @@ describe("LiveVoiceSession VAD threshold configuration", () => {
|
|
|
3189
3235
|
silenceThresholdMs: 1200,
|
|
3190
3236
|
maxTurnDurationMs: 30_000,
|
|
3191
3237
|
bargeInMinSpeechMs: 250,
|
|
3238
|
+
echoBargeInMargin: 1.5,
|
|
3239
|
+
echoEmaHalfLifeMs: 400,
|
|
3240
|
+
echoDrainSlackMs: 300,
|
|
3192
3241
|
});
|
|
3193
3242
|
|
|
3194
3243
|
const { frames, session } = createHarness({ viaFactory: true });
|
|
@@ -3302,6 +3351,10 @@ describe("LiveVoiceSession sustained-speech barge-in guard", () => {
|
|
|
3302
3351
|
// detector timers stay out of the guard's audio-duration accounting.
|
|
3303
3352
|
function createSpeakingTurnHarness(options: {
|
|
3304
3353
|
bargeInMinSpeechMs: number;
|
|
3354
|
+
echoEmaHalfLifeMs?: number;
|
|
3355
|
+
echoBargeInMargin?: number;
|
|
3356
|
+
echoDrainSlackMs?: number;
|
|
3357
|
+
ttsAudio?: Uint8Array;
|
|
3305
3358
|
finals?: string[];
|
|
3306
3359
|
startFrame?: LiveVoiceClientStartFrame;
|
|
3307
3360
|
}) {
|
|
@@ -3312,7 +3365,11 @@ describe("LiveVoiceSession sustained-speech barge-in guard", () => {
|
|
|
3312
3365
|
return { turnId: "bridge-turn", abort };
|
|
3313
3366
|
});
|
|
3314
3367
|
const streamTtsAudio = mock(async (ttsOptions: LiveVoiceTtsOptions) => {
|
|
3315
|
-
ttsOptions.onAudioChunk(
|
|
3368
|
+
ttsOptions.onAudioChunk(
|
|
3369
|
+
options.ttsAudio
|
|
3370
|
+
? makePcmTtsChunk(options.ttsAudio)
|
|
3371
|
+
: makeTtsChunk("assistant audio"),
|
|
3372
|
+
);
|
|
3316
3373
|
return makeTtsResult("assistant audio");
|
|
3317
3374
|
});
|
|
3318
3375
|
const harness = createHarness({
|
|
@@ -3320,6 +3377,9 @@ describe("LiveVoiceSession sustained-speech barge-in guard", () => {
|
|
|
3320
3377
|
startVoiceTurn,
|
|
3321
3378
|
streamTtsAudio,
|
|
3322
3379
|
bargeInMinSpeechMs: options.bargeInMinSpeechMs,
|
|
3380
|
+
echoEmaHalfLifeMs: options.echoEmaHalfLifeMs ?? 4,
|
|
3381
|
+
echoBargeInMargin: options.echoBargeInMargin ?? 1,
|
|
3382
|
+
echoDrainSlackMs: options.echoDrainSlackMs ?? 60_000,
|
|
3323
3383
|
turnDetectorConfig: { silenceThresholdMs: 5_000 },
|
|
3324
3384
|
...(options.startFrame ? { startFrame: options.startFrame } : {}),
|
|
3325
3385
|
});
|
|
@@ -3633,6 +3693,238 @@ describe("LiveVoiceSession sustained-speech barge-in guard", () => {
|
|
|
3633
3693
|
).toMatchObject({ type: "turn_cancelled", turnId: "live-turn-1" });
|
|
3634
3694
|
await waitFor(() => abort.mock.calls.length === 1);
|
|
3635
3695
|
});
|
|
3696
|
+
|
|
3697
|
+
describe("echo-adaptive barge-in", () => {
|
|
3698
|
+
const playbackEchoChunk = tonePcm(4_700, 200);
|
|
3699
|
+
const bargeInSpeechChunk = tonePcm(9_400, 530);
|
|
3700
|
+
const playbackReference = tonePcm(4_700, 200, SAMPLE_RATE * 2);
|
|
3701
|
+
|
|
3702
|
+
test("steady loud playback echo does not interrupt the turn", async () => {
|
|
3703
|
+
const { frames, session, abort, speakFirstReply } =
|
|
3704
|
+
createSpeakingTurnHarness({
|
|
3705
|
+
bargeInMinSpeechMs: 60,
|
|
3706
|
+
echoBargeInMargin: 1.5,
|
|
3707
|
+
echoEmaHalfLifeMs: 40,
|
|
3708
|
+
ttsAudio: playbackReference,
|
|
3709
|
+
});
|
|
3710
|
+
await speakFirstReply();
|
|
3711
|
+
const speechStartedBaseline = countType(frames, "speech_started");
|
|
3712
|
+
|
|
3713
|
+
for (let index = 0; index < 40; index += 1) {
|
|
3714
|
+
await session.handleBinaryAudio(playbackEchoChunk);
|
|
3715
|
+
}
|
|
3716
|
+
await flushAsyncCallbacks();
|
|
3717
|
+
|
|
3718
|
+
expect(countType(frames, "speech_started")).toBe(speechStartedBaseline);
|
|
3719
|
+
expect(countType(frames, "turn_cancelled")).toBe(0);
|
|
3720
|
+
expect(abort).not.toHaveBeenCalled();
|
|
3721
|
+
});
|
|
3722
|
+
|
|
3723
|
+
test("speech above the learned echo margin still interrupts", async () => {
|
|
3724
|
+
const { frames, session, abort, speakFirstReply } =
|
|
3725
|
+
createSpeakingTurnHarness({
|
|
3726
|
+
bargeInMinSpeechMs: 60,
|
|
3727
|
+
echoBargeInMargin: 1.5,
|
|
3728
|
+
echoEmaHalfLifeMs: 400,
|
|
3729
|
+
ttsAudio: playbackReference,
|
|
3730
|
+
});
|
|
3731
|
+
await speakFirstReply();
|
|
3732
|
+
|
|
3733
|
+
for (let index = 0; index < 25; index += 1) {
|
|
3734
|
+
await session.handleBinaryAudio(playbackEchoChunk);
|
|
3735
|
+
}
|
|
3736
|
+
for (let index = 0; index < 8; index += 1) {
|
|
3737
|
+
await session.handleBinaryAudio(bargeInSpeechChunk);
|
|
3738
|
+
}
|
|
3739
|
+
|
|
3740
|
+
await waitFor(() => countType(frames, "turn_cancelled") === 1);
|
|
3741
|
+
await waitFor(() => abort.mock.calls.length === 1);
|
|
3742
|
+
});
|
|
3743
|
+
|
|
3744
|
+
test("classified echo resets a partial guard run immediately", async () => {
|
|
3745
|
+
const { frames, session, abort, speakFirstReply } =
|
|
3746
|
+
createSpeakingTurnHarness({
|
|
3747
|
+
bargeInMinSpeechMs: 60,
|
|
3748
|
+
echoBargeInMargin: 1.5,
|
|
3749
|
+
echoEmaHalfLifeMs: 400,
|
|
3750
|
+
ttsAudio: playbackReference,
|
|
3751
|
+
});
|
|
3752
|
+
await speakFirstReply();
|
|
3753
|
+
|
|
3754
|
+
for (let index = 0; index < 25; index += 1) {
|
|
3755
|
+
await session.handleBinaryAudio(playbackEchoChunk);
|
|
3756
|
+
}
|
|
3757
|
+
for (let index = 0; index < 5; index += 1) {
|
|
3758
|
+
await session.handleBinaryAudio(bargeInSpeechChunk);
|
|
3759
|
+
}
|
|
3760
|
+
await session.handleBinaryAudio(playbackEchoChunk);
|
|
3761
|
+
await session.handleBinaryAudio(bargeInSpeechChunk);
|
|
3762
|
+
await flushAsyncCallbacks();
|
|
3763
|
+
|
|
3764
|
+
expect(countType(frames, "turn_cancelled")).toBe(0);
|
|
3765
|
+
expect(abort).not.toHaveBeenCalled();
|
|
3766
|
+
|
|
3767
|
+
for (let index = 0; index < 5; index += 1) {
|
|
3768
|
+
await session.handleBinaryAudio(bargeInSpeechChunk);
|
|
3769
|
+
}
|
|
3770
|
+
await waitFor(() => countType(frames, "turn_cancelled") === 1);
|
|
3771
|
+
});
|
|
3772
|
+
|
|
3773
|
+
test("quiet playback keeps fixed-threshold barge-in sensitivity", async () => {
|
|
3774
|
+
const { frames, session, abort, speakFirstReply } =
|
|
3775
|
+
createSpeakingTurnHarness({
|
|
3776
|
+
bargeInMinSpeechMs: 60,
|
|
3777
|
+
echoBargeInMargin: 1.5,
|
|
3778
|
+
echoEmaHalfLifeMs: 40,
|
|
3779
|
+
ttsAudio: playbackReference,
|
|
3780
|
+
});
|
|
3781
|
+
await speakFirstReply();
|
|
3782
|
+
|
|
3783
|
+
for (let index = 0; index < 31; index += 1) {
|
|
3784
|
+
await session.handleBinaryAudio(pcm(200));
|
|
3785
|
+
}
|
|
3786
|
+
for (let index = 0; index < 7; index += 1) {
|
|
3787
|
+
await session.handleBinaryAudio(bargeInSpeechChunk);
|
|
3788
|
+
}
|
|
3789
|
+
|
|
3790
|
+
await waitFor(() => countType(frames, "turn_cancelled") === 1);
|
|
3791
|
+
await waitFor(() => abort.mock.calls.length === 1);
|
|
3792
|
+
});
|
|
3793
|
+
|
|
3794
|
+
test("playback echo is not forwarded as transcription pre-roll", async () => {
|
|
3795
|
+
const { frames, session, transcribers, speakFirstReply } =
|
|
3796
|
+
createSpeakingTurnHarness({
|
|
3797
|
+
bargeInMinSpeechMs: 60,
|
|
3798
|
+
echoBargeInMargin: 1.5,
|
|
3799
|
+
echoEmaHalfLifeMs: 40,
|
|
3800
|
+
ttsAudio: playbackReference,
|
|
3801
|
+
});
|
|
3802
|
+
await speakFirstReply();
|
|
3803
|
+
|
|
3804
|
+
const echoChunk = playbackEchoChunk;
|
|
3805
|
+
for (let index = 0; index < 5; index += 1) {
|
|
3806
|
+
await session.handleBinaryAudio(echoChunk);
|
|
3807
|
+
}
|
|
3808
|
+
for (let index = 0; index < 7; index += 1) {
|
|
3809
|
+
await session.handleBinaryAudio(bargeInSpeechChunk);
|
|
3810
|
+
}
|
|
3811
|
+
await waitFor(() => countType(frames, "turn_cancelled") === 1);
|
|
3812
|
+
|
|
3813
|
+
const echoBuffer = Buffer.from(echoChunk);
|
|
3814
|
+
expect(
|
|
3815
|
+
transcribers.some((transcriber) =>
|
|
3816
|
+
transcriber.received.some((buffer) => buffer.equals(echoBuffer)),
|
|
3817
|
+
),
|
|
3818
|
+
).toBe(false);
|
|
3819
|
+
});
|
|
3820
|
+
|
|
3821
|
+
test("instant barge-in remains protected from onset echo", async () => {
|
|
3822
|
+
const { frames, session, abort, speakFirstReply } =
|
|
3823
|
+
createSpeakingTurnHarness({
|
|
3824
|
+
bargeInMinSpeechMs: 0,
|
|
3825
|
+
echoBargeInMargin: 1.5,
|
|
3826
|
+
echoEmaHalfLifeMs: 40,
|
|
3827
|
+
ttsAudio: playbackReference,
|
|
3828
|
+
});
|
|
3829
|
+
await speakFirstReply();
|
|
3830
|
+
|
|
3831
|
+
for (let index = 0; index < 30; index += 1) {
|
|
3832
|
+
await session.handleBinaryAudio(playbackEchoChunk);
|
|
3833
|
+
}
|
|
3834
|
+
await flushAsyncCallbacks();
|
|
3835
|
+
expect(countType(frames, "turn_cancelled")).toBe(0);
|
|
3836
|
+
|
|
3837
|
+
await session.handleBinaryAudio(bargeInSpeechChunk);
|
|
3838
|
+
await waitFor(() => countType(frames, "turn_cancelled") === 1);
|
|
3839
|
+
await waitFor(() => abort.mock.calls.length === 1);
|
|
3840
|
+
});
|
|
3841
|
+
|
|
3842
|
+
test("echo suppression covers the client playback tail", async () => {
|
|
3843
|
+
const { frames, session, abort, speakFirstReply, completeFirstReply } =
|
|
3844
|
+
createSpeakingTurnHarness({
|
|
3845
|
+
bargeInMinSpeechMs: 60,
|
|
3846
|
+
echoBargeInMargin: 1.5,
|
|
3847
|
+
echoEmaHalfLifeMs: 40,
|
|
3848
|
+
ttsAudio: playbackReference,
|
|
3849
|
+
});
|
|
3850
|
+
await speakFirstReply();
|
|
3851
|
+
completeFirstReply();
|
|
3852
|
+
await waitFor(() => frames.some((frame) => frame.type === "tts_done"));
|
|
3853
|
+
const speechStartedBaseline = countType(frames, "speech_started");
|
|
3854
|
+
|
|
3855
|
+
for (let index = 0; index < 40; index += 1) {
|
|
3856
|
+
await session.handleBinaryAudio(playbackEchoChunk);
|
|
3857
|
+
}
|
|
3858
|
+
await flushAsyncCallbacks();
|
|
3859
|
+
|
|
3860
|
+
expect(countType(frames, "speech_started")).toBe(speechStartedBaseline);
|
|
3861
|
+
expect(countType(frames, "turn_cancelled")).toBe(0);
|
|
3862
|
+
expect(abort).not.toHaveBeenCalled();
|
|
3863
|
+
});
|
|
3864
|
+
|
|
3865
|
+
test("speech at playback onset cannot seed its own echo threshold", async () => {
|
|
3866
|
+
const { frames, session, abort, speakFirstReply, transcribers } =
|
|
3867
|
+
createSpeakingTurnHarness({
|
|
3868
|
+
bargeInMinSpeechMs: 250,
|
|
3869
|
+
echoBargeInMargin: 1.5,
|
|
3870
|
+
echoEmaHalfLifeMs: 400,
|
|
3871
|
+
ttsAudio: playbackReference,
|
|
3872
|
+
});
|
|
3873
|
+
await speakFirstReply();
|
|
3874
|
+
|
|
3875
|
+
const onsetSpeech = tonePcm(9_400, 530, 7_200);
|
|
3876
|
+
await session.handleBinaryAudio(onsetSpeech);
|
|
3877
|
+
|
|
3878
|
+
await waitFor(() => countType(frames, "turn_cancelled") === 1);
|
|
3879
|
+
await waitFor(() => abort.mock.calls.length === 1);
|
|
3880
|
+
expect(
|
|
3881
|
+
transcribers.some((transcriber) =>
|
|
3882
|
+
transcriber.received.some((buffer) =>
|
|
3883
|
+
buffer.equals(Buffer.from(onsetSpeech)),
|
|
3884
|
+
),
|
|
3885
|
+
),
|
|
3886
|
+
).toBe(true);
|
|
3887
|
+
});
|
|
3888
|
+
|
|
3889
|
+
test("speech already in progress bypasses playback warm-up", async () => {
|
|
3890
|
+
let callbacks: VoiceTurnCallbacks | undefined;
|
|
3891
|
+
const abort = mock();
|
|
3892
|
+
const startVoiceTurn = mock(async (options: VoiceTurnOptions) => {
|
|
3893
|
+
callbacks ??= options.callbacks;
|
|
3894
|
+
return { turnId: "bridge-turn", abort };
|
|
3895
|
+
});
|
|
3896
|
+
const streamTtsAudio = mock(async (options: LiveVoiceTtsOptions) => {
|
|
3897
|
+
options.onAudioChunk(makeTtsChunk("assistant audio"));
|
|
3898
|
+
return makeTtsResult("assistant audio");
|
|
3899
|
+
});
|
|
3900
|
+
const { frames, session } = createHarness({
|
|
3901
|
+
finals: ["what's the weather", "actually never mind"],
|
|
3902
|
+
startVoiceTurn,
|
|
3903
|
+
streamTtsAudio,
|
|
3904
|
+
bargeInMinSpeechMs: 60,
|
|
3905
|
+
echoBargeInMargin: 1.5,
|
|
3906
|
+
echoEmaHalfLifeMs: 400,
|
|
3907
|
+
echoDrainSlackMs: 60_000,
|
|
3908
|
+
turnDetectorConfig: { silenceThresholdMs: 5_000 },
|
|
3909
|
+
});
|
|
3910
|
+
|
|
3911
|
+
await session.start();
|
|
3912
|
+
await session.handleBinaryAudio(LOUD_CHUNK);
|
|
3913
|
+
await session.handleClientFrame({ type: "ptt_release" });
|
|
3914
|
+
await waitFor(() => frames.some((frame) => frame.type === "thinking"));
|
|
3915
|
+
for (let index = 0; index < 3; index += 1) {
|
|
3916
|
+
await session.handleBinaryAudio(pcm(3_000));
|
|
3917
|
+
}
|
|
3918
|
+
callbacks?.assistant_text_delta?.(makeTextDelta("It is sunny today."));
|
|
3919
|
+
await waitFor(() => frames.some((frame) => frame.type === "tts_audio"));
|
|
3920
|
+
for (let index = 0; index < 3; index += 1) {
|
|
3921
|
+
await session.handleBinaryAudio(pcm(3_000));
|
|
3922
|
+
}
|
|
3923
|
+
|
|
3924
|
+
await waitFor(() => countType(frames, "turn_cancelled") === 1);
|
|
3925
|
+
await waitFor(() => abort.mock.calls.length === 1);
|
|
3926
|
+
});
|
|
3927
|
+
});
|
|
3636
3928
|
});
|
|
3637
3929
|
|
|
3638
3930
|
describe("LiveVoiceSession unified front-door endpointing", () => {
|
|
@@ -3670,15 +3962,6 @@ describe("LiveVoiceSession unified front-door endpointing", () => {
|
|
|
3670
3962
|
return { startVoiceTurn, calls, discard };
|
|
3671
3963
|
}
|
|
3672
3964
|
|
|
3673
|
-
// A decider that phrases nothing: the front-door leg IS the endpoint
|
|
3674
|
-
// decision, so these tests only need the decider seam to exist.
|
|
3675
|
-
function makeSilentDecider(): VoiceFrontDecider {
|
|
3676
|
-
return {
|
|
3677
|
-
generateAckText: async () => null,
|
|
3678
|
-
generateProgressText: async () => null,
|
|
3679
|
-
};
|
|
3680
|
-
}
|
|
3681
|
-
|
|
3682
3965
|
async function startWithPartial(
|
|
3683
3966
|
session: LiveVoiceSession,
|
|
3684
3967
|
transcribers: MockStreamingTranscriber[],
|
|
@@ -3691,12 +3974,10 @@ describe("LiveVoiceSession unified front-door endpointing", () => {
|
|
|
3691
3974
|
}
|
|
3692
3975
|
|
|
3693
3976
|
test("a chatty answer commits: verdict leg replaces the decider, frames follow commit order", async () => {
|
|
3694
|
-
const decider = makeSilentDecider();
|
|
3695
3977
|
const starter = makeVerdictTurnStarter([["Hey! Not much."]]);
|
|
3696
3978
|
const { frames, session, transcribers } = createHarness({
|
|
3697
3979
|
finals: ["hello world"],
|
|
3698
3980
|
startVoiceTurn: starter.startVoiceTurn,
|
|
3699
|
-
frontDecider: decider,
|
|
3700
3981
|
});
|
|
3701
3982
|
|
|
3702
3983
|
await startWithPartial(session, transcribers);
|