@vellumai/assistant 0.11.2 → 0.11.3-dev.202608080427.92f668a
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +69 -9
- package/docs/flux-turn-detection-spike.md +243 -0
- package/docs/stt-provider-onboarding.md +3 -1
- package/docs/trusted-contact-access.md +9 -5
- package/docs/vellum-doctor.md +195 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/channels.ts +50 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/ingress.ts +10 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/remote-web-pairing.ts +6 -0
- package/node_modules/@vellumai/ces-client/src/index.ts +1 -0
- package/node_modules/@vellumai/ces-client/src/rpc-client.ts +19 -2
- package/node_modules/@vellumai/environments/package.json +2 -1
- package/node_modules/@vellumai/environments/src/__tests__/cloud-assistant-hub-url.test.ts +38 -0
- package/node_modules/@vellumai/environments/src/__tests__/install-layout.test.ts +98 -0
- package/node_modules/@vellumai/environments/src/__tests__/package-boundary.test.ts +5 -5
- package/node_modules/@vellumai/environments/src/index.ts +17 -6
- package/node_modules/@vellumai/environments/src/install-layout.ts +49 -0
- package/node_modules/@vellumai/environments/src/seeds.ts +29 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/channels.ts +50 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/ingress.ts +10 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/remote-web-pairing.ts +6 -0
- package/node_modules/@vellumai/gateway-client/src/index.ts +4 -0
- package/node_modules/@vellumai/gateway-client/src/verification-session-contract.ts +91 -6
- package/node_modules/@vellumai/ipc-server-utils/src/endpoint.test.ts +36 -0
- package/node_modules/@vellumai/ipc-server-utils/src/endpoint.ts +142 -0
- package/node_modules/@vellumai/ipc-server-utils/src/index.ts +2 -0
- package/node_modules/@vellumai/ipc-server-utils/src/listen-options.ts +3 -0
- package/node_modules/@vellumai/ipc-server-utils/src/socket-watchdog.test.ts +15 -1
- package/node_modules/@vellumai/ipc-server-utils/src/socket-watchdog.ts +17 -2
- package/node_modules/@vellumai/service-contracts/src/channels.ts +50 -0
- package/node_modules/@vellumai/service-contracts/src/ingress.ts +10 -0
- package/node_modules/@vellumai/service-contracts/src/remote-web-pairing.ts +6 -0
- package/node_modules/@vellumai/slack-text/src/index.ts +16 -2
- package/openapi.yaml +619 -47
- package/package.json +1 -1
- package/scripts/voice-ttft-spike.ts +3 -3
- package/src/__tests__/agent-image-optimize.test.ts +4 -4
- package/src/__tests__/agent-loop-mutable-latest-user-message.test.ts +113 -66
- package/src/__tests__/agent-loop-override-profile.test.ts +38 -48
- package/src/__tests__/agent-loop-subagent-billing-headers.test.ts +166 -0
- package/src/__tests__/agent-wake-disk-pressure-callsite.test.ts +1 -0
- package/src/__tests__/agent-wake-override-profile.test.ts +1 -0
- package/src/__tests__/answered-question-persistence.test.ts +240 -0
- package/src/__tests__/anthropic-provider.test.ts +165 -30
- package/src/__tests__/app-control-flow.test.ts +9 -8
- package/src/__tests__/assistant-event-hub.test.ts +48 -0
- package/src/__tests__/attachments-store-heic-normalize.test.ts +16 -12
- package/src/__tests__/attachments-store.test.ts +130 -68
- package/src/__tests__/attachments.test.ts +36 -36
- package/src/__tests__/background-workers-disk-pressure.test.ts +1 -1
- package/src/__tests__/call-controller.test.ts +120 -0
- package/src/__tests__/call-start-guardian-guard.test.ts +18 -4
- package/src/__tests__/canned-reply-release.test.ts +86 -0
- package/src/__tests__/channel-availability-routes.test.ts +14 -1
- package/src/__tests__/channel-guardian.test.ts +59 -29
- package/src/__tests__/channel-readiness-telegram-remote.test.ts +154 -0
- package/src/__tests__/channel-setup-panel-ack.test.ts +6 -5
- package/src/__tests__/client-os-metadata-persistence.test.ts +103 -4
- package/src/__tests__/compactor-image-manifest-trust.test.ts +1 -1
- package/src/__tests__/config-loader-backfill.test.ts +19 -0
- package/src/__tests__/config-schema.test.ts +150 -7
- package/src/__tests__/consult-deadline.test.ts +65 -0
- package/src/__tests__/conversation-agent-loop.test.ts +438 -2
- package/src/__tests__/conversation-disk-view-integration.test.ts +5 -1
- package/src/__tests__/conversation-disk-view.test.ts +4 -4
- package/src/__tests__/conversation-error.test.ts +22 -3
- package/src/__tests__/conversation-fork-crud.test.ts +5 -1
- package/src/__tests__/conversation-fork-referential.test.ts +352 -0
- package/src/__tests__/conversation-fork-retrospective.test.ts +5 -1
- package/src/__tests__/conversation-group-tools.test.ts +32 -2
- package/src/__tests__/conversation-inference-profile-route.test.ts +27 -1
- package/src/__tests__/conversation-initial-prompt.test.ts +51 -22
- package/src/__tests__/conversation-key-store-bootstrap-cleanup.test.ts +119 -0
- package/src/__tests__/conversation-key-store-origin.test.ts +105 -0
- package/src/__tests__/conversation-list-recency-ordering.test.ts +104 -0
- package/src/__tests__/conversation-origin-at-creation.test.ts +153 -0
- package/src/__tests__/conversation-origin-channel-filter.test.ts +95 -0
- package/src/__tests__/conversation-placement-promotion.test.ts +172 -0
- package/src/__tests__/conversation-process-app-control-preactivation.test.ts +123 -0
- package/src/__tests__/conversation-queue.test.ts +433 -1
- package/src/__tests__/conversation-retry-route.test.ts +3 -1
- package/src/__tests__/conversation-routes-hidden-queue.test.ts +64 -1
- package/src/__tests__/conversation-store-ephemeral.test.ts +4 -0
- package/src/__tests__/conversation-store.test.ts +26 -10
- package/src/__tests__/conversation-summarize-route.test.ts +1 -1
- package/src/__tests__/conversation-surfaces-action-delivery.test.ts +15 -6
- package/src/__tests__/conversation-surfaces-activation-emit.test.ts +6 -5
- package/src/__tests__/conversation-surfaces-app-control.test.ts +12 -11
- package/src/__tests__/conversation-surfaces-app-open.test.ts +11 -5
- package/src/__tests__/conversation-surfaces-data-persist.test.ts +5 -4
- package/src/__tests__/conversation-surfaces-history-restored-completion.test.ts +967 -0
- package/src/__tests__/conversation-surfaces-persisted-info.test.ts +312 -0
- package/src/__tests__/conversation-surfaces-queued-emit.test.ts +6 -5
- package/src/__tests__/conversation-surfaces-standalone-payloads.test.ts +12 -8
- package/src/__tests__/conversation-surfaces-standalone.test.ts +21 -11
- package/src/__tests__/conversation-surfaces-state-update.test.ts +7 -11
- package/src/__tests__/conversation-surfaces-table-action.test.ts +7 -6
- package/src/__tests__/conversation-surfaces-task-progress.test.ts +11 -6
- package/src/__tests__/conversation-tool-setup-app-refresh.test.ts +9 -6
- package/src/__tests__/conversation-tool-setup-attribution.test.ts +70 -7
- package/src/__tests__/conversation-wait-for-idle.test.ts +6 -11
- package/src/__tests__/credential-prompt-route.test.ts +54 -7
- package/src/__tests__/credential-security-invariants.test.ts +1 -0
- package/src/__tests__/cu-unified-flow.test.ts +89 -28
- package/src/__tests__/custom-group-listing-visibility.test.ts +1 -1
- package/src/__tests__/db-schedule-syntax-migration.test.ts +4 -1
- package/src/__tests__/discord-access-request-privacy.test.ts +366 -0
- package/src/__tests__/discord-requester-notice-privacy.test.ts +87 -0
- package/src/__tests__/disk-pressure-guard.test.ts +55 -55
- package/src/__tests__/disk-pressure-routes.test.ts +10 -10
- package/src/__tests__/disk-pressure-tools.test.ts +2 -2
- package/src/__tests__/disk-usage.test.ts +35 -20
- package/src/__tests__/document-sync-tags.test.ts +379 -0
- package/src/__tests__/document-update-default-surface.test.ts +28 -0
- package/src/__tests__/dynamic-skill-background-guard.test.ts +2 -2
- package/src/__tests__/emit-signal-routing-intent.test.ts +3 -4
- package/src/__tests__/ensure-conversation-exists.test.ts +8 -5
- package/src/__tests__/events-dev-bypass-actor.test.ts +294 -1
- package/src/__tests__/evict-conversations-for-reload.test.ts +90 -0
- package/src/__tests__/file-list-tool.test.ts +12 -10
- package/src/__tests__/file-ops-service.test.ts +151 -44
- package/src/__tests__/file-read-container-boundary.test.ts +137 -0
- package/src/__tests__/file-read-tool.test.ts +21 -9
- package/src/__tests__/file-url-path-guard.test.ts +83 -0
- package/src/__tests__/filesystem-tools.test.ts +82 -66
- package/src/__tests__/guardian-outbound-http.test.ts +8 -8
- package/src/__tests__/helpers/gateway-verification-sessions-stub.ts +14 -1
- package/src/__tests__/helpers/hub-clients.ts +42 -0
- package/src/__tests__/helpers/mock-conversation.ts +46 -0
- package/src/__tests__/helpers/verification-sessions-ipc-sim.ts +61 -57
- package/src/__tests__/history-repair.test.ts +112 -12
- package/src/__tests__/host-bash-proxy.test.ts +10 -5
- package/src/__tests__/host-bash-routes.test.ts +67 -0
- package/src/__tests__/host-browser-proxy.test.ts +2 -2
- package/src/__tests__/host-cu-proxy.test.ts +3 -7
- package/src/__tests__/host-file-proxy.test.ts +3 -7
- package/src/__tests__/host-file-read-tool.test.ts +5 -1
- package/src/__tests__/host-transfer-proxy-targeted.test.ts +6 -4
- package/src/__tests__/http-conversation-lineage.test.ts +2 -5
- package/src/__tests__/http-user-message-parity.test.ts +44 -0
- package/src/__tests__/identity-routes.test.ts +13 -13
- package/src/__tests__/image-conversion.test.ts +29 -26
- package/src/__tests__/image-source-path-reinject.test.ts +2 -2
- package/src/__tests__/inference-profile-session-handler.test.ts +95 -0
- package/src/__tests__/inference-profile-session-ipc.test.ts +25 -0
- package/src/__tests__/introduction-card-resolver.test.ts +6 -1
- package/src/__tests__/list-messages-attachments.test.ts +56 -32
- package/src/__tests__/list-messages-background-tool-completion.test.ts +4 -4
- package/src/__tests__/list-messages-client-message-id.test.ts +2 -2
- package/src/__tests__/list-messages-hidden-metadata.test.ts +16 -16
- package/src/__tests__/list-messages-page-latest.test.ts +61 -61
- package/src/__tests__/list-messages-provider-error.test.ts +6 -6
- package/src/__tests__/list-messages-queued.test.ts +49 -6
- package/src/__tests__/list-messages-system-card.test.ts +4 -4
- package/src/__tests__/list-messages-tool-merge.test.ts +9 -9
- package/src/__tests__/llm-resolver-override-or-default.test.ts +141 -0
- package/src/__tests__/llm-usage-store.test.ts +55 -0
- package/src/__tests__/manual-token-reconciliation.test.ts +45 -0
- package/src/__tests__/mcp-list-plugin-servers.test.ts +250 -0
- package/src/__tests__/mcp-tool-annotations-risk.test.ts +133 -0
- package/src/__tests__/media-stream-output.test.ts +175 -0
- package/src/__tests__/media-stream-server-integration.test.ts +36 -0
- package/src/__tests__/media-stream-stt-session.test.ts +67 -0
- package/src/__tests__/migration-export-http.test.ts +1 -1
- package/src/__tests__/migration-import-commit-http.test.ts +1 -1
- package/src/__tests__/migration-import-preflight-http.test.ts +1 -1
- package/src/__tests__/migration-validate-http.test.ts +1 -1
- package/src/__tests__/mtime-cache.test.ts +65 -3
- package/src/__tests__/notification-platform-adapter.test.ts +104 -0
- package/src/__tests__/notification-schedule-notify-dedup.test.ts +1 -1
- package/src/__tests__/oauth-provider-profiles.test.ts +4 -2
- package/src/__tests__/oauth-provider-seed-logos.test.ts +65 -0
- package/src/__tests__/oauth-provider-serializer.test.ts +17 -0
- package/src/__tests__/oauth-providers-routes.test.ts +1 -0
- package/src/__tests__/openai-responses-prompt-cache.test.ts +5 -2
- package/src/__tests__/path-policy.test.ts +164 -1
- package/src/__tests__/persist-media-references.test.ts +2 -2
- package/src/__tests__/persist-unsendable-image-downscale.test.ts +5 -5
- package/src/__tests__/persist-unsendable-image.test.ts +23 -23
- package/src/__tests__/persona-resolver.test.ts +154 -0
- package/src/__tests__/plugin-api-webhook-url.test.ts +170 -0
- package/src/__tests__/plugin-import-boundary-guard.test.ts +6 -0
- package/src/__tests__/processing-flag-persist-failure.test.ts +15 -10
- package/src/__tests__/profile-availability-incomplete.test.ts +113 -0
- package/src/__tests__/profiler-routes.test.ts +13 -13
- package/src/__tests__/prompt-cache-cross-turn-stability.test.ts +593 -0
- package/src/__tests__/proxy-approval-callback.test.ts +3 -3
- package/src/__tests__/queued-message-cancel-and-steer.test.ts +189 -0
- package/src/__tests__/queued-message-steer-actor-scoping.test.ts +164 -0
- package/src/__tests__/resolve-trust-class.test.ts +20 -0
- package/src/__tests__/run-conversation-turn-persistence.test.ts +1 -1
- package/src/__tests__/run-due-schedules.test.ts +174 -0
- package/src/__tests__/runtime-attachment-metadata.test.ts +15 -7
- package/src/__tests__/scaffold-managed-skill-tool.test.ts +166 -0
- package/src/__tests__/schedule-routes.test.ts +730 -8
- package/src/__tests__/schedule-store.test.ts +540 -0
- package/src/__tests__/schedule-tools.test.ts +103 -0
- package/src/__tests__/scheduler-wake.test.ts +5 -0
- package/src/__tests__/secret-routes-platform-proxy.test.ts +29 -1
- package/src/__tests__/secure-keys.test.ts +91 -0
- package/src/__tests__/server-history-render.test.ts +97 -7
- package/src/__tests__/size-guard.test.ts +8 -8
- package/src/__tests__/skill-docs-sync-guard.test.ts +3 -2
- package/src/__tests__/skill-tool-factory.test.ts +247 -0
- package/src/__tests__/skills-install-staging.test.ts +15 -6
- package/src/__tests__/skills.test.ts +32 -0
- package/src/__tests__/slack-edit-ordering-characterization.test.ts +103 -0
- package/src/__tests__/slack-inbound-verification.test.ts +4 -1
- package/src/__tests__/slack-mention-binding-invariance.test.ts +103 -0
- package/src/__tests__/slack-mention-provider-leak-audit.test.ts +135 -0
- package/src/__tests__/starter-task-flow.test.ts +5 -4
- package/src/__tests__/steer-on-enqueue-question.test.ts +3 -2
- package/src/__tests__/stt-language-catalog-parity.test.ts +38 -0
- package/src/__tests__/subagent-allowlist-validation.test.ts +26 -4
- package/src/__tests__/subagent-cron-run-attribution.test.ts +301 -0
- package/src/__tests__/subagent-disposal.test.ts +10 -0
- package/src/__tests__/subagent-fork-notifications.test.ts +10 -0
- package/src/__tests__/subagent-fork-prompt-role.test.ts +85 -11
- package/src/__tests__/subagent-fork-spawn.test.ts +15 -7
- package/src/__tests__/subagent-manager-notify.test.ts +148 -0
- package/src/__tests__/subagent-notify-parent.test.ts +1 -1
- package/src/__tests__/subagent-persistence.test.ts +141 -0
- package/src/__tests__/subagent-role-registry.test.ts +172 -63
- package/src/__tests__/subagent-role-resolution.test.ts +88 -0
- package/src/__tests__/subagent-spawn-and-await.test.ts +48 -0
- package/src/__tests__/subagent-spawn-tool-fork.test.ts +111 -10
- package/src/__tests__/subagent-terminal-message.test.ts +88 -1
- package/src/__tests__/subagent-tool-gate-mode.test.ts +225 -10
- package/src/__tests__/subagent-tools.test.ts +1901 -108
- package/src/__tests__/surface-completion-compaction-boundary.test.ts +124 -0
- package/src/__tests__/surface-completion-in-flight-snapshot.test.ts +298 -0
- package/src/__tests__/system-prompt.test.ts +82 -7
- package/src/__tests__/tool-grant-request-escalation.test.ts +40 -1
- package/src/__tests__/tool-result-spool.test.ts +74 -18
- package/src/__tests__/tool-side-effects-documents.test.ts +174 -0
- package/src/__tests__/tool-side-effects-hook-failures.test.ts +129 -0
- package/src/__tests__/tools-audio-read.test.ts +10 -10
- package/src/__tests__/tools-get-route.test.ts +85 -12
- package/src/__tests__/tools-list-cli.test.ts +171 -0
- package/src/__tests__/top-level-renderer.test.ts +54 -1
- package/src/__tests__/trusted-contact-inline-approval-integration.test.ts +68 -1
- package/src/__tests__/trusted-contact-multichannel.test.ts +4 -1
- package/src/__tests__/turn-tail-chain.test.ts +135 -0
- package/src/__tests__/ui-choice-copy-surfaces.test.ts +6 -4
- package/src/__tests__/ui-visual-surface.test.ts +5 -4
- package/src/__tests__/ui-work-result-surface.test.ts +5 -4
- package/src/__tests__/unified-turn-context-visible-app.test.ts +7 -8
- package/src/__tests__/usage-routes.test.ts +62 -0
- package/src/__tests__/verification-outbound-delivery.test.ts +176 -0
- package/src/__tests__/visible-app-context.test.ts +20 -4
- package/src/__tests__/web-search-history.test.ts +96 -0
- package/src/__tests__/workspace-migration-139-clear-renamed-cost-profile-label.test.ts +6 -6
- package/src/__tests__/workspace-migration-140-repair-seed-pinned-memory-v3-live.test.ts +330 -0
- package/src/__tests__/workspace-migration-142-consolidate-voice-front-door.test.ts +150 -0
- package/src/acp/failure-error.ts +3 -5
- package/src/agent/attachments.ts +26 -24
- package/src/agent/history-repair/history-repair.ts +27 -26
- package/src/agent/image-optimize.ts +6 -6
- package/src/agent/loop.ts +40 -22
- package/src/agent/message-types.ts +3 -3
- package/src/api/constants/document-tools.ts +55 -0
- package/src/api/events/question-answered.ts +53 -0
- package/src/api/events/tool-result.ts +7 -0
- package/src/api/index.ts +13 -0
- package/src/api/responses/conversation-message.ts +11 -0
- package/src/api/surfaces-oauth-connect.test.ts +96 -0
- package/src/api/surfaces.ts +36 -8
- package/src/approvals/guardian-channel-delivery.ts +83 -0
- package/src/approvals/guardian-expiry-notifier.ts +13 -11
- package/src/approvals/guardian-request-resolvers.ts +222 -135
- package/src/calls/__tests__/tts-text-sanitizer.test.ts +13 -0
- package/src/calls/__tests__/voice-session-bridge.test.ts +131 -12
- package/src/calls/__tests__/voice-triage-escalate.test.ts +106 -0
- package/src/calls/call-controller.ts +30 -2
- package/src/calls/call-speech-output.ts +12 -4
- package/src/calls/call-transport.ts +18 -1
- package/src/calls/media-stream-output.ts +53 -5
- package/src/calls/media-stream-server.ts +21 -1
- package/src/calls/media-stream-stt-session.ts +49 -0
- package/src/calls/telephony-synthesis-language.ts +84 -0
- package/src/calls/tts-text-sanitizer.ts +12 -5
- package/src/calls/voice-session-bridge.ts +86 -17
- package/src/calls/voice-triage-escalate.ts +54 -7
- package/src/channels/__tests__/gateway-verification-sessions.test.ts +3 -3
- package/src/channels/__tests__/plugin-channel-declarations.test.ts +161 -0
- package/src/channels/config.ts +20 -2
- package/src/channels/gateway-verification-sessions.ts +7 -2
- package/src/channels/plugin-channel-declarations.ts +108 -0
- package/src/channels/types.ts +37 -1
- package/src/cli/AGENTS.md +5 -2
- package/src/cli/commands/__tests__/cache.test.ts +7 -4
- package/src/cli/commands/__tests__/cli-test-harness.ts +82 -0
- package/src/cli/commands/__tests__/conversations-wake.test.ts +51 -6
- package/src/cli/commands/__tests__/inference-models.test.ts +14 -0
- package/src/cli/commands/__tests__/inference-profiles.test.ts +44 -1
- package/src/cli/commands/__tests__/inference-providers.test.ts +7 -0
- package/src/cli/commands/__tests__/plugins.test.ts +692 -0
- package/src/cli/commands/__tests__/schedules.test.ts +136 -37
- package/src/cli/commands/conversations.ts +7 -3
- package/src/cli/commands/credentials.help.ts +7 -0
- package/src/cli/commands/inference-profiles.ts +9 -3
- package/src/cli/commands/inference.help.ts +16 -2
- package/src/cli/commands/mcp.help.ts +17 -6
- package/src/cli/commands/mcp.ts +9 -0
- package/src/cli/commands/memory/memory-retrospective.ts +18 -0
- package/src/cli/commands/notifications.ts +3 -6
- package/src/cli/commands/oauth/connect-surface-guidance.test.ts +16 -0
- package/src/cli/commands/oauth/connect-surface-guidance.ts +25 -7
- package/src/cli/commands/oauth/connect.test.ts +26 -1
- package/src/cli/commands/oauth/connect.ts +4 -1
- package/src/cli/commands/oauth/index.help.ts +3 -1
- package/src/cli/commands/oauth/providers.ts +3 -0
- package/src/cli/commands/plugins.help.ts +25 -2
- package/src/cli/commands/plugins.ts +289 -10
- package/src/cli/commands/schedules.help.ts +14 -12
- package/src/cli/commands/schedules.ts +19 -5
- package/src/cli/commands/stt.help.ts +27 -2
- package/src/cli/commands/tools.help.ts +4 -2
- package/src/cli/commands/tools.ts +38 -0
- package/src/cli/lib/__tests__/inspect-plugin.test.ts +1 -0
- package/src/cli/lib/__tests__/install-from-github.test.ts +79 -0
- package/src/cli/lib/__tests__/install-from-platform.test.ts +47 -1
- package/src/cli/lib/__tests__/plugin-surfaces.test.ts +86 -7
- package/src/cli/lib/__tests__/upgrade-plugin.test.ts +161 -0
- package/src/cli/lib/install-from-github.ts +61 -0
- package/src/cli/lib/install-from-platform.ts +13 -2
- package/src/cli/lib/plugin-surfaces.ts +124 -4
- package/src/cli/lib/upgrade-plugin.ts +60 -0
- package/src/cli.ts +1 -1
- package/src/config/__tests__/default-profile-catalog.test.ts +11 -1
- package/src/config/__tests__/deployment-context-defaults.test.ts +73 -0
- package/src/config/__tests__/memory-retrospective-schema.test.ts +40 -0
- package/src/config/__tests__/plugin-updates-schema.test.ts +43 -0
- package/src/config/__tests__/profile-tool-support.test.ts +131 -0
- package/src/config/bundled-skills/AGENTS.md +1 -1
- package/src/config/bundled-skills/acp/SKILL.md +6 -7
- package/src/config/bundled-skills/app-builder/SKILL.md +1 -1
- package/src/config/bundled-skills/app-builder/references/RESPONSIVE.md +3 -3
- package/src/config/bundled-skills/media-processing/services/preprocess.ts +14 -4
- package/src/config/bundled-skills/phone-calls/references/CONFIG.md +15 -15
- package/src/config/bundled-skills/schedule/SKILL.md +5 -1
- package/src/config/bundled-skills/schedule/TOOLS.json +2 -2
- package/src/config/bundled-skills/settings/TOOLS.json +3 -3
- package/src/config/bundled-skills/settings/tools/voice-config-update.ts +1 -0
- package/src/config/bundled-skills/subagent/SKILL.md +53 -22
- package/src/config/bundled-skills/subagent/TOOLS.json +14 -13
- package/src/config/bundled-skills/transcribe/tools/transcribe-media.test.ts +22 -1
- package/src/config/bundled-skills/transcribe/tools/transcribe-media.ts +9 -2
- package/src/config/bundled-skills/visualize/SKILL.md +1 -1
- package/src/config/call-site-defaults.ts +4 -5
- package/src/config/default-profile-catalog.ts +9 -5
- package/src/config/feature-flag-registry.json +25 -17
- package/src/config/llm-resolver.ts +45 -15
- package/src/config/loader.ts +75 -1
- package/src/config/profile-tool-support.ts +110 -0
- package/src/config/sanitize-for-transfer.ts +11 -0
- package/src/config/schema.ts +4 -0
- package/src/config/schemas/__tests__/live-voice.test.ts +142 -4
- package/src/config/schemas/__tests__/stt.test.ts +20 -5
- package/src/config/schemas/call-site-catalog.ts +4 -4
- package/src/config/schemas/calls.ts +0 -4
- package/src/config/schemas/channels.ts +13 -0
- package/src/config/schemas/live-voice.ts +82 -23
- package/src/config/schemas/llm.ts +1 -1
- package/src/config/schemas/memory-retrospective.ts +25 -0
- package/src/config/schemas/plugin-updates.ts +59 -0
- package/src/config/schemas/services.ts +8 -4
- package/src/config/schemas/stt.ts +18 -5
- package/src/config/schemas/timeouts.ts +1 -1
- package/src/config/schemas/tts.ts +63 -0
- package/src/contacts/__tests__/guardian-delivery-reader.test.ts +16 -0
- package/src/contacts/guardian-delivery-reader.ts +29 -2
- package/src/context/compactor.ts +4 -4
- package/src/context/post-turn-tool-result-truncation.ts +32 -16
- package/src/context/strip-injections.ts +42 -0
- package/src/context/tool-result-spool.ts +32 -18
- package/src/credential-execution/ces-runtime.ts +3 -0
- package/src/credential-execution/client.ts +22 -1
- package/src/credential-execution/process-manager.test.ts +99 -0
- package/src/credential-execution/process-manager.ts +62 -16
- package/src/daemon/__tests__/abort-null-controller.test.ts +141 -5
- package/src/daemon/__tests__/conversation-surfaces-launch.test.ts +8 -12
- package/src/daemon/__tests__/install-assistant-command.test.ts +234 -0
- package/src/daemon/__tests__/turn-tail-assistant-reply-notify.test.ts +2 -4
- package/src/daemon/__tests__/turn-tail-deleted-conversation.test.ts +20 -29
- package/src/daemon/channel-ui-capability.ts +4 -4
- package/src/daemon/conversation-agent-loop-handlers.ts +117 -61
- package/src/daemon/conversation-agent-loop.ts +318 -148
- package/src/daemon/conversation-attachments.ts +1 -1
- package/src/daemon/conversation-error.ts +8 -1
- package/src/daemon/conversation-initial-prompt.ts +16 -38
- package/src/daemon/conversation-lifecycle.ts +154 -32
- package/src/daemon/conversation-messaging.ts +106 -43
- package/src/daemon/conversation-process.ts +131 -33
- package/src/daemon/conversation-queue-manager.ts +10 -0
- package/src/daemon/conversation-runtime-assembly.ts +9 -4
- package/src/daemon/conversation-store.ts +52 -3
- package/src/daemon/conversation-surface-state.ts +9 -0
- package/src/daemon/conversation-surfaces.ts +403 -210
- package/src/daemon/conversation-tool-setup.ts +56 -7
- package/src/daemon/conversation-turn-finalize.ts +134 -89
- package/src/daemon/conversation.ts +111 -27
- package/src/daemon/disk-pressure-guard-lifecycle.ts +2 -2
- package/src/daemon/disk-pressure-guard.ts +3 -3
- package/src/daemon/doordash-steps.ts +5 -5
- package/src/daemon/external-plugins-bootstrap.ts +7 -1
- package/src/daemon/handlers/config-channels.ts +48 -5
- package/src/daemon/handlers/conversation-history.ts +1 -1
- package/src/daemon/handlers/conversations.ts +62 -17
- package/src/daemon/handlers/shared.ts +12 -0
- package/src/daemon/handlers/skills.ts +11 -1
- package/src/daemon/identity-helpers.ts +38 -10
- package/src/daemon/install-assistant-command.ts +322 -0
- package/src/daemon/lifecycle.ts +14 -3
- package/src/daemon/message-types/conversations.ts +6 -7
- package/src/daemon/message-types/sync.ts +1 -0
- package/src/daemon/persist-media-references.ts +23 -14
- package/src/daemon/process-message.ts +6 -6
- package/src/daemon/tool-setup-types.ts +19 -122
- package/src/daemon/tool-side-effects.ts +17 -94
- package/src/daemon/trust-context-types.ts +39 -0
- package/src/daemon/trust-context.ts +14 -6
- package/src/daemon/turn-tail-chain.ts +73 -0
- package/src/daemon/web-search-history.ts +47 -33
- package/src/heartbeat/heartbeat-service.ts +6 -6
- package/src/inbound/__tests__/oauth-callback-url.test.ts +83 -0
- package/src/inbound/oauth-callback-url.ts +61 -0
- package/src/ipc/__tests__/socket-path.test.ts +21 -0
- package/src/ipc/assistant-server.ts +17 -10
- package/src/ipc/cli-client.ts +10 -1
- package/src/ipc/routes/__tests__/documents-sync-ipc-routes.test.ts +56 -0
- package/src/ipc/routes/documents-sync-ipc-routes.ts +42 -0
- package/src/ipc/socket-cleanup.ts +9 -7
- package/src/ipc/socket-path.ts +4 -96
- package/src/live-voice/__tests__/live-voice-agent-turn.test.ts +1 -104
- package/src/live-voice/__tests__/live-voice-archive.test.ts +16 -16
- package/src/live-voice/__tests__/live-voice-attach-image.test.ts +209 -0
- package/src/live-voice/__tests__/live-voice-connection.test.ts +20 -0
- package/src/live-voice/__tests__/live-voice-events.test.ts +7 -8
- package/src/live-voice/__tests__/live-voice-flux-turn-end.test.ts +932 -0
- package/src/live-voice/__tests__/live-voice-integration.test.ts +12 -5
- package/src/live-voice/__tests__/live-voice-metrics.test.ts +115 -8
- package/src/live-voice/__tests__/live-voice-photo.test.ts +100 -0
- package/src/live-voice/__tests__/live-voice-progress.test.ts +179 -146
- package/src/live-voice/__tests__/live-voice-session-telemetry.test.ts +265 -0
- package/src/live-voice/__tests__/live-voice-stt.test.ts +324 -2
- package/src/live-voice/__tests__/live-voice-triage-escalate.test.ts +131 -2
- package/src/live-voice/__tests__/live-voice-tts-session.test.ts +0 -483
- package/src/live-voice/__tests__/live-voice-tts.test.ts +148 -2
- package/src/live-voice/__tests__/live-voice-vad.test.ts +313 -32
- package/src/live-voice/__tests__/progress-narration.test.ts +214 -0
- package/src/live-voice/__tests__/progress-phrases.test.ts +167 -0
- package/src/live-voice/__tests__/protocol.test.ts +52 -0
- package/src/live-voice/__tests__/runtime-websocket-shell.test.ts +3 -0
- package/src/live-voice/live-voice-archive.ts +13 -13
- package/src/live-voice/live-voice-connection.ts +7 -1
- package/src/live-voice/live-voice-metrics.ts +57 -32
- package/src/live-voice/live-voice-photo.ts +168 -0
- package/src/live-voice/live-voice-session.ts +1137 -351
- package/src/live-voice/live-voice-tts.ts +18 -2
- package/src/live-voice/progress-narration.ts +277 -0
- package/src/live-voice/progress-phrases.ts +105 -2
- package/src/live-voice/protocol.ts +110 -2
- package/src/mcp/__tests__/mcp-auth-orchestrator.test.ts +0 -1
- package/src/mcp/__tests__/mcp-oauth-client-registration.test.ts +196 -0
- package/src/mcp/__tests__/mcp-oauth-provider.test.ts +9 -9
- package/src/mcp/client.ts +14 -7
- package/src/mcp/mcp-auth-orchestrator.ts +13 -22
- package/src/mcp/mcp-oauth-provider.ts +195 -240
- package/src/messaging/providers/__tests__/callback-routing.test.ts +5 -1
- package/src/messaging/providers/__tests__/transport-dispatch.test.ts +66 -4
- package/src/messaging/providers/callback-routing.ts +1 -0
- package/src/messaging/providers/discord/api.ts +312 -0
- package/src/messaging/providers/discord/dm-delivery.test.ts +204 -0
- package/src/messaging/providers/discord/render.test.ts +204 -0
- package/src/messaging/providers/discord/render.ts +195 -0
- package/src/messaging/providers/discord/send.test.ts +179 -0
- package/src/messaging/providers/discord/send.ts +212 -0
- package/src/messaging/providers/discord/transport.ts +81 -0
- package/src/messaging/providers/index.ts +3 -1
- package/src/messaging/providers/retry-policy.test.ts +247 -0
- package/src/messaging/providers/retry-policy.ts +206 -0
- package/src/messaging/providers/slack/mention-source.test.ts +285 -0
- package/src/messaging/providers/slack/mention-source.ts +287 -0
- package/src/messaging/providers/telegram-bot/api.ts +47 -130
- package/src/messaging/providers/whatsapp/api.ts +47 -136
- package/src/monitoring/__tests__/db-integrity-sample.test.ts +4 -1
- package/src/monitoring/__tests__/plugin-auto-update.test.ts +495 -0
- package/src/monitoring/db-integrity-sample.ts +6 -1
- package/src/monitoring/plugin-auto-update.ts +463 -0
- package/src/monitoring/resource-sampler.ts +21 -5
- package/src/monitoring/worker.ts +13 -0
- package/src/notifications/__tests__/assistant-reply-producer.test.ts +375 -2
- package/src/notifications/__tests__/broadcaster.test.ts +37 -1
- package/src/notifications/__tests__/copy-composer.test.ts +165 -0
- package/src/notifications/__tests__/emit-signal-pipeline-failure.test.ts +205 -0
- package/src/notifications/__tests__/notification-utils.test.ts +249 -0
- package/src/notifications/adapters/platform.ts +31 -2
- package/src/notifications/assistant-reply-producer.ts +79 -32
- package/src/notifications/broadcaster.ts +30 -10
- package/src/notifications/copy-composer.ts +69 -0
- package/src/notifications/emit-signal.ts +82 -15
- package/src/notifications/events-store.ts +10 -2
- package/src/notifications/home-feed-side-effect.ts +6 -3
- package/src/notifications/notification-utils.ts +173 -5
- package/src/notifications/signal.ts +13 -0
- package/src/oauth/AGENTS.md +4 -2
- package/src/oauth/byo-connection.ts +6 -1
- package/src/oauth/credential-token-resolver.ts +3 -10
- package/src/oauth/manual-token-connection.ts +29 -64
- package/src/oauth/manual-token-providers.ts +62 -0
- package/src/oauth/provider-serializer.ts +13 -0
- package/src/oauth/seed-providers.ts +62 -11
- package/src/onboarding/onboarding-events-store.ts +91 -1
- package/src/permissions/question-prompter.test.ts +4 -0
- package/src/permissions/question-prompter.ts +21 -4
- package/src/persistence/__tests__/conversation-lineage.test.ts +248 -0
- package/src/persistence/attachments-store.ts +15 -15
- package/src/persistence/conversation-attention-store.ts +30 -10
- package/src/persistence/conversation-bootstrap.ts +17 -0
- package/src/persistence/conversation-crud.ts +363 -97
- package/src/persistence/conversation-group-migration.ts +1 -1
- package/src/persistence/conversation-key-store.ts +38 -10
- package/src/persistence/conversation-lineage.ts +226 -0
- package/src/persistence/conversation-plugin-facade.ts +28 -3
- package/src/persistence/conversation-queries.ts +207 -33
- package/src/persistence/conversation-types.ts +107 -1
- package/src/persistence/jobs-store.ts +32 -8
- package/src/persistence/llm-usage-store.ts +22 -0
- package/src/persistence/migrations/361-normalize-managed-connection-rows.test.ts +168 -0
- package/src/persistence/migrations/361-normalize-managed-connection-rows.ts +76 -0
- package/src/persistence/migrations/362-add-conversation-subagent-kind.test.ts +187 -0
- package/src/persistence/migrations/362-add-conversation-subagent-kind.ts +93 -0
- package/src/persistence/migrations/363-backfill-schedule-inference-profile.test.ts +229 -0
- package/src/persistence/migrations/363-backfill-schedule-inference-profile.ts +127 -0
- package/src/persistence/migrations/364-add-schedule-source-key.test.ts +120 -0
- package/src/persistence/migrations/364-add-schedule-source-key.ts +43 -0
- package/src/persistence/migrations/365-add-conversation-fork-strategy.ts +27 -0
- package/src/persistence/schema/conversations.ts +32 -0
- package/src/persistence/schema/infrastructure.ts +3 -0
- package/src/persistence/steps.ts +30 -0
- package/src/persistence/subagent-store.ts +148 -6
- package/src/platform/managed-speech.test.ts +20 -0
- package/src/platform/managed-speech.ts +9 -0
- package/src/plugin-api/__tests__/oauth-callback-url-export.test.ts +29 -0
- package/src/plugin-api/index.ts +28 -5
- package/src/plugin-api/webhook-url.ts +151 -0
- package/src/plugins/__tests__/installed-plugin-dirs.test.ts +82 -0
- package/src/plugins/__tests__/mcp-servers.test.ts +370 -0
- package/src/plugins/__tests__/source-fingerprint.test.ts +5 -1
- package/src/plugins/collect-source-versions.ts +4 -27
- package/src/plugins/defaults/exploration-drift/hooks/post-tool-use.ts +5 -5
- package/src/plugins/defaults/exploration-drift/package.json +1 -1
- package/src/plugins/defaults/image-recovery/hooks/post-model-call.ts +4 -2
- package/src/plugins/defaults/image-recovery/recover.ts +67 -57
- package/src/plugins/defaults/index.ts +1 -1
- package/src/plugins/defaults/memory/__tests__/conversation-queries.test.ts +64 -18
- package/src/plugins/defaults/memory/__tests__/jobs-worker-outcome-contract.test.ts +242 -0
- package/src/plugins/defaults/memory/__tests__/jobs-worker-retrospective-sweep.test.ts +24 -1
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-enqueue.test.ts +48 -3
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-job.test.ts +511 -19
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-prompt.test.ts +39 -3
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-provider-path.test.ts +493 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-wake-chain.test.ts +5 -4
- package/src/plugins/defaults/memory/__tests__/memory-tier-boundary-guard.test.ts +5 -1
- package/src/plugins/defaults/memory/job-handlers.ts +87 -5
- package/src/plugins/defaults/memory/jobs-worker.ts +60 -6
- package/src/plugins/defaults/memory/memory-retrospective-constants.ts +11 -0
- package/src/plugins/defaults/memory/memory-retrospective-enqueue.ts +29 -0
- package/src/plugins/defaults/memory/memory-retrospective-job.ts +342 -60
- package/src/plugins/defaults/memory/memory-retrospective-prompt.ts +23 -3
- package/src/plugins/defaults/memory/substrate/consolidation-lock.ts +9 -4
- package/src/plugins/defaults/memory/substrate/skill-content.ts +8 -1
- package/src/plugins/defaults/memory/substrate/sweep-job.ts +7 -0
- package/src/plugins/defaults/memory/v1/graph/injection.ts +1 -1
- package/src/plugins/defaults/memory/v2/router.ts +8 -0
- package/src/plugins/defaults/turn-context/unified-turn-context.ts +1 -2
- package/src/plugins/external-plugin-loader.ts +64 -8
- package/src/plugins/installed-plugin-dirs.ts +101 -0
- package/src/plugins/mcp-servers.ts +343 -0
- package/src/plugins/mtime-cache.ts +34 -22
- package/src/plugins/types.ts +69 -8
- package/src/prompts/persona-resolver.ts +6 -2
- package/src/prompts/system-prompt.ts +72 -17
- package/src/providers/__tests__/inference.test.ts +40 -0
- package/src/providers/__tests__/preflight-resolved-config.test.ts +58 -1
- package/src/providers/__tests__/retry-callsite.test.ts +72 -2
- package/src/providers/__tests__/vellum-connection-routing.test.ts +42 -11
- package/src/providers/anthropic/client.ts +112 -95
- package/src/providers/connection-resolution.ts +35 -1
- package/src/providers/gemini/client.ts +5 -5
- package/src/providers/inference/adapter-factory.ts +16 -10
- package/src/providers/inference/auth.ts +16 -0
- package/src/providers/inference/connection-availability.ts +164 -12
- package/src/providers/inference/connections.ts +25 -1
- package/src/providers/media-resolve.ts +21 -11
- package/src/providers/model-catalog.ts +15 -0
- package/src/providers/openai/chat-completions-provider.ts +4 -4
- package/src/providers/openai/responses-provider.ts +12 -25
- package/src/providers/registry.ts +9 -4
- package/src/providers/retry.ts +25 -0
- package/src/providers/server-tool-pairing.ts +134 -0
- package/src/providers/speech-to-text/__tests__/deepgram-flux-frames.test.ts +433 -0
- package/src/providers/speech-to-text/__tests__/deepgram-flux-realtime.test.ts +620 -0
- package/src/providers/speech-to-text/__tests__/provider-catalog.test.ts +34 -0
- package/src/providers/speech-to-text/__tests__/resolve.test.ts +472 -12
- package/src/providers/speech-to-text/deepgram-flux-frames.ts +395 -0
- package/src/providers/speech-to-text/deepgram-flux-realtime.ts +719 -0
- package/src/providers/speech-to-text/deepgram-realtime.test.ts +283 -1
- package/src/providers/speech-to-text/deepgram-realtime.ts +117 -3
- package/src/providers/speech-to-text/deepgram.ts +1 -0
- package/src/providers/speech-to-text/provider-catalog.ts +137 -8
- package/src/providers/speech-to-text/resolve.ts +105 -12
- package/src/providers/speech-to-text/vellum-managed.ts +7 -1
- package/src/providers/speech-to-text/vellum-speech-relay-connection.ts +3 -1
- package/src/providers/types.ts +26 -6
- package/src/providers/vellum-model-routing.ts +6 -7
- package/src/runtime/AGENTS.md +2 -0
- package/src/runtime/__tests__/agent-wake.test.ts +16 -16
- package/src/runtime/__tests__/background-job-runner.test.ts +0 -144
- package/src/runtime/__tests__/desktop-presence.test.ts +162 -0
- package/src/runtime/__tests__/local-actor-identity-force-refresh.test.ts +104 -0
- package/src/runtime/agent-wake.ts +63 -24
- package/src/runtime/assistant-event-hub.ts +110 -12
- package/src/runtime/auth/__tests__/guard-tests.test.ts +2 -0
- package/src/runtime/auth/__tests__/scopes.test.ts +20 -0
- package/src/runtime/auth/same-actor.ts +56 -8
- package/src/runtime/auth/scopes.ts +12 -0
- package/src/runtime/auth/types.ts +2 -0
- package/src/runtime/background-job-runner.ts +5 -27
- package/src/runtime/channel-readiness-service.ts +59 -3
- package/src/runtime/channel-readiness-types.ts +18 -0
- package/src/runtime/desktop-presence.ts +69 -0
- package/src/runtime/http-server.ts +12 -0
- package/src/runtime/local-actor-identity.ts +29 -14
- package/src/runtime/migrations/__tests__/vbundle-symlink-streaming-importer.test.ts +0 -0
- package/src/runtime/routes/__tests__/channel-verification-routes.test.ts +1 -1
- package/src/runtime/routes/__tests__/client-routes.test.ts +186 -14
- package/src/runtime/routes/__tests__/conversation-list-routes.test.ts +547 -1
- package/src/runtime/routes/__tests__/conversation-management-routes.test.ts +162 -7
- package/src/runtime/routes/__tests__/conversation-query-routes.test.ts +14 -10
- package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +340 -3
- package/src/runtime/routes/__tests__/llm-call-sites-routes.test.ts +28 -1
- package/src/runtime/routes/__tests__/plugins-routes.test.ts +217 -1
- package/src/runtime/routes/__tests__/question-routes.test.ts +64 -0
- package/src/runtime/routes/__tests__/retrospective-routes.test.ts +20 -0
- package/src/runtime/routes/__tests__/skill-history-route.test.ts +116 -0
- package/src/runtime/routes/__tests__/sse-actor-principal-heal.test.ts +165 -0
- package/src/runtime/routes/__tests__/stt-routes.test.ts +25 -0
- package/src/runtime/routes/__tests__/surface-action-routes.test.ts +2 -0
- package/src/runtime/routes/__tests__/wake-conversation-routes.test.ts +60 -2
- package/src/runtime/routes/archive-utils.ts +16 -12
- package/src/runtime/routes/attachment-routes.ts +13 -6
- package/src/runtime/routes/canned-reply-release.ts +54 -0
- package/src/runtime/routes/channel-availability-routes.ts +32 -14
- package/src/runtime/routes/channel-verification-routes.ts +2 -27
- package/src/runtime/routes/chatgpt-subscription-auth-routes.ts +5 -0
- package/src/runtime/routes/client-routes.ts +114 -2
- package/src/runtime/routes/conversation-cli-routes.ts +10 -12
- package/src/runtime/routes/conversation-list-routes.ts +59 -10
- package/src/runtime/routes/conversation-management-routes.ts +47 -18
- package/src/runtime/routes/conversation-query-routes.ts +32 -53
- package/src/runtime/routes/conversation-routes.ts +360 -293
- package/src/runtime/routes/credential-prompt-routes.ts +20 -2
- package/src/runtime/routes/debug-routes.ts +4 -1
- package/src/runtime/routes/default-provider-routes.ts +7 -11
- package/src/runtime/routes/documents-routes.ts +44 -1
- package/src/runtime/routes/events-routes.ts +29 -1
- package/src/runtime/routes/host-app-control-routes.ts +2 -0
- package/src/runtime/routes/host-bash-routes.ts +2 -0
- package/src/runtime/routes/host-browser-routes.ts +2 -0
- package/src/runtime/routes/host-cu-routes.ts +2 -0
- package/src/runtime/routes/host-file-routes.ts +2 -0
- package/src/runtime/routes/host-transfer-routes.ts +4 -0
- package/src/runtime/routes/identity-routes.ts +6 -6
- package/src/runtime/routes/inbound-message-handler.ts +5 -5
- package/src/runtime/routes/inbound-stages/acl-enforcement.ts +7 -3
- package/src/runtime/routes/inbound-stages/background-dispatch.ts +14 -0
- package/src/runtime/routes/inbound-stages/bootstrap-intercept.ts +2 -2
- package/src/runtime/routes/inbound-stages/guardian-activation-intercept.test.ts +14 -11
- package/src/runtime/routes/inbound-stages/guardian-activation-intercept.ts +14 -17
- package/src/runtime/routes/inference-profile-availability-guard.ts +171 -0
- package/src/runtime/routes/inference-profile-session-handler.ts +26 -2
- package/src/runtime/routes/inference-profile-session-routes.ts +2 -1
- package/src/runtime/routes/inference-profiles-routes.ts +152 -73
- package/src/runtime/routes/inference-provider-connection-routes.ts +0 -16
- package/src/runtime/routes/llm-call-sites-routes.ts +13 -0
- package/src/runtime/routes/log-export-routes.ts +1 -1
- package/src/runtime/routes/mcp-auth-routes.ts +141 -57
- package/src/runtime/routes/notification-routes.ts +0 -14
- package/src/runtime/routes/oauth-providers.ts +1 -0
- package/src/runtime/routes/plugins-routes.ts +127 -4
- package/src/runtime/routes/profiler-routes.ts +4 -2
- package/src/runtime/routes/question-routes.ts +28 -10
- package/src/runtime/routes/retrospective-routes.ts +9 -3
- package/src/runtime/routes/schedule-routes.ts +210 -12
- package/src/runtime/routes/secret-routes.ts +40 -23
- package/src/runtime/routes/settings-routes.ts +41 -13
- package/src/runtime/routes/skills-routes.ts +79 -0
- package/src/runtime/routes/sse-actor-principal-heal.ts +113 -0
- package/src/runtime/routes/stt-routes.ts +31 -25
- package/src/runtime/routes/ui-snapshot-routes.ts +1 -0
- package/src/runtime/routes/wake-conversation-routes.ts +23 -0
- package/src/runtime/services/__tests__/conversation-serializer.test.ts +1 -0
- package/src/runtime/services/conversation-serializer.ts +24 -9
- package/src/runtime/sync/documents-sidecar-publish.test.ts +101 -0
- package/src/runtime/sync/resource-sync-events.ts +79 -1
- package/src/runtime/sync/worker-daemon-notify.test.ts +34 -0
- package/src/runtime/sync/worker-daemon-notify.ts +44 -6
- package/src/runtime/verification-outbound-actions.ts +250 -222
- package/src/runtime/verification-templates.ts +41 -30
- package/src/schedule/__tests__/plugin-schedule-declarations.test.ts +614 -0
- package/src/schedule/__tests__/plugin-schedule-reconciler.test.ts +1066 -0
- package/src/schedule/inference-profile.ts +64 -3
- package/src/schedule/plugin-schedule-declarations.ts +518 -0
- package/src/schedule/plugin-schedule-reconciler.ts +605 -0
- package/src/schedule/plugin-schedules-gate.ts +23 -0
- package/src/schedule/schedule-store.ts +520 -26
- package/src/schedule/scheduler.ts +38 -0
- package/src/security/credential-backend.ts +51 -4
- package/src/security/encrypted-store.ts +64 -42
- package/src/security/oauth2.ts +6 -22
- package/src/security/secure-keys.ts +55 -7
- package/src/skills/catalog-install.ts +20 -7
- package/src/skills/inline-command-runner.ts +2 -7
- package/src/skills/skill-history.test.ts +269 -0
- package/src/skills/skill-history.ts +250 -0
- package/src/skills/skillssh-registry.ts +1 -1
- package/src/stt/__tests__/daemon-batch-transcriber.test.ts +22 -0
- package/src/stt/__tests__/language-metadata.test.ts +85 -0
- package/src/stt/__tests__/speech-energy.test.ts +79 -0
- package/src/stt/__tests__/types.test.ts +94 -0
- package/src/stt/daemon-batch-transcriber.ts +18 -6
- package/src/stt/language-metadata.ts +65 -0
- package/src/stt/speech-energy.ts +115 -12
- package/src/stt/stt-stream-session.ts +8 -4
- package/src/stt/types.ts +119 -0
- package/src/subagent/__tests__/consult-prompt.test.ts +25 -0
- package/src/subagent/consult-prompt.ts +3 -1
- package/src/subagent/consult-transcript.ts +1 -1
- package/src/subagent/manager.ts +304 -44
- package/src/subagent/progress-events.ts +35 -0
- package/src/subagent/role-resolution.ts +88 -0
- package/src/subagent/types.ts +303 -67
- package/src/subagent/validate-allowlists.ts +3 -4
- package/src/telegram/__tests__/webhook-health.test.ts +73 -1
- package/src/telegram/webhook-health.ts +101 -3
- package/src/telemetry/__tests__/telemetry-event-fixtures.ts +2 -0
- package/src/telemetry/activation-funnel.ts +6 -1
- package/src/telemetry/live-voice-funnel.ts +75 -0
- package/src/telemetry/telemetry-event-sources.ts +6 -0
- package/src/telemetry/telemetry-wire-source.json +1 -1
- package/src/telemetry/telemetry-wire.generated.ts +2 -0
- package/src/telemetry/types.ts +23 -4
- package/src/telemetry/usage-telemetry-reporter.test.ts +4 -0
- package/src/tools/__tests__/subagent-workflow-tool-input-schemas.test.ts +1 -1
- package/src/tools/ask-question/ask-question-tool.test.ts +118 -3
- package/src/tools/ask-question/ask-question-tool.ts +56 -2
- package/src/tools/browser/browser-execution.ts +1 -1
- package/src/tools/calls/call-start.ts +8 -5
- package/src/tools/conversation-groups/move_to_group.ts +10 -12
- package/src/tools/document/document-tool.ts +11 -4
- package/src/tools/execution-timeout.ts +9 -2
- package/src/tools/executor.ts +7 -6
- package/src/tools/filesystem/edit.ts +1 -1
- package/src/tools/filesystem/list.ts +1 -1
- package/src/tools/filesystem/read.ts +7 -13
- package/src/tools/filesystem/search.ts +23 -21
- package/src/tools/filesystem/write.ts +4 -1
- package/src/tools/host-filesystem/edit.ts +7 -2
- package/src/tools/host-filesystem/read.ts +21 -8
- package/src/tools/host-filesystem/write.ts +10 -2
- package/src/tools/host-terminal/host-shell.ts +2 -2
- package/src/tools/mcp/mcp-tool-factory.ts +25 -1
- package/src/tools/permission-checker.ts +8 -3
- package/src/tools/schedule/delete.ts +13 -1
- package/src/tools/schedule/list.ts +14 -3
- package/src/tools/schedule/update.ts +56 -33
- package/src/tools/shared/filesystem/audio-read.ts +6 -4
- package/src/tools/shared/filesystem/file-ops-service.ts +176 -116
- package/src/tools/shared/filesystem/image-read.ts +12 -12
- package/src/tools/shared/filesystem/path-policy.ts +70 -5
- package/src/tools/shared/filesystem/size-guard.ts +6 -6
- package/src/tools/shared/input-misuse.ts +72 -0
- package/src/tools/skills/scaffold-managed.ts +82 -0
- package/src/tools/skills/skill-tool-factory.ts +14 -1
- package/src/tools/subagent/consult-deadline.ts +12 -7
- package/src/tools/subagent/message.ts +3 -1
- package/src/tools/subagent/read.ts +66 -4
- package/src/tools/subagent/spawn.ts +500 -78
- package/src/tools/tool-approval-handler.ts +45 -4
- package/src/tools/types.ts +22 -3
- package/src/tools/ui-surface/surface-shape-docs.ts +1 -1
- package/src/tools/ui-surface/visual-validation.ts +2 -0
- package/src/tts/__tests__/provider-adapters.test.ts +147 -0
- package/src/tts/__tests__/speakable-segments.test.ts +470 -0
- package/src/tts/language-voices.ts +23 -0
- package/src/tts/providers/deepgram-provider.ts +3 -1
- package/src/tts/providers/elevenlabs-provider.ts +73 -1
- package/src/tts/providers/xai-provider.ts +28 -2
- package/src/tts/speakable-segments.ts +293 -23
- package/src/tts/synthesis-stream.ts +7 -0
- package/src/tts/types.ts +7 -0
- package/src/usage/__tests__/subagent-attribution.test.ts +109 -0
- package/src/usage/subagent-attribution.ts +108 -0
- package/src/util/__tests__/language-subtag.test.ts +54 -0
- package/src/util/__tests__/sqlite-retry.test.ts +112 -0
- package/src/util/abort-reasons.ts +16 -0
- package/src/util/ansi.ts +44 -0
- package/src/util/disk-usage.ts +17 -13
- package/src/util/image-conversion.ts +20 -13
- package/src/util/language-subtag.ts +43 -0
- package/src/util/schedule-source-key.ts +23 -0
- package/src/util/unicode.ts +1 -1
- package/src/util/worker-process.ts +7 -1
- package/src/workspace/git-service.ts +33 -9
- package/src/workspace/migrations/140-repair-seed-pinned-memory-v3-live.ts +260 -0
- package/src/workspace/migrations/141-stt-english-default-to-multilingual.ts +111 -0
- package/src/workspace/migrations/142-consolidate-voice-front-door.ts +70 -0
- package/src/workspace/migrations/__tests__/141-stt-english-default-to-multilingual.test.ts +116 -0
- package/src/workspace/migrations/registry.ts +6 -0
- package/src/workspace/top-level-renderer.ts +15 -2
- package/src/__tests__/tool-side-effects-slack-dm.test.ts +0 -357
- package/src/daemon/install-symlink.ts +0 -207
- package/src/live-voice/__tests__/front-decision.test.ts +0 -525
- package/src/live-voice/front-decision.ts +0 -429
- package/src/notifications/deferred-emit.ts +0 -147
|
@@ -3,6 +3,8 @@ import { describe, expect, test } from "bun:test";
|
|
|
3
3
|
import {
|
|
4
4
|
DEFAULT_SPEECH_ENERGY_THRESHOLD,
|
|
5
5
|
detectPcm16SpeechActivity,
|
|
6
|
+
pcm16MaxNormalizedCorrelation,
|
|
7
|
+
pcm16MeanAmplitude,
|
|
6
8
|
} from "../speech-energy.js";
|
|
7
9
|
|
|
8
10
|
/** Build a PCM16LE buffer from an array of sample values. */
|
|
@@ -66,3 +68,80 @@ describe("detectPcm16SpeechActivity", () => {
|
|
|
66
68
|
expect(detectPcm16SpeechActivity(quiet, 500)).toBe(false);
|
|
67
69
|
});
|
|
68
70
|
});
|
|
71
|
+
|
|
72
|
+
describe("pcm16MeanAmplitude", () => {
|
|
73
|
+
test("returns 0 for an empty buffer", () => {
|
|
74
|
+
expect(pcm16MeanAmplitude(Buffer.alloc(0))).toBe(0);
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
test("returns the exact mean absolute amplitude", () => {
|
|
78
|
+
expect(pcm16MeanAmplitude(pcm16([1_000, -2_000, 3_000, -4_000]))).toBe(
|
|
79
|
+
2_500,
|
|
80
|
+
);
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
test("ignores a trailing odd byte", () => {
|
|
84
|
+
const samples = pcm16([3_000, -3_000]);
|
|
85
|
+
expect(
|
|
86
|
+
pcm16MeanAmplitude(Buffer.concat([samples, Buffer.from([0x01])])),
|
|
87
|
+
).toBe(3_000);
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
test("matches the detector's threshold comparison", () => {
|
|
91
|
+
const chunks = [
|
|
92
|
+
Buffer.alloc(0),
|
|
93
|
+
pcm16([0, 0]),
|
|
94
|
+
pcm16([500, -500]),
|
|
95
|
+
pcm16([DEFAULT_SPEECH_ENERGY_THRESHOLD]),
|
|
96
|
+
pcm16([DEFAULT_SPEECH_ENERGY_THRESHOLD + 1]),
|
|
97
|
+
];
|
|
98
|
+
for (const chunk of chunks) {
|
|
99
|
+
expect(detectPcm16SpeechActivity(chunk)).toBe(
|
|
100
|
+
pcm16MeanAmplitude(chunk) > DEFAULT_SPEECH_ENERGY_THRESHOLD,
|
|
101
|
+
);
|
|
102
|
+
}
|
|
103
|
+
});
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
describe("pcm16MaxNormalizedCorrelation", () => {
|
|
107
|
+
const wave = (frequency: number, count: number): Buffer =>
|
|
108
|
+
pcm16(
|
|
109
|
+
Array.from({ length: count }, (_, index) =>
|
|
110
|
+
Math.round(
|
|
111
|
+
8_000 * Math.sin((2 * Math.PI * frequency * index) / 16_000),
|
|
112
|
+
),
|
|
113
|
+
),
|
|
114
|
+
);
|
|
115
|
+
|
|
116
|
+
test("finds a matching waveform at an arbitrary reference offset", () => {
|
|
117
|
+
const input = wave(240, 800);
|
|
118
|
+
const reference = Buffer.concat([wave(410, 400), input, wave(610, 400)]);
|
|
119
|
+
|
|
120
|
+
expect(pcm16MaxNormalizedCorrelation(input, reference)).toBeGreaterThan(
|
|
121
|
+
0.99,
|
|
122
|
+
);
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
test("is invariant to gain and polarity", () => {
|
|
126
|
+
const inputSamples = Array.from({ length: 800 }, (_, index) =>
|
|
127
|
+
Math.round(5_000 * Math.sin((2 * Math.PI * index) / 83)),
|
|
128
|
+
);
|
|
129
|
+
const inverted = pcm16(inputSamples.map((sample) => -2 * sample));
|
|
130
|
+
|
|
131
|
+
expect(
|
|
132
|
+
pcm16MaxNormalizedCorrelation(pcm16(inputSamples), inverted),
|
|
133
|
+
).toBeGreaterThan(0.99);
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
test("rejects an unrelated waveform and flat power", () => {
|
|
137
|
+
expect(
|
|
138
|
+
pcm16MaxNormalizedCorrelation(wave(240, 800), wave(610, 1_600)),
|
|
139
|
+
).toBeLessThan(0.3);
|
|
140
|
+
expect(
|
|
141
|
+
pcm16MaxNormalizedCorrelation(
|
|
142
|
+
pcm16(new Array(800).fill(3_000)),
|
|
143
|
+
pcm16(new Array(1_600).fill(3_000)),
|
|
144
|
+
),
|
|
145
|
+
).toBe(0);
|
|
146
|
+
});
|
|
147
|
+
});
|
|
@@ -2,10 +2,14 @@ import { describe, expect, test } from "bun:test";
|
|
|
2
2
|
|
|
3
3
|
import type {
|
|
4
4
|
SttStreamServerClosedEvent,
|
|
5
|
+
SttStreamServerEagerTurnEndEvent,
|
|
5
6
|
SttStreamServerErrorEvent,
|
|
6
7
|
SttStreamServerEvent,
|
|
7
8
|
SttStreamServerFinalEvent,
|
|
8
9
|
SttStreamServerPartialEvent,
|
|
10
|
+
SttStreamServerTurnEndEvent,
|
|
11
|
+
SttStreamServerTurnResumedEvent,
|
|
12
|
+
SttStreamServerTurnStartEvent,
|
|
9
13
|
} from "../types.js";
|
|
10
14
|
|
|
11
15
|
// ---------------------------------------------------------------------------
|
|
@@ -87,3 +91,93 @@ describe("SttStreamServerEvent types", () => {
|
|
|
87
91
|
expect(_label).toBeUndefined();
|
|
88
92
|
});
|
|
89
93
|
});
|
|
94
|
+
|
|
95
|
+
describe("turn-detection server events", () => {
|
|
96
|
+
test("turn-start event compiles and round-trips", () => {
|
|
97
|
+
const event: SttStreamServerTurnStartEvent = { type: "turn-start" };
|
|
98
|
+
const asUnion: SttStreamServerEvent = event;
|
|
99
|
+
expect(asUnion.type).toBe("turn-start");
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
test("eager-turn-end event carries the speculated transcript", () => {
|
|
103
|
+
const event: SttStreamServerEagerTurnEndEvent = {
|
|
104
|
+
type: "eager-turn-end",
|
|
105
|
+
text: "is this the end",
|
|
106
|
+
};
|
|
107
|
+
const asUnion: SttStreamServerEvent = event;
|
|
108
|
+
expect(asUnion.type).toBe("eager-turn-end");
|
|
109
|
+
expect(event.text).toBe("is this the end");
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
test("turn-resumed event compiles and round-trips", () => {
|
|
113
|
+
const event: SttStreamServerTurnResumedEvent = { type: "turn-resumed" };
|
|
114
|
+
const asUnion: SttStreamServerEvent = event;
|
|
115
|
+
expect(asUnion.type).toBe("turn-resumed");
|
|
116
|
+
});
|
|
117
|
+
|
|
118
|
+
test("turn-end event carries the committed transcript and optional confidence", () => {
|
|
119
|
+
const withoutConfidence: SttStreamServerTurnEndEvent = {
|
|
120
|
+
type: "turn-end",
|
|
121
|
+
text: "done speaking",
|
|
122
|
+
};
|
|
123
|
+
expect(withoutConfidence.confidence).toBeUndefined();
|
|
124
|
+
|
|
125
|
+
const event: SttStreamServerTurnEndEvent = {
|
|
126
|
+
type: "turn-end",
|
|
127
|
+
text: "done speaking",
|
|
128
|
+
confidence: 0.82,
|
|
129
|
+
};
|
|
130
|
+
const asUnion: SttStreamServerEvent = event;
|
|
131
|
+
expect(asUnion.type).toBe("turn-end");
|
|
132
|
+
expect(event.text).toBe("done speaking");
|
|
133
|
+
expect(event.confidence).toBe(0.82);
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
test("a consumer switch over every variant is exhaustive", () => {
|
|
137
|
+
// Fails to compile if a variant is added to SttStreamServerEvent
|
|
138
|
+
// without a case here, which is what forces consumers to be revisited.
|
|
139
|
+
const label = (event: SttStreamServerEvent): string => {
|
|
140
|
+
switch (event.type) {
|
|
141
|
+
case "partial":
|
|
142
|
+
case "final":
|
|
143
|
+
case "finalized":
|
|
144
|
+
case "error":
|
|
145
|
+
case "closed":
|
|
146
|
+
return "transcript";
|
|
147
|
+
case "turn-start":
|
|
148
|
+
case "eager-turn-end":
|
|
149
|
+
case "turn-resumed":
|
|
150
|
+
case "turn-end":
|
|
151
|
+
return "turn";
|
|
152
|
+
default: {
|
|
153
|
+
const _exhaustive: never = event;
|
|
154
|
+
return _exhaustive;
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
};
|
|
158
|
+
|
|
159
|
+
const events: SttStreamServerEvent[] = [
|
|
160
|
+
{ type: "partial", text: "a" },
|
|
161
|
+
{ type: "final", text: "a" },
|
|
162
|
+
{ type: "finalized" },
|
|
163
|
+
{ type: "error", category: "provider-error", message: "boom" },
|
|
164
|
+
{ type: "closed" },
|
|
165
|
+
{ type: "turn-start" },
|
|
166
|
+
{ type: "eager-turn-end", text: "a" },
|
|
167
|
+
{ type: "turn-resumed" },
|
|
168
|
+
{ type: "turn-end", text: "a" },
|
|
169
|
+
];
|
|
170
|
+
|
|
171
|
+
expect(events.map(label)).toEqual([
|
|
172
|
+
"transcript",
|
|
173
|
+
"transcript",
|
|
174
|
+
"transcript",
|
|
175
|
+
"transcript",
|
|
176
|
+
"transcript",
|
|
177
|
+
"turn",
|
|
178
|
+
"turn",
|
|
179
|
+
"turn",
|
|
180
|
+
"turn",
|
|
181
|
+
]);
|
|
182
|
+
});
|
|
183
|
+
});
|
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
* - xAI (`xai`)
|
|
14
14
|
*/
|
|
15
15
|
|
|
16
|
+
import { batchBoundaryGapReason } from "../providers/speech-to-text/provider-catalog.js";
|
|
16
17
|
import type {
|
|
17
18
|
BatchTranscriber,
|
|
18
19
|
SttProviderId,
|
|
@@ -214,6 +215,8 @@ class VellumManagedBatchTranscriber implements BatchTranscriber {
|
|
|
214
215
|
readonly providerId = "vellum" as const;
|
|
215
216
|
readonly boundaryId = "daemon-batch" as const;
|
|
216
217
|
|
|
218
|
+
constructor(private readonly language: string | undefined) {}
|
|
219
|
+
|
|
217
220
|
async transcribe(
|
|
218
221
|
request: SttTranscribeRequest,
|
|
219
222
|
): Promise<SttTranscribeResult> {
|
|
@@ -223,6 +226,7 @@ class VellumManagedBatchTranscriber implements BatchTranscriber {
|
|
|
223
226
|
request.audio,
|
|
224
227
|
request.mimeType,
|
|
225
228
|
request.signal,
|
|
229
|
+
this.language,
|
|
226
230
|
);
|
|
227
231
|
}
|
|
228
232
|
}
|
|
@@ -237,12 +241,14 @@ class VellumManagedBatchTranscriber implements BatchTranscriber {
|
|
|
237
241
|
* is unavailable.
|
|
238
242
|
*
|
|
239
243
|
* `language` is the spoken language, forwarded to providers whose batch API
|
|
240
|
-
* accepts one: Deepgram (where `"multi"` also pins nova-3)
|
|
244
|
+
* accepts one: Deepgram (where `"multi"` also pins nova-3), xAI (where
|
|
241
245
|
* `"multi"` is dropped because it is a Deepgram-specific value, not a
|
|
242
|
-
* language code)
|
|
243
|
-
*
|
|
244
|
-
*
|
|
245
|
-
*
|
|
246
|
+
* language code), and the vellum managed path, whose platform proxy passes
|
|
247
|
+
* it to Deepgram server-side. Whisper and Gemini auto-detect natively and
|
|
248
|
+
* take no language parameter, so it is silently ignored for them.
|
|
249
|
+
*
|
|
250
|
+
* Throws an {@link SttError} for a provider with no batch endpoint at all,
|
|
251
|
+
* which no key can fix and `null` would understate.
|
|
246
252
|
*/
|
|
247
253
|
export function createDaemonBatchTranscriber(
|
|
248
254
|
apiKey: string | null | undefined,
|
|
@@ -251,7 +257,7 @@ export function createDaemonBatchTranscriber(
|
|
|
251
257
|
): BatchTranscriber | null {
|
|
252
258
|
// vellum authenticates via the platform connection, not an API key.
|
|
253
259
|
if (providerId === "vellum") {
|
|
254
|
-
return new VellumManagedBatchTranscriber();
|
|
260
|
+
return new VellumManagedBatchTranscriber(language);
|
|
255
261
|
}
|
|
256
262
|
if (!apiKey) {
|
|
257
263
|
return null;
|
|
@@ -266,6 +272,12 @@ export function createDaemonBatchTranscriber(
|
|
|
266
272
|
return new GoogleGeminiBatchTranscriber(apiKey);
|
|
267
273
|
case "xai":
|
|
268
274
|
return new XAIBatchTranscriber(apiKey, language);
|
|
275
|
+
case "deepgram-flux":
|
|
276
|
+
// Same copy the resolver raises, so a direct factory caller and a
|
|
277
|
+
// config-driven one report the mismatch identically.
|
|
278
|
+
throw new SttError("provider-error", batchBoundaryGapReason(providerId), {
|
|
279
|
+
userFacing: true,
|
|
280
|
+
});
|
|
269
281
|
default: {
|
|
270
282
|
// Exhaustive check — compile error if a new SttProviderId is added
|
|
271
283
|
// without a corresponding case here.
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pure helpers for tallying and ranking detected-language tags emitted
|
|
3
|
+
* by streaming STT providers (e.g. Deepgram nova-3 `multi` per-word tags).
|
|
4
|
+
*
|
|
5
|
+
* Tag normalization is `baseLanguageSubtag` (util/language-subtag.ts);
|
|
6
|
+
* these helpers layer vote-counting on top of it so provider adapters and
|
|
7
|
+
* event consumers share one notion of a dominant language.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import { baseLanguageSubtag } from "../util/language-subtag.js";
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Cast one vote into a language tally for a transcript chunk's dominant
|
|
14
|
+
* detected language. `languages` is dominance-ranked, so only the first
|
|
15
|
+
* entry votes: secondary tags at full weight would let a minority
|
|
16
|
+
* language outvote the dominant one. Blank and absent tags cast no vote;
|
|
17
|
+
* regional variants count toward their base subtag.
|
|
18
|
+
*/
|
|
19
|
+
export function voteDominantLanguage(
|
|
20
|
+
tally: Map<string, number>,
|
|
21
|
+
languages: readonly string[] | undefined,
|
|
22
|
+
): void {
|
|
23
|
+
const dominant = baseLanguageSubtag(languages?.[0]);
|
|
24
|
+
if (dominant !== undefined) {
|
|
25
|
+
tally.set(dominant, (tally.get(dominant) ?? 0) + 1);
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* The dominant tag in a language tally: most votes wins, ties break by
|
|
31
|
+
* first insertion (Map iteration is insertion order). Undefined for an
|
|
32
|
+
* empty tally.
|
|
33
|
+
*/
|
|
34
|
+
export function dominantLanguageTag(
|
|
35
|
+
tally: ReadonlyMap<string, number>,
|
|
36
|
+
): string | undefined {
|
|
37
|
+
let dominant: string | undefined;
|
|
38
|
+
let dominantCount = 0;
|
|
39
|
+
for (const [tag, count] of tally) {
|
|
40
|
+
if (count > dominantCount) {
|
|
41
|
+
dominant = tag;
|
|
42
|
+
dominantCount = count;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
return dominant;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Tally normalized language tags and return them most-frequent-first.
|
|
50
|
+
* Ties are broken by first appearance in the input. Blank tags are
|
|
51
|
+
* skipped; regional variants count toward their base subtag.
|
|
52
|
+
*/
|
|
53
|
+
export function rankLanguages(tags: Iterable<string>): string[] {
|
|
54
|
+
// Map iteration order is insertion order, so a stable sort by count
|
|
55
|
+
// leaves tied tags in first-appearance order.
|
|
56
|
+
const counts = new Map<string, number>();
|
|
57
|
+
for (const tag of tags) {
|
|
58
|
+
const normalized = baseLanguageSubtag(tag);
|
|
59
|
+
if (normalized === undefined) {
|
|
60
|
+
continue;
|
|
61
|
+
}
|
|
62
|
+
counts.set(normalized, (counts.get(normalized) ?? 0) + 1);
|
|
63
|
+
}
|
|
64
|
+
return [...counts.entries()].sort((a, b) => b[1] - a[1]).map(([tag]) => tag);
|
|
65
|
+
}
|
package/src/stt/speech-energy.ts
CHANGED
|
@@ -18,6 +18,120 @@
|
|
|
18
18
|
*/
|
|
19
19
|
export const DEFAULT_SPEECH_ENERGY_THRESHOLD = 800;
|
|
20
20
|
|
|
21
|
+
/**
|
|
22
|
+
* Return the mean absolute amplitude of little-endian signed PCM16 audio.
|
|
23
|
+
* Empty buffers return 0, and a trailing odd byte is ignored.
|
|
24
|
+
*/
|
|
25
|
+
export function pcm16MeanAmplitude(chunk: Buffer): number {
|
|
26
|
+
const sampleCount = Math.floor(chunk.length / 2);
|
|
27
|
+
if (sampleCount === 0) {
|
|
28
|
+
return 0;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
let totalAmplitude = 0;
|
|
32
|
+
for (let i = 0; i < sampleCount; i += 1) {
|
|
33
|
+
totalAmplitude += Math.abs(chunk.readInt16LE(i * 2));
|
|
34
|
+
}
|
|
35
|
+
return totalAmplitude / sampleCount;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Find the strongest normalized correlation between a PCM16 input window and
|
|
40
|
+
* any same-length window in a PCM16 reference. The samples are block-averaged
|
|
41
|
+
* before matching so this stays cheap enough for live audio and remains
|
|
42
|
+
* tolerant of the low-pass filtering introduced by speakers and microphones.
|
|
43
|
+
*
|
|
44
|
+
* The absolute coefficient makes polarity inversion harmless. A flat input or
|
|
45
|
+
* reference has no identifying waveform and returns 0 instead of being treated
|
|
46
|
+
* as a match based on level alone.
|
|
47
|
+
*/
|
|
48
|
+
export function pcm16MaxNormalizedCorrelation(
|
|
49
|
+
input: Buffer,
|
|
50
|
+
reference: Buffer,
|
|
51
|
+
downsampleFactor = 8,
|
|
52
|
+
): number {
|
|
53
|
+
if (!Number.isInteger(downsampleFactor) || downsampleFactor <= 0) {
|
|
54
|
+
return 0;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
let inputSamples = blockAveragePcm16(input, downsampleFactor);
|
|
58
|
+
const referenceSamples = blockAveragePcm16(reference, downsampleFactor);
|
|
59
|
+
if (inputSamples.length > referenceSamples.length) {
|
|
60
|
+
inputSamples = inputSamples.subarray(0, referenceSamples.length);
|
|
61
|
+
}
|
|
62
|
+
if (inputSamples.length < 2) {
|
|
63
|
+
return 0;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
let inputSum = 0;
|
|
67
|
+
for (let index = 0; index < inputSamples.length; index += 1) {
|
|
68
|
+
inputSum += inputSamples[index]!;
|
|
69
|
+
}
|
|
70
|
+
const inputMean = inputSum / inputSamples.length;
|
|
71
|
+
const centeredInput = new Float64Array(inputSamples.length);
|
|
72
|
+
let inputEnergy = 0;
|
|
73
|
+
for (let index = 0; index < inputSamples.length; index += 1) {
|
|
74
|
+
const centered = inputSamples[index]! - inputMean;
|
|
75
|
+
centeredInput[index] = centered;
|
|
76
|
+
inputEnergy += centered * centered;
|
|
77
|
+
}
|
|
78
|
+
if (inputEnergy === 0) {
|
|
79
|
+
return 0;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
const prefixSum = new Float64Array(referenceSamples.length + 1);
|
|
83
|
+
const prefixSquareSum = new Float64Array(referenceSamples.length + 1);
|
|
84
|
+
for (let index = 0; index < referenceSamples.length; index += 1) {
|
|
85
|
+
const sample = referenceSamples[index]!;
|
|
86
|
+
prefixSum[index + 1] = prefixSum[index]! + sample;
|
|
87
|
+
prefixSquareSum[index + 1] = prefixSquareSum[index]! + sample * sample;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
let best = 0;
|
|
91
|
+
const windowLength = inputSamples.length;
|
|
92
|
+
for (
|
|
93
|
+
let offset = 0;
|
|
94
|
+
offset + windowLength <= referenceSamples.length;
|
|
95
|
+
offset += 1
|
|
96
|
+
) {
|
|
97
|
+
const referenceSum = prefixSum[offset + windowLength]! - prefixSum[offset]!;
|
|
98
|
+
const referenceSquareSum =
|
|
99
|
+
prefixSquareSum[offset + windowLength]! - prefixSquareSum[offset]!;
|
|
100
|
+
const referenceEnergy =
|
|
101
|
+
referenceSquareSum - (referenceSum * referenceSum) / windowLength;
|
|
102
|
+
if (referenceEnergy <= 0) {
|
|
103
|
+
continue;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
let dotProduct = 0;
|
|
107
|
+
for (let index = 0; index < windowLength; index += 1) {
|
|
108
|
+
dotProduct += centeredInput[index]! * referenceSamples[offset + index]!;
|
|
109
|
+
}
|
|
110
|
+
const correlation =
|
|
111
|
+
Math.abs(dotProduct) / Math.sqrt(inputEnergy * referenceEnergy);
|
|
112
|
+
best = Math.max(best, Math.min(correlation, 1));
|
|
113
|
+
}
|
|
114
|
+
return best;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
function blockAveragePcm16(
|
|
118
|
+
chunk: Buffer,
|
|
119
|
+
downsampleFactor: number,
|
|
120
|
+
): Float64Array {
|
|
121
|
+
const sampleCount = Math.floor(chunk.length / 2);
|
|
122
|
+
const blockCount = Math.floor(sampleCount / downsampleFactor);
|
|
123
|
+
const result = new Float64Array(blockCount);
|
|
124
|
+
for (let block = 0; block < blockCount; block += 1) {
|
|
125
|
+
let sum = 0;
|
|
126
|
+
const firstSample = block * downsampleFactor;
|
|
127
|
+
for (let offset = 0; offset < downsampleFactor; offset += 1) {
|
|
128
|
+
sum += chunk.readInt16LE((firstSample + offset) * 2);
|
|
129
|
+
}
|
|
130
|
+
result[block] = sum / downsampleFactor;
|
|
131
|
+
}
|
|
132
|
+
return result;
|
|
133
|
+
}
|
|
134
|
+
|
|
21
135
|
/**
|
|
22
136
|
* Detect speech activity in a chunk of little-endian signed 16-bit mono
|
|
23
137
|
* PCM samples.
|
|
@@ -34,16 +148,5 @@ export function detectPcm16SpeechActivity(
|
|
|
34
148
|
chunk: Buffer,
|
|
35
149
|
threshold = DEFAULT_SPEECH_ENERGY_THRESHOLD,
|
|
36
150
|
): boolean {
|
|
37
|
-
|
|
38
|
-
if (sampleCount === 0) {
|
|
39
|
-
return false;
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
let totalAmplitude = 0;
|
|
43
|
-
for (let i = 0; i < sampleCount; i++) {
|
|
44
|
-
totalAmplitude += Math.abs(chunk.readInt16LE(i * 2));
|
|
45
|
-
}
|
|
46
|
-
const avgAmplitude = totalAmplitude / sampleCount;
|
|
47
|
-
|
|
48
|
-
return avgAmplitude > threshold;
|
|
151
|
+
return pcm16MeanAmplitude(chunk) > threshold;
|
|
49
152
|
}
|
|
@@ -5,8 +5,11 @@
|
|
|
5
5
|
* a client WebSocket connection and a provider-specific streaming adapter.
|
|
6
6
|
* The orchestrator accepts client events (`audio`, `stop`), routes audio
|
|
7
7
|
* frames to the resolved {@link StreamingTranscriber}, and emits normalized
|
|
8
|
-
* server events
|
|
9
|
-
*
|
|
8
|
+
* server events back over the same WebSocket connection with per-session
|
|
9
|
+
* ordering guarantees: transcripts (`partial`, `final`, `finalized`),
|
|
10
|
+
* turn boundaries from providers that detect them (`turn-start`,
|
|
11
|
+
* `eager-turn-end`, `turn-resumed`, `turn-end`), and termination
|
|
12
|
+
* (`error`, `closed`).
|
|
10
13
|
*
|
|
11
14
|
* Session lifecycle:
|
|
12
15
|
* 1. Client opens a WebSocket to `/v1/stt/stream` with required `mimeType`
|
|
@@ -17,8 +20,9 @@
|
|
|
17
20
|
* `resolveStreamingTranscriber()` and starts the provider session.
|
|
18
21
|
* 3. The client sends `audio` frames (binary or base64-encoded JSON) and
|
|
19
22
|
* a `stop` event when recording is complete.
|
|
20
|
-
* 4. The provider emits
|
|
21
|
-
* orchestrator forwards to the client as
|
|
23
|
+
* 4. The provider emits transcript events, plus turn-boundary events when it
|
|
24
|
+
* detects turns, and the orchestrator forwards each to the client as a
|
|
25
|
+
* JSON frame.
|
|
22
26
|
* 5. The session closes deterministically on client disconnect, `stop`
|
|
23
27
|
* event, idle timeout, or runtime shutdown.
|
|
24
28
|
*
|
package/src/stt/types.ts
CHANGED
|
@@ -23,6 +23,11 @@
|
|
|
23
23
|
export type SttProviderId =
|
|
24
24
|
| "openai-whisper"
|
|
25
25
|
| "deepgram"
|
|
26
|
+
/**
|
|
27
|
+
* Deepgram's Flux conversational speech model. Shares the `deepgram`
|
|
28
|
+
* credential and is streaming-only: Flux has no batch endpoint.
|
|
29
|
+
*/
|
|
30
|
+
| "deepgram-flux"
|
|
26
31
|
| "google-gemini"
|
|
27
32
|
| "xai"
|
|
28
33
|
| "vellum";
|
|
@@ -65,6 +70,26 @@ export type ConversationStreamingMode =
|
|
|
65
70
|
| "incremental-batch"
|
|
66
71
|
| "none";
|
|
67
72
|
|
|
73
|
+
/**
|
|
74
|
+
* Turn-detection support mode.
|
|
75
|
+
*
|
|
76
|
+
* Describes whether a provider decides end-of-turn itself, in-band with the
|
|
77
|
+
* audio it is already transcribing, rather than leaving the boundary to the
|
|
78
|
+
* session's local silence timer and front-door hold verdict.
|
|
79
|
+
*
|
|
80
|
+
* - `"provider"`: the provider emits turn lifecycle events (`turn-start`,
|
|
81
|
+
* `turn-end`) on its transcript stream, and a live-voice session may let
|
|
82
|
+
* those events commit the turn.
|
|
83
|
+
* - `"none"`: the provider transcribes only; the local silence boundary owns
|
|
84
|
+
* end-of-turn.
|
|
85
|
+
*
|
|
86
|
+
* A provider declaring `"provider"` should number its turns: the staleness
|
|
87
|
+
* check prefers the turn index carried on the event, and degrades to the
|
|
88
|
+
* session's local VAD generation counter when one is absent, which understates
|
|
89
|
+
* the recorded latency for a turn the caller resumed and stopped again.
|
|
90
|
+
*/
|
|
91
|
+
export type SttTurnDetectionMode = "provider" | "none";
|
|
92
|
+
|
|
68
93
|
// ---------------------------------------------------------------------------
|
|
69
94
|
// Boundary identifier
|
|
70
95
|
// ---------------------------------------------------------------------------
|
|
@@ -245,6 +270,10 @@ export type SttStreamServerEvent =
|
|
|
245
270
|
| SttStreamServerPartialEvent
|
|
246
271
|
| SttStreamServerFinalEvent
|
|
247
272
|
| SttStreamServerFinalizedEvent
|
|
273
|
+
| SttStreamServerTurnStartEvent
|
|
274
|
+
| SttStreamServerEagerTurnEndEvent
|
|
275
|
+
| SttStreamServerTurnResumedEvent
|
|
276
|
+
| SttStreamServerTurnEndEvent
|
|
248
277
|
| SttStreamServerErrorEvent
|
|
249
278
|
| SttStreamServerClosedEvent;
|
|
250
279
|
|
|
@@ -263,6 +292,14 @@ export interface SttStreamServerPartialEvent {
|
|
|
263
292
|
* provider does not surface confidence on interim chunks.
|
|
264
293
|
*/
|
|
265
294
|
readonly confidence?: number;
|
|
295
|
+
/**
|
|
296
|
+
* All detected languages of this chunk in dominance order (first entry
|
|
297
|
+
* dominant), already normalized to lowercase base subtags (e.g. "en",
|
|
298
|
+
* "hi") by the emitting adapter; consumers must not re-normalize. Only
|
|
299
|
+
* providers with code-switching metadata (currently Deepgram nova-3
|
|
300
|
+
* `multi`) populate this. Absence means "unknown", never English.
|
|
301
|
+
*/
|
|
302
|
+
readonly languages?: readonly string[];
|
|
266
303
|
}
|
|
267
304
|
|
|
268
305
|
/**
|
|
@@ -279,6 +316,14 @@ export interface SttStreamServerFinalEvent {
|
|
|
279
316
|
* provider does not surface confidence on this chunk.
|
|
280
317
|
*/
|
|
281
318
|
readonly confidence?: number;
|
|
319
|
+
/**
|
|
320
|
+
* All detected languages of this chunk in dominance order (first entry
|
|
321
|
+
* dominant), already normalized to lowercase base subtags (e.g. "en",
|
|
322
|
+
* "hi") by the emitting adapter; consumers must not re-normalize. Only
|
|
323
|
+
* providers with code-switching metadata (currently Deepgram nova-3
|
|
324
|
+
* `multi`) populate this. Absence means "unknown", never English.
|
|
325
|
+
*/
|
|
326
|
+
readonly languages?: readonly string[];
|
|
282
327
|
/**
|
|
283
328
|
* True when this final is the flush response to
|
|
284
329
|
* {@link StreamingTranscriber.finalizeUtterance} — i.e. it commits audio
|
|
@@ -298,6 +343,80 @@ export interface SttStreamServerFinalizedEvent {
|
|
|
298
343
|
readonly type: "finalized";
|
|
299
344
|
}
|
|
300
345
|
|
|
346
|
+
// ---------------------------------------------------------------------------
|
|
347
|
+
// Turn-detection events
|
|
348
|
+
//
|
|
349
|
+
// Consumers that ignore these keep committing transcripts from `final`
|
|
350
|
+
// exactly as they do without a turn-detecting provider.
|
|
351
|
+
// ---------------------------------------------------------------------------
|
|
352
|
+
|
|
353
|
+
/**
|
|
354
|
+
* The provider's turn model detected the start of a user turn.
|
|
355
|
+
*
|
|
356
|
+
* Emitted only by providers with model-integrated turn detection; providers
|
|
357
|
+
* without it never emit these.
|
|
358
|
+
*/
|
|
359
|
+
export interface SttStreamServerTurnStartEvent {
|
|
360
|
+
readonly type: "turn-start";
|
|
361
|
+
/**
|
|
362
|
+
* Provider's index for the turn being opened, monotonically increasing
|
|
363
|
+
* within a stream. Undefined when the provider does not number its turns.
|
|
364
|
+
*/
|
|
365
|
+
readonly turnIndex?: number;
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
/**
|
|
369
|
+
* The provider's turn model speculates the user has finished, ahead of its
|
|
370
|
+
* committed end-of-turn decision. A {@link SttStreamServerTurnResumedEvent}
|
|
371
|
+
* retracts the speculation; a {@link SttStreamServerTurnEndEvent} confirms it.
|
|
372
|
+
*
|
|
373
|
+
* Emitted only by providers with model-integrated turn detection; providers
|
|
374
|
+
* without it never emit these.
|
|
375
|
+
*/
|
|
376
|
+
export interface SttStreamServerEagerTurnEndEvent {
|
|
377
|
+
readonly type: "eager-turn-end";
|
|
378
|
+
/** Transcript text as of the speculated turn end. */
|
|
379
|
+
readonly text: string;
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
/**
|
|
383
|
+
* The user kept speaking after an {@link SttStreamServerEagerTurnEndEvent},
|
|
384
|
+
* retracting the speculated turn end.
|
|
385
|
+
*
|
|
386
|
+
* Emitted only by providers with model-integrated turn detection; providers
|
|
387
|
+
* without it never emit these.
|
|
388
|
+
*/
|
|
389
|
+
export interface SttStreamServerTurnResumedEvent {
|
|
390
|
+
readonly type: "turn-resumed";
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
/**
|
|
394
|
+
* The provider's turn model committed an end of turn: the user has finished
|
|
395
|
+
* speaking and a reply can be dispatched.
|
|
396
|
+
*
|
|
397
|
+
* Emitted only by providers with model-integrated turn detection; providers
|
|
398
|
+
* without it never emit these.
|
|
399
|
+
*/
|
|
400
|
+
export interface SttStreamServerTurnEndEvent {
|
|
401
|
+
readonly type: "turn-end";
|
|
402
|
+
/** Committed transcript text for the completed turn. */
|
|
403
|
+
readonly text: string;
|
|
404
|
+
/**
|
|
405
|
+
* Provider-emitted end-of-turn confidence in [0, 1]. Undefined when the
|
|
406
|
+
* provider does not surface a score.
|
|
407
|
+
*/
|
|
408
|
+
readonly confidence?: number;
|
|
409
|
+
/**
|
|
410
|
+
* Provider's index for the turn being closed, matching the
|
|
411
|
+
* {@link SttStreamServerTurnStartEvent} that opened it. A consumer can
|
|
412
|
+
* compare it against the newest turn it has seen opened to tell an
|
|
413
|
+
* end-of-turn for the turn still in progress from one the provider has
|
|
414
|
+
* already superseded. Undefined when the provider does not number its
|
|
415
|
+
* turns.
|
|
416
|
+
*/
|
|
417
|
+
readonly turnIndex?: number;
|
|
418
|
+
}
|
|
419
|
+
|
|
301
420
|
/** An error occurred during streaming transcription. */
|
|
302
421
|
export interface SttStreamServerErrorEvent {
|
|
303
422
|
readonly type: "error";
|
|
@@ -20,6 +20,31 @@ describe("buildAdvisorSystem", () => {
|
|
|
20
20
|
expect(prompt).not.toContain("<agent_system_prompt>");
|
|
21
21
|
});
|
|
22
22
|
|
|
23
|
+
test("tells the advisor it has read-only tools for verifying decisive facts", () => {
|
|
24
|
+
const prompt = buildAdvisorSystem(null);
|
|
25
|
+
expect(prompt).toContain("read-only tools");
|
|
26
|
+
expect(prompt).toContain("read files");
|
|
27
|
+
expect(prompt).toContain("verification, not exploration");
|
|
28
|
+
expect(prompt).toContain("You cannot change anything");
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
test("does not offer the advisor a memory or conversation search", () => {
|
|
32
|
+
// The advisor's read tools stop at the workspace. Naming `recall` here
|
|
33
|
+
// would advertise a search the role allowlist does not grant, and would
|
|
34
|
+
// contradict the scope the consult framing promises the user.
|
|
35
|
+
const prompt = buildAdvisorSystem(null);
|
|
36
|
+
expect(prompt).not.toContain("recall");
|
|
37
|
+
expect(prompt).toContain("you cannot see other conversations");
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
test("does not claim the advisor is tool-less", () => {
|
|
41
|
+
// The advisor can open a file to check a fact; a prompt that says otherwise
|
|
42
|
+
// suppresses the read it was given tools for.
|
|
43
|
+
const prompt = buildAdvisorSystem(null);
|
|
44
|
+
expect(prompt).not.toContain("You have no tools");
|
|
45
|
+
expect(prompt).not.toContain("cannot search, read files, or run commands");
|
|
46
|
+
});
|
|
47
|
+
|
|
23
48
|
test("keeps the situational context pack out of the system prompt", () => {
|
|
24
49
|
// System Prompt Minimalism: the pack rides in the request turn instead.
|
|
25
50
|
expect(buildAdvisorSystem("parent")).not.toContain("<agent_environment>");
|