@vellumai/assistant 0.11.2 → 0.11.3-staging.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +58 -3
- package/docs/trusted-contact-access.md +9 -5
- package/docs/vellum-doctor.md +195 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/channels.ts +39 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/ingress.ts +10 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/remote-web-pairing.ts +6 -0
- package/node_modules/@vellumai/ces-client/src/index.ts +1 -0
- package/node_modules/@vellumai/ces-client/src/rpc-client.ts +19 -2
- package/node_modules/@vellumai/environments/package.json +2 -1
- package/node_modules/@vellumai/environments/src/__tests__/cloud-assistant-hub-url.test.ts +38 -0
- package/node_modules/@vellumai/environments/src/__tests__/install-layout.test.ts +98 -0
- package/node_modules/@vellumai/environments/src/__tests__/package-boundary.test.ts +5 -5
- package/node_modules/@vellumai/environments/src/index.ts +17 -6
- package/node_modules/@vellumai/environments/src/install-layout.ts +49 -0
- package/node_modules/@vellumai/environments/src/seeds.ts +29 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/channels.ts +39 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/ingress.ts +10 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/remote-web-pairing.ts +6 -0
- package/node_modules/@vellumai/gateway-client/src/index.ts +4 -0
- package/node_modules/@vellumai/gateway-client/src/verification-session-contract.ts +91 -6
- package/node_modules/@vellumai/ipc-server-utils/src/endpoint.test.ts +36 -0
- package/node_modules/@vellumai/ipc-server-utils/src/endpoint.ts +142 -0
- package/node_modules/@vellumai/ipc-server-utils/src/index.ts +2 -0
- package/node_modules/@vellumai/ipc-server-utils/src/listen-options.ts +3 -0
- package/node_modules/@vellumai/ipc-server-utils/src/socket-watchdog.test.ts +15 -1
- package/node_modules/@vellumai/ipc-server-utils/src/socket-watchdog.ts +17 -2
- package/node_modules/@vellumai/service-contracts/src/channels.ts +39 -0
- package/node_modules/@vellumai/service-contracts/src/ingress.ts +10 -0
- package/node_modules/@vellumai/service-contracts/src/remote-web-pairing.ts +6 -0
- package/node_modules/@vellumai/slack-text/src/index.ts +16 -2
- package/openapi.yaml +589 -30
- package/package.json +1 -1
- package/src/__tests__/agent-image-optimize.test.ts +4 -4
- package/src/__tests__/agent-loop-mutable-latest-user-message.test.ts +113 -66
- package/src/__tests__/agent-loop-override-profile.test.ts +38 -48
- package/src/__tests__/agent-loop-subagent-billing-headers.test.ts +166 -0
- package/src/__tests__/agent-wake-disk-pressure-callsite.test.ts +1 -0
- package/src/__tests__/agent-wake-override-profile.test.ts +1 -0
- package/src/__tests__/answered-question-persistence.test.ts +240 -0
- package/src/__tests__/anthropic-provider.test.ts +165 -30
- package/src/__tests__/app-control-flow.test.ts +9 -8
- package/src/__tests__/assistant-event-hub.test.ts +48 -0
- package/src/__tests__/attachments-store-heic-normalize.test.ts +16 -12
- package/src/__tests__/attachments-store.test.ts +130 -68
- package/src/__tests__/attachments.test.ts +36 -36
- package/src/__tests__/background-workers-disk-pressure.test.ts +1 -1
- package/src/__tests__/call-start-guardian-guard.test.ts +18 -4
- package/src/__tests__/canned-reply-release.test.ts +86 -0
- package/src/__tests__/channel-guardian.test.ts +59 -29
- package/src/__tests__/channel-readiness-telegram-remote.test.ts +154 -0
- package/src/__tests__/channel-setup-panel-ack.test.ts +6 -5
- package/src/__tests__/client-os-metadata-persistence.test.ts +103 -4
- package/src/__tests__/compactor-image-manifest-trust.test.ts +1 -1
- package/src/__tests__/consult-deadline.test.ts +65 -0
- package/src/__tests__/conversation-agent-loop.test.ts +438 -2
- package/src/__tests__/conversation-disk-view-integration.test.ts +5 -1
- package/src/__tests__/conversation-disk-view.test.ts +4 -4
- package/src/__tests__/conversation-error.test.ts +22 -3
- package/src/__tests__/conversation-fork-crud.test.ts +5 -1
- package/src/__tests__/conversation-fork-referential.test.ts +352 -0
- package/src/__tests__/conversation-fork-retrospective.test.ts +5 -1
- package/src/__tests__/conversation-group-tools.test.ts +32 -2
- package/src/__tests__/conversation-inference-profile-route.test.ts +27 -1
- package/src/__tests__/conversation-initial-prompt.test.ts +51 -22
- package/src/__tests__/conversation-key-store-bootstrap-cleanup.test.ts +119 -0
- package/src/__tests__/conversation-key-store-origin.test.ts +105 -0
- package/src/__tests__/conversation-list-recency-ordering.test.ts +104 -0
- package/src/__tests__/conversation-origin-at-creation.test.ts +153 -0
- package/src/__tests__/conversation-origin-channel-filter.test.ts +95 -0
- package/src/__tests__/conversation-placement-promotion.test.ts +172 -0
- package/src/__tests__/conversation-process-app-control-preactivation.test.ts +123 -0
- package/src/__tests__/conversation-queue.test.ts +433 -1
- package/src/__tests__/conversation-retry-route.test.ts +3 -1
- package/src/__tests__/conversation-routes-hidden-queue.test.ts +64 -1
- package/src/__tests__/conversation-store-ephemeral.test.ts +4 -0
- package/src/__tests__/conversation-store.test.ts +26 -10
- package/src/__tests__/conversation-summarize-route.test.ts +1 -1
- package/src/__tests__/conversation-surfaces-action-delivery.test.ts +15 -6
- package/src/__tests__/conversation-surfaces-activation-emit.test.ts +6 -5
- package/src/__tests__/conversation-surfaces-app-control.test.ts +12 -11
- package/src/__tests__/conversation-surfaces-app-open.test.ts +11 -5
- package/src/__tests__/conversation-surfaces-data-persist.test.ts +5 -4
- package/src/__tests__/conversation-surfaces-history-restored-completion.test.ts +967 -0
- package/src/__tests__/conversation-surfaces-persisted-info.test.ts +312 -0
- package/src/__tests__/conversation-surfaces-queued-emit.test.ts +6 -5
- package/src/__tests__/conversation-surfaces-standalone-payloads.test.ts +12 -8
- package/src/__tests__/conversation-surfaces-standalone.test.ts +21 -11
- package/src/__tests__/conversation-surfaces-state-update.test.ts +7 -11
- package/src/__tests__/conversation-surfaces-table-action.test.ts +7 -6
- package/src/__tests__/conversation-surfaces-task-progress.test.ts +11 -6
- package/src/__tests__/conversation-tool-setup-app-refresh.test.ts +9 -6
- package/src/__tests__/conversation-tool-setup-attribution.test.ts +70 -7
- package/src/__tests__/conversation-wait-for-idle.test.ts +6 -11
- package/src/__tests__/credential-prompt-route.test.ts +57 -7
- package/src/__tests__/credential-security-invariants.test.ts +1 -0
- package/src/__tests__/cu-unified-flow.test.ts +89 -28
- package/src/__tests__/custom-group-listing-visibility.test.ts +1 -1
- package/src/__tests__/db-schedule-syntax-migration.test.ts +4 -1
- package/src/__tests__/discord-access-request-privacy.test.ts +366 -0
- package/src/__tests__/discord-requester-notice-privacy.test.ts +87 -0
- package/src/__tests__/disk-pressure-guard.test.ts +55 -55
- package/src/__tests__/disk-pressure-routes.test.ts +10 -10
- package/src/__tests__/disk-pressure-tools.test.ts +2 -2
- package/src/__tests__/disk-usage.test.ts +35 -20
- package/src/__tests__/document-sync-tags.test.ts +379 -0
- package/src/__tests__/document-update-default-surface.test.ts +28 -0
- package/src/__tests__/dynamic-skill-background-guard.test.ts +2 -2
- package/src/__tests__/emit-signal-routing-intent.test.ts +3 -4
- package/src/__tests__/ensure-conversation-exists.test.ts +8 -5
- package/src/__tests__/events-dev-bypass-actor.test.ts +183 -1
- package/src/__tests__/evict-conversations-for-reload.test.ts +90 -0
- package/src/__tests__/file-list-tool.test.ts +12 -10
- package/src/__tests__/file-ops-service.test.ts +151 -44
- package/src/__tests__/file-read-container-boundary.test.ts +137 -0
- package/src/__tests__/file-read-tool.test.ts +21 -9
- package/src/__tests__/file-url-path-guard.test.ts +83 -0
- package/src/__tests__/filesystem-tools.test.ts +82 -66
- package/src/__tests__/guardian-outbound-http.test.ts +8 -8
- package/src/__tests__/helpers/gateway-verification-sessions-stub.ts +14 -1
- package/src/__tests__/helpers/hub-clients.ts +42 -0
- package/src/__tests__/helpers/mock-conversation.ts +46 -0
- package/src/__tests__/helpers/verification-sessions-ipc-sim.ts +61 -57
- package/src/__tests__/history-repair.test.ts +112 -12
- package/src/__tests__/host-bash-proxy.test.ts +10 -5
- package/src/__tests__/host-bash-routes.test.ts +67 -0
- package/src/__tests__/host-browser-proxy.test.ts +2 -2
- package/src/__tests__/host-cu-proxy.test.ts +3 -7
- package/src/__tests__/host-file-proxy.test.ts +3 -7
- package/src/__tests__/host-file-read-tool.test.ts +5 -1
- package/src/__tests__/host-transfer-proxy-targeted.test.ts +6 -4
- package/src/__tests__/http-conversation-lineage.test.ts +2 -5
- package/src/__tests__/http-user-message-parity.test.ts +44 -0
- package/src/__tests__/identity-routes.test.ts +13 -13
- package/src/__tests__/image-conversion.test.ts +29 -26
- package/src/__tests__/image-source-path-reinject.test.ts +2 -2
- package/src/__tests__/inference-profile-session-handler.test.ts +95 -0
- package/src/__tests__/inference-profile-session-ipc.test.ts +25 -0
- package/src/__tests__/introduction-card-resolver.test.ts +6 -1
- package/src/__tests__/list-messages-attachments.test.ts +56 -32
- package/src/__tests__/list-messages-background-tool-completion.test.ts +4 -4
- package/src/__tests__/list-messages-client-message-id.test.ts +2 -2
- package/src/__tests__/list-messages-hidden-metadata.test.ts +16 -16
- package/src/__tests__/list-messages-page-latest.test.ts +61 -61
- package/src/__tests__/list-messages-provider-error.test.ts +6 -6
- package/src/__tests__/list-messages-queued.test.ts +49 -6
- package/src/__tests__/list-messages-system-card.test.ts +4 -4
- package/src/__tests__/list-messages-tool-merge.test.ts +9 -9
- package/src/__tests__/llm-resolver-override-or-default.test.ts +141 -0
- package/src/__tests__/llm-usage-store.test.ts +55 -0
- package/src/__tests__/manual-token-reconciliation.test.ts +45 -0
- package/src/__tests__/mcp-tool-annotations-risk.test.ts +133 -0
- package/src/__tests__/media-stream-server-integration.test.ts +36 -0
- package/src/__tests__/migration-export-http.test.ts +1 -1
- package/src/__tests__/migration-import-commit-http.test.ts +1 -1
- package/src/__tests__/migration-import-preflight-http.test.ts +1 -1
- package/src/__tests__/migration-validate-http.test.ts +1 -1
- package/src/__tests__/mtime-cache.test.ts +65 -3
- package/src/__tests__/notification-platform-adapter.test.ts +104 -0
- package/src/__tests__/notification-schedule-notify-dedup.test.ts +1 -1
- package/src/__tests__/oauth-provider-profiles.test.ts +4 -2
- package/src/__tests__/oauth-provider-seed-logos.test.ts +65 -0
- package/src/__tests__/oauth-provider-serializer.test.ts +17 -0
- package/src/__tests__/oauth-providers-routes.test.ts +1 -0
- package/src/__tests__/openai-responses-prompt-cache.test.ts +5 -2
- package/src/__tests__/path-policy.test.ts +164 -1
- package/src/__tests__/persist-media-references.test.ts +2 -2
- package/src/__tests__/persist-unsendable-image-downscale.test.ts +5 -5
- package/src/__tests__/persist-unsendable-image.test.ts +23 -23
- package/src/__tests__/persona-resolver.test.ts +154 -0
- package/src/__tests__/plugin-api-webhook-url.test.ts +170 -0
- package/src/__tests__/plugin-import-boundary-guard.test.ts +6 -0
- package/src/__tests__/processing-flag-persist-failure.test.ts +15 -10
- package/src/__tests__/profile-availability-incomplete.test.ts +113 -0
- package/src/__tests__/profiler-routes.test.ts +13 -13
- package/src/__tests__/prompt-cache-cross-turn-stability.test.ts +593 -0
- package/src/__tests__/proxy-approval-callback.test.ts +3 -3
- package/src/__tests__/queued-message-cancel-and-steer.test.ts +189 -0
- package/src/__tests__/queued-message-steer-actor-scoping.test.ts +164 -0
- package/src/__tests__/resolve-trust-class.test.ts +20 -0
- package/src/__tests__/run-conversation-turn-persistence.test.ts +1 -1
- package/src/__tests__/run-due-schedules.test.ts +174 -0
- package/src/__tests__/runtime-attachment-metadata.test.ts +15 -7
- package/src/__tests__/scaffold-managed-skill-tool.test.ts +166 -0
- package/src/__tests__/schedule-routes.test.ts +730 -8
- package/src/__tests__/schedule-store.test.ts +540 -0
- package/src/__tests__/schedule-tools.test.ts +103 -0
- package/src/__tests__/scheduler-wake.test.ts +5 -0
- package/src/__tests__/secret-routes-platform-proxy.test.ts +29 -1
- package/src/__tests__/secure-keys.test.ts +91 -0
- package/src/__tests__/server-history-render.test.ts +97 -7
- package/src/__tests__/size-guard.test.ts +8 -8
- package/src/__tests__/skill-docs-sync-guard.test.ts +3 -2
- package/src/__tests__/skill-tool-factory.test.ts +247 -0
- package/src/__tests__/skills-install-staging.test.ts +15 -6
- package/src/__tests__/slack-edit-ordering-characterization.test.ts +103 -0
- package/src/__tests__/slack-inbound-verification.test.ts +4 -1
- package/src/__tests__/slack-mention-binding-invariance.test.ts +103 -0
- package/src/__tests__/slack-mention-provider-leak-audit.test.ts +135 -0
- package/src/__tests__/starter-task-flow.test.ts +5 -4
- package/src/__tests__/steer-on-enqueue-question.test.ts +3 -2
- package/src/__tests__/stt-language-catalog-parity.test.ts +38 -0
- package/src/__tests__/subagent-allowlist-validation.test.ts +26 -4
- package/src/__tests__/subagent-cron-run-attribution.test.ts +301 -0
- package/src/__tests__/subagent-disposal.test.ts +10 -0
- package/src/__tests__/subagent-fork-notifications.test.ts +10 -0
- package/src/__tests__/subagent-fork-prompt-role.test.ts +85 -11
- package/src/__tests__/subagent-fork-spawn.test.ts +15 -7
- package/src/__tests__/subagent-manager-notify.test.ts +148 -0
- package/src/__tests__/subagent-notify-parent.test.ts +1 -1
- package/src/__tests__/subagent-persistence.test.ts +141 -0
- package/src/__tests__/subagent-role-registry.test.ts +172 -63
- package/src/__tests__/subagent-role-resolution.test.ts +88 -0
- package/src/__tests__/subagent-spawn-and-await.test.ts +48 -0
- package/src/__tests__/subagent-spawn-tool-fork.test.ts +111 -10
- package/src/__tests__/subagent-terminal-message.test.ts +88 -1
- package/src/__tests__/subagent-tool-gate-mode.test.ts +225 -10
- package/src/__tests__/subagent-tools.test.ts +1901 -108
- package/src/__tests__/surface-completion-compaction-boundary.test.ts +124 -0
- package/src/__tests__/surface-completion-in-flight-snapshot.test.ts +298 -0
- package/src/__tests__/system-prompt.test.ts +82 -7
- package/src/__tests__/tool-grant-request-escalation.test.ts +40 -1
- package/src/__tests__/tool-result-spool.test.ts +74 -18
- package/src/__tests__/tool-side-effects-documents.test.ts +174 -0
- package/src/__tests__/tool-side-effects-hook-failures.test.ts +129 -0
- package/src/__tests__/tools-audio-read.test.ts +10 -10
- package/src/__tests__/tools-get-route.test.ts +85 -12
- package/src/__tests__/tools-list-cli.test.ts +171 -0
- package/src/__tests__/top-level-renderer.test.ts +54 -1
- package/src/__tests__/trusted-contact-inline-approval-integration.test.ts +68 -1
- package/src/__tests__/trusted-contact-multichannel.test.ts +4 -1
- package/src/__tests__/turn-tail-chain.test.ts +135 -0
- package/src/__tests__/ui-choice-copy-surfaces.test.ts +6 -4
- package/src/__tests__/ui-visual-surface.test.ts +5 -4
- package/src/__tests__/ui-work-result-surface.test.ts +5 -4
- package/src/__tests__/unified-turn-context-visible-app.test.ts +7 -8
- package/src/__tests__/usage-routes.test.ts +62 -0
- package/src/__tests__/verification-outbound-delivery.test.ts +176 -0
- package/src/__tests__/visible-app-context.test.ts +20 -4
- package/src/__tests__/web-search-history.test.ts +96 -0
- package/src/__tests__/workspace-migration-139-clear-renamed-cost-profile-label.test.ts +6 -6
- package/src/__tests__/workspace-migration-140-repair-seed-pinned-memory-v3-live.test.ts +330 -0
- package/src/acp/failure-error.ts +3 -5
- package/src/agent/attachments.ts +26 -24
- package/src/agent/history-repair/history-repair.ts +27 -26
- package/src/agent/image-optimize.ts +6 -6
- package/src/agent/loop.ts +40 -22
- package/src/agent/message-types.ts +3 -3
- package/src/api/constants/document-tools.ts +55 -0
- package/src/api/events/question-answered.ts +53 -0
- package/src/api/events/tool-result.ts +7 -0
- package/src/api/index.ts +13 -0
- package/src/api/responses/conversation-message.ts +11 -0
- package/src/api/surfaces-oauth-connect.test.ts +96 -0
- package/src/api/surfaces.ts +36 -8
- package/src/approvals/guardian-channel-delivery.ts +83 -0
- package/src/approvals/guardian-expiry-notifier.ts +13 -11
- package/src/approvals/guardian-request-resolvers.ts +222 -135
- package/src/calls/__tests__/voice-session-bridge.test.ts +50 -0
- package/src/calls/media-stream-server.ts +9 -1
- package/src/calls/voice-session-bridge.ts +38 -0
- package/src/channels/__tests__/gateway-verification-sessions.test.ts +3 -3
- package/src/channels/config.ts +7 -2
- package/src/channels/gateway-verification-sessions.ts +7 -2
- package/src/channels/types.ts +7 -1
- package/src/cli/commands/__tests__/cache.test.ts +7 -4
- package/src/cli/commands/__tests__/cli-test-harness.ts +82 -0
- package/src/cli/commands/__tests__/conversations-wake.test.ts +51 -6
- package/src/cli/commands/__tests__/inference-models.test.ts +14 -0
- package/src/cli/commands/__tests__/inference-profiles.test.ts +44 -1
- package/src/cli/commands/__tests__/inference-providers.test.ts +7 -0
- package/src/cli/commands/__tests__/plugins.test.ts +692 -0
- package/src/cli/commands/__tests__/schedules.test.ts +136 -37
- package/src/cli/commands/conversations.ts +7 -3
- package/src/cli/commands/credentials.help.ts +7 -0
- package/src/cli/commands/inference-profiles.ts +9 -3
- package/src/cli/commands/inference.help.ts +16 -2
- package/src/cli/commands/mcp.help.ts +4 -2
- package/src/cli/commands/memory/memory-retrospective.ts +18 -0
- package/src/cli/commands/notifications.ts +3 -6
- package/src/cli/commands/oauth/connect-surface-guidance.test.ts +16 -0
- package/src/cli/commands/oauth/connect-surface-guidance.ts +25 -7
- package/src/cli/commands/oauth/connect.test.ts +26 -1
- package/src/cli/commands/oauth/connect.ts +4 -1
- package/src/cli/commands/oauth/index.help.ts +3 -1
- package/src/cli/commands/oauth/providers.ts +3 -0
- package/src/cli/commands/plugins.help.ts +25 -2
- package/src/cli/commands/plugins.ts +289 -10
- package/src/cli/commands/schedules.help.ts +14 -12
- package/src/cli/commands/schedules.ts +19 -5
- package/src/cli/commands/tools.help.ts +4 -2
- package/src/cli/commands/tools.ts +38 -0
- package/src/cli/lib/__tests__/inspect-plugin.test.ts +1 -0
- package/src/cli/lib/__tests__/install-from-github.test.ts +79 -0
- package/src/cli/lib/__tests__/install-from-platform.test.ts +47 -1
- package/src/cli/lib/__tests__/plugin-surfaces.test.ts +86 -7
- package/src/cli/lib/__tests__/upgrade-plugin.test.ts +122 -0
- package/src/cli/lib/install-from-github.ts +61 -0
- package/src/cli/lib/install-from-platform.ts +13 -2
- package/src/cli/lib/plugin-surfaces.ts +124 -4
- package/src/cli/lib/upgrade-plugin.ts +18 -0
- package/src/cli.ts +1 -1
- package/src/config/__tests__/deployment-context-defaults.test.ts +73 -0
- package/src/config/__tests__/memory-retrospective-schema.test.ts +40 -0
- package/src/config/__tests__/plugin-updates-schema.test.ts +43 -0
- package/src/config/__tests__/profile-tool-support.test.ts +131 -0
- package/src/config/bundled-skills/AGENTS.md +1 -1
- package/src/config/bundled-skills/app-builder/SKILL.md +1 -1
- package/src/config/bundled-skills/app-builder/references/RESPONSIVE.md +3 -3
- package/src/config/bundled-skills/schedule/SKILL.md +5 -1
- package/src/config/bundled-skills/schedule/TOOLS.json +2 -2
- package/src/config/bundled-skills/settings/TOOLS.json +2 -2
- package/src/config/bundled-skills/settings/tools/voice-config-update.ts +1 -0
- package/src/config/bundled-skills/subagent/SKILL.md +53 -22
- package/src/config/bundled-skills/subagent/TOOLS.json +14 -13
- package/src/config/bundled-skills/visualize/SKILL.md +1 -1
- package/src/config/default-profile-catalog.ts +6 -2
- package/src/config/feature-flag-registry.json +25 -17
- package/src/config/llm-resolver.ts +45 -15
- package/src/config/loader.ts +70 -1
- package/src/config/profile-tool-support.ts +110 -0
- package/src/config/sanitize-for-transfer.ts +11 -0
- package/src/config/schema.ts +4 -0
- package/src/config/schemas/__tests__/stt.test.ts +20 -5
- package/src/config/schemas/channels.ts +13 -0
- package/src/config/schemas/memory-retrospective.ts +25 -0
- package/src/config/schemas/plugin-updates.ts +55 -0
- package/src/config/schemas/services.ts +8 -4
- package/src/config/schemas/stt.ts +17 -5
- package/src/config/schemas/timeouts.ts +1 -1
- package/src/contacts/__tests__/guardian-delivery-reader.test.ts +16 -0
- package/src/contacts/guardian-delivery-reader.ts +29 -2
- package/src/context/compactor.ts +4 -4
- package/src/context/post-turn-tool-result-truncation.ts +32 -16
- package/src/context/strip-injections.ts +42 -0
- package/src/context/tool-result-spool.ts +32 -18
- package/src/credential-execution/ces-runtime.ts +3 -0
- package/src/credential-execution/client.ts +22 -1
- package/src/credential-execution/process-manager.test.ts +99 -0
- package/src/credential-execution/process-manager.ts +62 -16
- package/src/daemon/__tests__/abort-null-controller.test.ts +141 -5
- package/src/daemon/__tests__/conversation-surfaces-launch.test.ts +8 -12
- package/src/daemon/__tests__/install-assistant-command.test.ts +234 -0
- package/src/daemon/__tests__/turn-tail-assistant-reply-notify.test.ts +2 -4
- package/src/daemon/__tests__/turn-tail-deleted-conversation.test.ts +20 -29
- package/src/daemon/channel-ui-capability.ts +4 -4
- package/src/daemon/conversation-agent-loop-handlers.ts +117 -61
- package/src/daemon/conversation-agent-loop.ts +318 -148
- package/src/daemon/conversation-attachments.ts +1 -1
- package/src/daemon/conversation-error.ts +8 -1
- package/src/daemon/conversation-initial-prompt.ts +16 -38
- package/src/daemon/conversation-lifecycle.ts +154 -32
- package/src/daemon/conversation-messaging.ts +106 -43
- package/src/daemon/conversation-process.ts +131 -33
- package/src/daemon/conversation-queue-manager.ts +10 -0
- package/src/daemon/conversation-runtime-assembly.ts +9 -4
- package/src/daemon/conversation-store.ts +52 -3
- package/src/daemon/conversation-surface-state.ts +9 -0
- package/src/daemon/conversation-surfaces.ts +403 -210
- package/src/daemon/conversation-tool-setup.ts +56 -7
- package/src/daemon/conversation-turn-finalize.ts +134 -89
- package/src/daemon/conversation.ts +111 -27
- package/src/daemon/disk-pressure-guard-lifecycle.ts +2 -2
- package/src/daemon/disk-pressure-guard.ts +3 -3
- package/src/daemon/doordash-steps.ts +5 -5
- package/src/daemon/external-plugins-bootstrap.ts +7 -1
- package/src/daemon/handlers/config-channels.ts +48 -5
- package/src/daemon/handlers/conversation-history.ts +1 -1
- package/src/daemon/handlers/conversations.ts +62 -17
- package/src/daemon/handlers/shared.ts +12 -0
- package/src/daemon/handlers/skills.ts +11 -1
- package/src/daemon/identity-helpers.ts +38 -10
- package/src/daemon/install-assistant-command.ts +322 -0
- package/src/daemon/lifecycle.ts +14 -3
- package/src/daemon/message-types/conversations.ts +6 -7
- package/src/daemon/message-types/sync.ts +1 -0
- package/src/daemon/persist-media-references.ts +23 -14
- package/src/daemon/process-message.ts +6 -6
- package/src/daemon/tool-setup-types.ts +19 -122
- package/src/daemon/tool-side-effects.ts +17 -94
- package/src/daemon/trust-context-types.ts +39 -0
- package/src/daemon/trust-context.ts +14 -6
- package/src/daemon/turn-tail-chain.ts +73 -0
- package/src/daemon/web-search-history.ts +47 -33
- package/src/heartbeat/heartbeat-service.ts +6 -6
- package/src/ipc/__tests__/socket-path.test.ts +21 -0
- package/src/ipc/assistant-server.ts +17 -10
- package/src/ipc/cli-client.ts +10 -1
- package/src/ipc/routes/__tests__/documents-sync-ipc-routes.test.ts +56 -0
- package/src/ipc/routes/documents-sync-ipc-routes.ts +42 -0
- package/src/ipc/socket-cleanup.ts +9 -7
- package/src/ipc/socket-path.ts +4 -96
- package/src/live-voice/__tests__/live-voice-archive.test.ts +16 -16
- package/src/live-voice/__tests__/live-voice-attach-image.test.ts +209 -0
- package/src/live-voice/__tests__/live-voice-connection.test.ts +20 -0
- package/src/live-voice/__tests__/live-voice-integration.test.ts +7 -5
- package/src/live-voice/__tests__/live-voice-session-telemetry.test.ts +265 -0
- package/src/live-voice/__tests__/live-voice-stt.test.ts +6 -1
- package/src/live-voice/__tests__/live-voice-vad.test.ts +35 -15
- package/src/live-voice/__tests__/protocol.test.ts +52 -0
- package/src/live-voice/__tests__/runtime-websocket-shell.test.ts +3 -0
- package/src/live-voice/live-voice-archive.ts +13 -13
- package/src/live-voice/live-voice-connection.ts +7 -1
- package/src/live-voice/live-voice-photo.ts +169 -0
- package/src/live-voice/live-voice-session.ts +108 -4
- package/src/live-voice/protocol.ts +89 -1
- package/src/mcp/client.ts +10 -0
- package/src/messaging/providers/__tests__/callback-routing.test.ts +5 -1
- package/src/messaging/providers/__tests__/transport-dispatch.test.ts +66 -4
- package/src/messaging/providers/callback-routing.ts +1 -0
- package/src/messaging/providers/discord/api.ts +312 -0
- package/src/messaging/providers/discord/dm-delivery.test.ts +204 -0
- package/src/messaging/providers/discord/render.test.ts +204 -0
- package/src/messaging/providers/discord/render.ts +195 -0
- package/src/messaging/providers/discord/send.test.ts +179 -0
- package/src/messaging/providers/discord/send.ts +212 -0
- package/src/messaging/providers/discord/transport.ts +81 -0
- package/src/messaging/providers/index.ts +3 -1
- package/src/messaging/providers/retry-policy.test.ts +247 -0
- package/src/messaging/providers/retry-policy.ts +206 -0
- package/src/messaging/providers/slack/mention-source.test.ts +285 -0
- package/src/messaging/providers/slack/mention-source.ts +287 -0
- package/src/messaging/providers/telegram-bot/api.ts +47 -130
- package/src/messaging/providers/whatsapp/api.ts +47 -136
- package/src/monitoring/__tests__/db-integrity-sample.test.ts +4 -1
- package/src/monitoring/__tests__/plugin-auto-update.test.ts +332 -0
- package/src/monitoring/db-integrity-sample.ts +6 -1
- package/src/monitoring/plugin-auto-update.ts +359 -0
- package/src/monitoring/resource-sampler.ts +21 -5
- package/src/monitoring/worker.ts +13 -0
- package/src/notifications/__tests__/assistant-reply-producer.test.ts +375 -2
- package/src/notifications/__tests__/broadcaster.test.ts +37 -1
- package/src/notifications/__tests__/copy-composer.test.ts +165 -0
- package/src/notifications/__tests__/emit-signal-pipeline-failure.test.ts +205 -0
- package/src/notifications/__tests__/notification-utils.test.ts +249 -0
- package/src/notifications/adapters/platform.ts +31 -2
- package/src/notifications/assistant-reply-producer.ts +79 -32
- package/src/notifications/broadcaster.ts +30 -10
- package/src/notifications/copy-composer.ts +69 -0
- package/src/notifications/emit-signal.ts +82 -15
- package/src/notifications/events-store.ts +10 -2
- package/src/notifications/home-feed-side-effect.ts +6 -3
- package/src/notifications/notification-utils.ts +173 -5
- package/src/notifications/signal.ts +12 -0
- package/src/oauth/AGENTS.md +4 -2
- package/src/oauth/byo-connection.ts +6 -1
- package/src/oauth/credential-token-resolver.ts +3 -10
- package/src/oauth/manual-token-connection.ts +29 -64
- package/src/oauth/manual-token-providers.ts +62 -0
- package/src/oauth/provider-serializer.ts +13 -0
- package/src/oauth/seed-providers.ts +62 -11
- package/src/onboarding/onboarding-events-store.ts +91 -1
- package/src/permissions/question-prompter.test.ts +4 -0
- package/src/permissions/question-prompter.ts +21 -4
- package/src/persistence/__tests__/conversation-lineage.test.ts +248 -0
- package/src/persistence/attachments-store.ts +15 -15
- package/src/persistence/conversation-attention-store.ts +30 -10
- package/src/persistence/conversation-bootstrap.ts +17 -0
- package/src/persistence/conversation-crud.ts +363 -97
- package/src/persistence/conversation-group-migration.ts +1 -1
- package/src/persistence/conversation-key-store.ts +38 -10
- package/src/persistence/conversation-lineage.ts +226 -0
- package/src/persistence/conversation-plugin-facade.ts +28 -3
- package/src/persistence/conversation-queries.ts +207 -33
- package/src/persistence/conversation-types.ts +107 -1
- package/src/persistence/jobs-store.ts +32 -8
- package/src/persistence/llm-usage-store.ts +22 -0
- package/src/persistence/migrations/361-normalize-managed-connection-rows.test.ts +168 -0
- package/src/persistence/migrations/361-normalize-managed-connection-rows.ts +76 -0
- package/src/persistence/migrations/362-add-conversation-subagent-kind.test.ts +187 -0
- package/src/persistence/migrations/362-add-conversation-subagent-kind.ts +93 -0
- package/src/persistence/migrations/363-backfill-schedule-inference-profile.test.ts +229 -0
- package/src/persistence/migrations/363-backfill-schedule-inference-profile.ts +127 -0
- package/src/persistence/migrations/364-add-schedule-source-key.test.ts +120 -0
- package/src/persistence/migrations/364-add-schedule-source-key.ts +43 -0
- package/src/persistence/migrations/365-add-conversation-fork-strategy.ts +27 -0
- package/src/persistence/schema/conversations.ts +32 -0
- package/src/persistence/schema/infrastructure.ts +3 -0
- package/src/persistence/steps.ts +30 -0
- package/src/persistence/subagent-store.ts +148 -6
- package/src/platform/managed-speech.test.ts +20 -0
- package/src/platform/managed-speech.ts +9 -0
- package/src/plugin-api/index.ts +8 -0
- package/src/plugin-api/webhook-url.ts +151 -0
- package/src/plugins/__tests__/installed-plugin-dirs.test.ts +82 -0
- package/src/plugins/__tests__/source-fingerprint.test.ts +5 -1
- package/src/plugins/collect-source-versions.ts +4 -27
- package/src/plugins/defaults/exploration-drift/hooks/post-tool-use.ts +5 -5
- package/src/plugins/defaults/exploration-drift/package.json +1 -1
- package/src/plugins/defaults/image-recovery/hooks/post-model-call.ts +4 -2
- package/src/plugins/defaults/image-recovery/recover.ts +67 -57
- package/src/plugins/defaults/index.ts +1 -1
- package/src/plugins/defaults/memory/__tests__/conversation-queries.test.ts +64 -18
- package/src/plugins/defaults/memory/__tests__/jobs-worker-outcome-contract.test.ts +242 -0
- package/src/plugins/defaults/memory/__tests__/jobs-worker-retrospective-sweep.test.ts +24 -1
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-enqueue.test.ts +48 -3
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-job.test.ts +511 -19
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-prompt.test.ts +39 -3
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-provider-path.test.ts +493 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-wake-chain.test.ts +5 -4
- package/src/plugins/defaults/memory/__tests__/memory-tier-boundary-guard.test.ts +5 -1
- package/src/plugins/defaults/memory/job-handlers.ts +87 -5
- package/src/plugins/defaults/memory/jobs-worker.ts +60 -6
- package/src/plugins/defaults/memory/memory-retrospective-constants.ts +11 -0
- package/src/plugins/defaults/memory/memory-retrospective-enqueue.ts +29 -0
- package/src/plugins/defaults/memory/memory-retrospective-job.ts +342 -60
- package/src/plugins/defaults/memory/memory-retrospective-prompt.ts +23 -3
- package/src/plugins/defaults/memory/substrate/consolidation-lock.ts +9 -4
- package/src/plugins/defaults/memory/substrate/sweep-job.ts +7 -0
- package/src/plugins/defaults/memory/v1/graph/injection.ts +1 -1
- package/src/plugins/defaults/memory/v2/router.ts +8 -0
- package/src/plugins/defaults/turn-context/unified-turn-context.ts +1 -2
- package/src/plugins/external-plugin-loader.ts +17 -8
- package/src/plugins/installed-plugin-dirs.ts +101 -0
- package/src/plugins/mtime-cache.ts +34 -22
- package/src/plugins/types.ts +69 -8
- package/src/prompts/persona-resolver.ts +6 -2
- package/src/prompts/system-prompt.ts +72 -17
- package/src/providers/__tests__/inference.test.ts +40 -0
- package/src/providers/__tests__/preflight-resolved-config.test.ts +58 -1
- package/src/providers/__tests__/retry-callsite.test.ts +67 -0
- package/src/providers/__tests__/vellum-connection-routing.test.ts +42 -11
- package/src/providers/anthropic/client.ts +112 -95
- package/src/providers/connection-resolution.ts +35 -1
- package/src/providers/gemini/client.ts +5 -5
- package/src/providers/inference/adapter-factory.ts +16 -10
- package/src/providers/inference/auth.ts +16 -0
- package/src/providers/inference/connection-availability.ts +164 -12
- package/src/providers/inference/connections.ts +25 -1
- package/src/providers/media-resolve.ts +21 -11
- package/src/providers/model-catalog.ts +15 -0
- package/src/providers/openai/chat-completions-provider.ts +4 -4
- package/src/providers/openai/responses-provider.ts +12 -25
- package/src/providers/registry.ts +9 -4
- package/src/providers/retry.ts +25 -0
- package/src/providers/server-tool-pairing.ts +134 -0
- package/src/providers/speech-to-text/__tests__/resolve.test.ts +187 -6
- package/src/providers/speech-to-text/deepgram.ts +1 -0
- package/src/providers/speech-to-text/resolve.ts +80 -10
- package/src/providers/speech-to-text/vellum-managed.ts +7 -1
- package/src/providers/speech-to-text/vellum-speech-relay-connection.ts +3 -1
- package/src/providers/types.ts +26 -6
- package/src/providers/vellum-model-routing.ts +6 -7
- package/src/runtime/AGENTS.md +2 -0
- package/src/runtime/__tests__/agent-wake.test.ts +16 -16
- package/src/runtime/__tests__/background-job-runner.test.ts +0 -144
- package/src/runtime/__tests__/desktop-presence.test.ts +162 -0
- package/src/runtime/agent-wake.ts +63 -24
- package/src/runtime/assistant-event-hub.ts +87 -12
- package/src/runtime/auth/__tests__/guard-tests.test.ts +2 -0
- package/src/runtime/auth/__tests__/scopes.test.ts +20 -0
- package/src/runtime/auth/same-actor.ts +56 -8
- package/src/runtime/auth/scopes.ts +12 -0
- package/src/runtime/auth/types.ts +2 -0
- package/src/runtime/background-job-runner.ts +5 -27
- package/src/runtime/channel-readiness-service.ts +59 -3
- package/src/runtime/channel-readiness-types.ts +18 -0
- package/src/runtime/desktop-presence.ts +69 -0
- package/src/runtime/http-server.ts +12 -0
- package/src/runtime/local-actor-identity.ts +11 -9
- package/src/runtime/migrations/__tests__/vbundle-symlink-streaming-importer.test.ts +0 -0
- package/src/runtime/routes/__tests__/channel-verification-routes.test.ts +1 -1
- package/src/runtime/routes/__tests__/client-routes.test.ts +186 -14
- package/src/runtime/routes/__tests__/conversation-list-routes.test.ts +547 -1
- package/src/runtime/routes/__tests__/conversation-management-routes.test.ts +162 -7
- package/src/runtime/routes/__tests__/conversation-query-routes.test.ts +14 -10
- package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +340 -3
- package/src/runtime/routes/__tests__/llm-call-sites-routes.test.ts +28 -1
- package/src/runtime/routes/__tests__/plugins-routes.test.ts +173 -1
- package/src/runtime/routes/__tests__/question-routes.test.ts +64 -0
- package/src/runtime/routes/__tests__/retrospective-routes.test.ts +20 -0
- package/src/runtime/routes/__tests__/skill-history-route.test.ts +116 -0
- package/src/runtime/routes/__tests__/wake-conversation-routes.test.ts +60 -2
- package/src/runtime/routes/archive-utils.ts +16 -12
- package/src/runtime/routes/attachment-routes.ts +13 -6
- package/src/runtime/routes/canned-reply-release.ts +54 -0
- package/src/runtime/routes/channel-verification-routes.ts +2 -27
- package/src/runtime/routes/chatgpt-subscription-auth-routes.ts +5 -0
- package/src/runtime/routes/client-routes.ts +114 -2
- package/src/runtime/routes/conversation-cli-routes.ts +10 -12
- package/src/runtime/routes/conversation-list-routes.ts +59 -10
- package/src/runtime/routes/conversation-management-routes.ts +47 -18
- package/src/runtime/routes/conversation-query-routes.ts +32 -53
- package/src/runtime/routes/conversation-routes.ts +360 -293
- package/src/runtime/routes/credential-prompt-routes.ts +23 -2
- package/src/runtime/routes/debug-routes.ts +4 -1
- package/src/runtime/routes/default-provider-routes.ts +7 -11
- package/src/runtime/routes/documents-routes.ts +44 -1
- package/src/runtime/routes/events-routes.ts +28 -1
- package/src/runtime/routes/host-app-control-routes.ts +2 -0
- package/src/runtime/routes/host-bash-routes.ts +2 -0
- package/src/runtime/routes/host-browser-routes.ts +2 -0
- package/src/runtime/routes/host-cu-routes.ts +2 -0
- package/src/runtime/routes/host-file-routes.ts +2 -0
- package/src/runtime/routes/host-transfer-routes.ts +4 -0
- package/src/runtime/routes/identity-routes.ts +6 -6
- package/src/runtime/routes/inbound-message-handler.ts +5 -5
- package/src/runtime/routes/inbound-stages/acl-enforcement.ts +7 -3
- package/src/runtime/routes/inbound-stages/background-dispatch.ts +14 -0
- package/src/runtime/routes/inbound-stages/bootstrap-intercept.ts +2 -2
- package/src/runtime/routes/inbound-stages/guardian-activation-intercept.test.ts +14 -11
- package/src/runtime/routes/inbound-stages/guardian-activation-intercept.ts +14 -17
- package/src/runtime/routes/inference-profile-availability-guard.ts +171 -0
- package/src/runtime/routes/inference-profile-session-handler.ts +26 -2
- package/src/runtime/routes/inference-profile-session-routes.ts +2 -1
- package/src/runtime/routes/inference-profiles-routes.ts +152 -73
- package/src/runtime/routes/inference-provider-connection-routes.ts +0 -16
- package/src/runtime/routes/llm-call-sites-routes.ts +13 -0
- package/src/runtime/routes/log-export-routes.ts +1 -1
- package/src/runtime/routes/notification-routes.ts +0 -14
- package/src/runtime/routes/oauth-providers.ts +1 -0
- package/src/runtime/routes/plugins-routes.ts +106 -1
- package/src/runtime/routes/profiler-routes.ts +4 -2
- package/src/runtime/routes/question-routes.ts +28 -10
- package/src/runtime/routes/retrospective-routes.ts +9 -3
- package/src/runtime/routes/schedule-routes.ts +210 -12
- package/src/runtime/routes/secret-routes.ts +40 -23
- package/src/runtime/routes/settings-routes.ts +41 -13
- package/src/runtime/routes/skills-routes.ts +79 -0
- package/src/runtime/routes/ui-snapshot-routes.ts +1 -0
- package/src/runtime/routes/wake-conversation-routes.ts +23 -0
- package/src/runtime/services/__tests__/conversation-serializer.test.ts +1 -0
- package/src/runtime/services/conversation-serializer.ts +24 -9
- package/src/runtime/sync/documents-sidecar-publish.test.ts +101 -0
- package/src/runtime/sync/resource-sync-events.ts +79 -1
- package/src/runtime/sync/worker-daemon-notify.test.ts +34 -0
- package/src/runtime/sync/worker-daemon-notify.ts +44 -6
- package/src/runtime/verification-outbound-actions.ts +250 -222
- package/src/runtime/verification-templates.ts +41 -30
- package/src/schedule/__tests__/plugin-schedule-declarations.test.ts +614 -0
- package/src/schedule/__tests__/plugin-schedule-reconciler.test.ts +1066 -0
- package/src/schedule/inference-profile.ts +64 -3
- package/src/schedule/plugin-schedule-declarations.ts +518 -0
- package/src/schedule/plugin-schedule-reconciler.ts +605 -0
- package/src/schedule/plugin-schedules-gate.ts +23 -0
- package/src/schedule/schedule-store.ts +520 -26
- package/src/schedule/scheduler.ts +38 -0
- package/src/security/credential-backend.ts +51 -4
- package/src/security/encrypted-store.ts +64 -42
- package/src/security/secure-keys.ts +55 -7
- package/src/skills/catalog-install.ts +20 -7
- package/src/skills/inline-command-runner.ts +2 -7
- package/src/skills/skill-history.test.ts +269 -0
- package/src/skills/skill-history.ts +250 -0
- package/src/skills/skillssh-registry.ts +1 -1
- package/src/stt/daemon-batch-transcriber.ts +8 -6
- package/src/subagent/__tests__/consult-prompt.test.ts +25 -0
- package/src/subagent/consult-prompt.ts +3 -1
- package/src/subagent/consult-transcript.ts +1 -1
- package/src/subagent/manager.ts +304 -44
- package/src/subagent/progress-events.ts +35 -0
- package/src/subagent/role-resolution.ts +88 -0
- package/src/subagent/types.ts +302 -67
- package/src/subagent/validate-allowlists.ts +3 -4
- package/src/telegram/__tests__/webhook-health.test.ts +73 -1
- package/src/telegram/webhook-health.ts +101 -3
- package/src/telemetry/__tests__/telemetry-event-fixtures.ts +2 -0
- package/src/telemetry/activation-funnel.ts +6 -1
- package/src/telemetry/live-voice-funnel.ts +75 -0
- package/src/telemetry/telemetry-event-sources.ts +6 -0
- package/src/telemetry/telemetry-wire-source.json +1 -1
- package/src/telemetry/telemetry-wire.generated.ts +2 -0
- package/src/telemetry/types.ts +23 -4
- package/src/telemetry/usage-telemetry-reporter.test.ts +4 -0
- package/src/tools/__tests__/subagent-workflow-tool-input-schemas.test.ts +1 -1
- package/src/tools/ask-question/ask-question-tool.test.ts +118 -3
- package/src/tools/ask-question/ask-question-tool.ts +56 -2
- package/src/tools/browser/browser-execution.ts +1 -1
- package/src/tools/calls/call-start.ts +8 -5
- package/src/tools/conversation-groups/move_to_group.ts +10 -12
- package/src/tools/document/document-tool.ts +11 -4
- package/src/tools/execution-timeout.ts +9 -2
- package/src/tools/executor.ts +7 -6
- package/src/tools/filesystem/edit.ts +1 -1
- package/src/tools/filesystem/list.ts +1 -1
- package/src/tools/filesystem/read.ts +7 -13
- package/src/tools/filesystem/search.ts +23 -21
- package/src/tools/filesystem/write.ts +4 -1
- package/src/tools/host-filesystem/edit.ts +7 -2
- package/src/tools/host-filesystem/read.ts +21 -8
- package/src/tools/host-filesystem/write.ts +10 -2
- package/src/tools/host-terminal/host-shell.ts +2 -2
- package/src/tools/mcp/mcp-tool-factory.ts +25 -1
- package/src/tools/permission-checker.ts +8 -3
- package/src/tools/schedule/delete.ts +13 -1
- package/src/tools/schedule/list.ts +14 -3
- package/src/tools/schedule/update.ts +56 -33
- package/src/tools/shared/filesystem/audio-read.ts +6 -4
- package/src/tools/shared/filesystem/file-ops-service.ts +176 -116
- package/src/tools/shared/filesystem/image-read.ts +12 -12
- package/src/tools/shared/filesystem/path-policy.ts +70 -5
- package/src/tools/shared/filesystem/size-guard.ts +6 -6
- package/src/tools/shared/input-misuse.ts +72 -0
- package/src/tools/skills/scaffold-managed.ts +82 -0
- package/src/tools/skills/skill-tool-factory.ts +14 -1
- package/src/tools/subagent/consult-deadline.ts +12 -7
- package/src/tools/subagent/message.ts +3 -1
- package/src/tools/subagent/read.ts +66 -4
- package/src/tools/subagent/spawn.ts +500 -78
- package/src/tools/tool-approval-handler.ts +45 -4
- package/src/tools/types.ts +22 -3
- package/src/tools/ui-surface/surface-shape-docs.ts +1 -1
- package/src/tools/ui-surface/visual-validation.ts +2 -0
- package/src/usage/__tests__/subagent-attribution.test.ts +109 -0
- package/src/usage/subagent-attribution.ts +108 -0
- package/src/util/__tests__/sqlite-retry.test.ts +112 -0
- package/src/util/abort-reasons.ts +16 -0
- package/src/util/ansi.ts +44 -0
- package/src/util/disk-usage.ts +17 -13
- package/src/util/image-conversion.ts +20 -13
- package/src/util/schedule-source-key.ts +23 -0
- package/src/util/worker-process.ts +7 -1
- package/src/workspace/git-service.ts +33 -9
- package/src/workspace/migrations/140-repair-seed-pinned-memory-v3-live.ts +260 -0
- package/src/workspace/migrations/141-stt-english-default-to-multilingual.ts +111 -0
- package/src/workspace/migrations/__tests__/141-stt-english-default-to-multilingual.test.ts +126 -0
- package/src/workspace/migrations/registry.ts +4 -0
- package/src/workspace/top-level-renderer.ts +15 -2
- package/src/__tests__/tool-side-effects-slack-dm.test.ts +0 -357
- package/src/daemon/install-symlink.ts +0 -207
- package/src/notifications/deferred-emit.ts +0 -147
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Persisting a photo taken during a live-voice call.
|
|
3
|
+
*
|
|
4
|
+
* A photo lands in the conversation as its own user message, the moment the
|
|
5
|
+
* shutter fires, and **runs no turn**. That single choice is what makes the
|
|
6
|
+
* order of shutter and speech irrelevant: whatever the user says next, before
|
|
7
|
+
* or after the snap, is answered by a model whose history already contains the
|
|
8
|
+
* image.
|
|
9
|
+
*
|
|
10
|
+
* The alternatives were both worse and both were tried. Attaching the photo to
|
|
11
|
+
* the *next* spoken turn strands it whenever the user speaks first: the
|
|
12
|
+
* question "what's this?" dispatches without an image and the photo waits for
|
|
13
|
+
* a sentence that has already been said. Dispatching a turn *for* the photo
|
|
14
|
+
* races the sentence the user is in the middle of saying, so the assistant
|
|
15
|
+
* answers the picture and the words separately.
|
|
16
|
+
*
|
|
17
|
+
* The cost, accepted: media survives a context-overflow retry only on the most
|
|
18
|
+
* recent user message (`conversation-media-retry.ts`), so on a long call a
|
|
19
|
+
* photo can be stripped where one riding the spoken turn would have survived.
|
|
20
|
+
* Stranding is the worse failure, and it happens every time rather than only
|
|
21
|
+
* under overflow.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
import { v7 as uuidv7 } from "uuid";
|
|
25
|
+
|
|
26
|
+
import { persistQueuedMessageBody } from "../daemon/conversation-messaging.js";
|
|
27
|
+
import { getOrCreateConversation } from "../daemon/conversation-store.js";
|
|
28
|
+
import {
|
|
29
|
+
getAttachmentsByIds,
|
|
30
|
+
getSourcePathsForAttachments,
|
|
31
|
+
} from "../persistence/attachments-store.js";
|
|
32
|
+
import { recordConversationPersistedSeq } from "../persistence/conversation-crud.js";
|
|
33
|
+
import { broadcastMessage } from "../runtime/assistant-event-hub.js";
|
|
34
|
+
import { getCurrentSeq } from "../runtime/assistant-stream-state.js";
|
|
35
|
+
import { publishConversationMessagesChanged } from "../runtime/sync/resource-sync-events.js";
|
|
36
|
+
import { getLogger } from "../util/logger.js";
|
|
37
|
+
|
|
38
|
+
const log = getLogger("live-voice-photo");
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* How long to wait for an in-flight turn before giving up on a photo.
|
|
42
|
+
*
|
|
43
|
+
* Persisting takes the conversation's processing lock, which a running turn
|
|
44
|
+
* holds for as long as it runs, tools included. The wait is generous because
|
|
45
|
+
* the alternative is dropping a photo the user watched themselves take, and
|
|
46
|
+
* the photo is not urgent: nothing is blocked on it except the next thing they
|
|
47
|
+
* say.
|
|
48
|
+
*/
|
|
49
|
+
const PROCESSING_WAIT_MS = 30_000;
|
|
50
|
+
const PROCESSING_POLL_MS = 100;
|
|
51
|
+
|
|
52
|
+
/** What the user message says. The image blocks carry the actual content. */
|
|
53
|
+
const PHOTO_MESSAGE_CONTENT = "[photo]";
|
|
54
|
+
|
|
55
|
+
export interface LiveVoicePhotoResult {
|
|
56
|
+
readonly ok: boolean;
|
|
57
|
+
readonly messageId?: string;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Hydrate attachment ids into the shape `persistQueuedMessageBody` stores.
|
|
62
|
+
* Mirrors the HTTP send path's own `resolveAttachments`, so a photo taken
|
|
63
|
+
* mid-call is stored exactly like one attached to a typed message.
|
|
64
|
+
*/
|
|
65
|
+
function resolvePhotoAttachments(attachmentIds: string[]) {
|
|
66
|
+
const resolved = getAttachmentsByIds(attachmentIds, {
|
|
67
|
+
hydrateFileData: true,
|
|
68
|
+
});
|
|
69
|
+
const sourcePaths = getSourcePathsForAttachments(attachmentIds);
|
|
70
|
+
return resolved.map((a) => ({
|
|
71
|
+
id: a.id,
|
|
72
|
+
filename: a.originalFilename,
|
|
73
|
+
mimeType: a.mimeType,
|
|
74
|
+
data: a.dataBase64,
|
|
75
|
+
...(sourcePaths.has(a.id) ? { filePath: sourcePaths.get(a.id) } : {}),
|
|
76
|
+
}));
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** Resolve once the conversation is not mid-turn, or false on timeout. */
|
|
80
|
+
async function waitForIdle(conversation: {
|
|
81
|
+
isProcessing: () => boolean;
|
|
82
|
+
}): Promise<boolean> {
|
|
83
|
+
const deadline = Date.now() + PROCESSING_WAIT_MS;
|
|
84
|
+
while (conversation.isProcessing()) {
|
|
85
|
+
if (Date.now() >= deadline) {
|
|
86
|
+
return false;
|
|
87
|
+
}
|
|
88
|
+
await new Promise((resolve) => setTimeout(resolve, PROCESSING_POLL_MS));
|
|
89
|
+
}
|
|
90
|
+
return true;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Persist a photo into the conversation as a user message, running no turn.
|
|
95
|
+
*
|
|
96
|
+
* Takes and releases the processing lock the way the built-in slash commands
|
|
97
|
+
* do: they are the existing precedent for "persist a user message and answer
|
|
98
|
+
* without the agent loop", and the lock is what keeps this write from
|
|
99
|
+
* interleaving with a turn's own persist.
|
|
100
|
+
*
|
|
101
|
+
* Never throws. A photo that cannot be stored must not take the call down with
|
|
102
|
+
* it, and the caller reports the failure to the user instead.
|
|
103
|
+
*/
|
|
104
|
+
export async function persistLiveVoicePhoto(
|
|
105
|
+
conversationId: string,
|
|
106
|
+
attachmentId: string,
|
|
107
|
+
): Promise<LiveVoicePhotoResult> {
|
|
108
|
+
try {
|
|
109
|
+
const attachments = resolvePhotoAttachments([attachmentId]);
|
|
110
|
+
if (attachments.length === 0) {
|
|
111
|
+
log.warn({ attachmentId }, "Live-voice photo attachment did not resolve");
|
|
112
|
+
return { ok: false };
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
const conversation = await getOrCreateConversation(conversationId);
|
|
116
|
+
|
|
117
|
+
// A turn holds the lock for its whole run. Waiting rather than queueing:
|
|
118
|
+
// the conversation's queue drains into a turn, which is the one thing this
|
|
119
|
+
// must not cause.
|
|
120
|
+
if (!(await waitForIdle(conversation))) {
|
|
121
|
+
log.warn(
|
|
122
|
+
{ conversationId, attachmentId },
|
|
123
|
+
"Live-voice photo timed out waiting for the conversation to go idle",
|
|
124
|
+
);
|
|
125
|
+
return { ok: false };
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
conversation.setProcessing(true);
|
|
129
|
+
try {
|
|
130
|
+
const persisted = await persistQueuedMessageBody(conversation, {
|
|
131
|
+
content: PHOTO_MESSAGE_CONTENT,
|
|
132
|
+
attachments,
|
|
133
|
+
requestId: uuidv7(),
|
|
134
|
+
metadata: {
|
|
135
|
+
// Marks the row as something the user did on a call rather than
|
|
136
|
+
// typed, the same way a voice turn's own user message is marked.
|
|
137
|
+
voiceSessionTurn: true,
|
|
138
|
+
livePhoto: true,
|
|
139
|
+
},
|
|
140
|
+
});
|
|
141
|
+
|
|
142
|
+
// The row exists but no turn will announce it, so the clients that
|
|
143
|
+
// render this conversation have to be told directly, or the photo does
|
|
144
|
+
// not appear until something else forces a refetch.
|
|
145
|
+
broadcastMessage({
|
|
146
|
+
type: "user_message_echo",
|
|
147
|
+
text: PHOTO_MESSAGE_CONTENT,
|
|
148
|
+
conversationId,
|
|
149
|
+
messageId: persisted.id,
|
|
150
|
+
});
|
|
151
|
+
recordConversationPersistedSeq(conversationId, getCurrentSeq());
|
|
152
|
+
publishConversationMessagesChanged(conversationId);
|
|
153
|
+
|
|
154
|
+
return { ok: true, messageId: persisted.id };
|
|
155
|
+
} finally {
|
|
156
|
+
conversation.setProcessing(false);
|
|
157
|
+
// Anything queued behind the lock we just held still has to run. Without
|
|
158
|
+
// this a message queued during the photo's write sits until the next
|
|
159
|
+
// turn ends.
|
|
160
|
+
void conversation.kickDrainQueue("loop_complete", "live_voice_photo");
|
|
161
|
+
}
|
|
162
|
+
} catch (err) {
|
|
163
|
+
log.warn(
|
|
164
|
+
{ err, conversationId, attachmentId },
|
|
165
|
+
"Failed to persist a live-voice photo",
|
|
166
|
+
);
|
|
167
|
+
return { ok: false };
|
|
168
|
+
}
|
|
169
|
+
}
|
|
@@ -38,6 +38,10 @@ import {
|
|
|
38
38
|
import { ABORT_WATCHDOG_MS } from "../daemon/abort-watchdog.js";
|
|
39
39
|
import { isRefusedInReadOnlyPass } from "../daemon/conversation-tool-setup.js";
|
|
40
40
|
import type { TrustContext } from "../daemon/trust-context-types.js";
|
|
41
|
+
import {
|
|
42
|
+
recordLiveVoiceSessionEnded,
|
|
43
|
+
recordLiveVoiceSessionStarted,
|
|
44
|
+
} from "../onboarding/onboarding-events-store.js";
|
|
41
45
|
import { isInstalledStaticSkillLoad } from "../permissions/checker.js";
|
|
42
46
|
import { ensureConversationExists } from "../persistence/conversation-crud.js";
|
|
43
47
|
import {
|
|
@@ -54,6 +58,7 @@ import type {
|
|
|
54
58
|
SttStreamServerEvent,
|
|
55
59
|
} from "../stt/types.js";
|
|
56
60
|
import { getSubagentManager } from "../subagent/index.js";
|
|
61
|
+
import { liveVoiceEndScreen } from "../telemetry/live-voice-funnel.js";
|
|
57
62
|
import { getToolOwner } from "../tools/registry.js";
|
|
58
63
|
import { extractSpeakableSegments } from "../tts/speakable-segments.js";
|
|
59
64
|
import { createAbortReason } from "../util/abort-reasons.js";
|
|
@@ -84,6 +89,7 @@ import {
|
|
|
84
89
|
type LiveVoiceTurnSeedMarks,
|
|
85
90
|
type VoiceEndpointAction,
|
|
86
91
|
} from "./live-voice-metrics.js";
|
|
92
|
+
import { persistLiveVoicePhoto } from "./live-voice-photo.js";
|
|
87
93
|
import {
|
|
88
94
|
type LiveVoiceSession as LiveVoiceSessionContract,
|
|
89
95
|
type LiveVoiceSessionCloseReason,
|
|
@@ -97,6 +103,7 @@ import type {
|
|
|
97
103
|
} from "./live-voice-tts.js";
|
|
98
104
|
import { pickProgressPhrase } from "./progress-phrases.js";
|
|
99
105
|
import {
|
|
106
|
+
type LiveVoiceClientAttachImageFrame,
|
|
100
107
|
type LiveVoiceClientFrame,
|
|
101
108
|
type LiveVoiceClientUpdateConfigFrame,
|
|
102
109
|
LiveVoiceProtocolErrorCode,
|
|
@@ -972,6 +979,16 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
972
979
|
private outboundFrames: Promise<void> = Promise.resolve();
|
|
973
980
|
private activeAssistantTurn: ActiveAssistantTurn | null = null;
|
|
974
981
|
private sessionEndMetricsEmitted = false;
|
|
982
|
+
/**
|
|
983
|
+
* Protocol error code of the failure that killed the session, latched by
|
|
984
|
+
* {@link sendFrame} when an error frame goes out on an already-`failed`
|
|
985
|
+
* session. Both fatal paths (`failStartup` before `ready`, a failed
|
|
986
|
+
* utterance arm after it) set `state = "failed"` before sending, and no
|
|
987
|
+
* other error frame does, so this catches exactly the session-ending
|
|
988
|
+
* failures and ignores the recoverable mid-session ones. `null` on a
|
|
989
|
+
* session that never failed.
|
|
990
|
+
*/
|
|
991
|
+
private failureCode: LiveVoiceProtocolErrorCode | null = null;
|
|
975
992
|
// Non-null iff the start frame requested turnDetection "server_vad".
|
|
976
993
|
private readonly turnDetector: MediaTurnDetector | null;
|
|
977
994
|
// Energy gate for server-VAD speech classification; undefined defers to
|
|
@@ -1184,6 +1201,12 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1184
1201
|
return;
|
|
1185
1202
|
}
|
|
1186
1203
|
|
|
1204
|
+
// Before the preflight, not after: a session rejected for missing
|
|
1205
|
+
// credentials is precisely the one the failure rate needs to count, and
|
|
1206
|
+
// recording the start only once `ready` goes out would hide every such
|
|
1207
|
+
// session from both the numerator and the denominator.
|
|
1208
|
+
recordLiveVoiceSessionStarted(this.context.sessionId);
|
|
1209
|
+
|
|
1187
1210
|
if (this.resolveCredentialReadiness) {
|
|
1188
1211
|
const readiness = await this.resolveCredentialReadiness();
|
|
1189
1212
|
if (readiness.status === "not-ready") {
|
|
@@ -1237,9 +1260,44 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1237
1260
|
case "update_config":
|
|
1238
1261
|
this.applyConfigUpdate(frame);
|
|
1239
1262
|
return;
|
|
1263
|
+
case "attach_image":
|
|
1264
|
+
this.persistPhoto(frame);
|
|
1265
|
+
return;
|
|
1240
1266
|
}
|
|
1241
1267
|
}
|
|
1242
1268
|
|
|
1269
|
+
/**
|
|
1270
|
+
* Persist a photo taken mid-call into the conversation, running no turn.
|
|
1271
|
+
*
|
|
1272
|
+
* Fire-and-forget on purpose: the persist waits out any in-flight turn, and
|
|
1273
|
+
* the socket must keep pumping audio meanwhile. The client already showed a
|
|
1274
|
+
* thumbnail from the local frame, so nothing on screen is waiting on this.
|
|
1275
|
+
*
|
|
1276
|
+
* The photo becomes its own user message rather than riding the next spoken
|
|
1277
|
+
* turn, which is what makes shutter-then-speak and speak-then-shutter
|
|
1278
|
+
* behave the same: either way the model's history has the image by the time
|
|
1279
|
+
* it answers. See `live-voice-photo.ts` for the full reasoning.
|
|
1280
|
+
*/
|
|
1281
|
+
private persistPhoto(frame: LiveVoiceClientAttachImageFrame): void {
|
|
1282
|
+
void persistLiveVoicePhoto(this.conversationId, frame.attachmentId).then(
|
|
1283
|
+
(result) => {
|
|
1284
|
+
if (!result.ok && !this.isClosed) {
|
|
1285
|
+
void this.sendFrame({
|
|
1286
|
+
type: "error",
|
|
1287
|
+
code: LiveVoiceProtocolErrorCode.InvalidFrame,
|
|
1288
|
+
message: "Could not attach that photo to the conversation.",
|
|
1289
|
+
// Names the photo as the casualty so the client can retract the
|
|
1290
|
+
// thumbnail it already showed, rather than filing this with the
|
|
1291
|
+
// transient transcriber and TTS blips that share `recoverable`.
|
|
1292
|
+
frameType: "attach_image",
|
|
1293
|
+
// The session is fine; only this photo failed.
|
|
1294
|
+
recoverable: true,
|
|
1295
|
+
});
|
|
1296
|
+
}
|
|
1297
|
+
},
|
|
1298
|
+
);
|
|
1299
|
+
}
|
|
1300
|
+
|
|
1243
1301
|
/**
|
|
1244
1302
|
* Apply a mid-session `update_config` frame: retune the live turn detector's
|
|
1245
1303
|
* pause ("pause before reply") and/or the barge-in guard ("interrupt
|
|
@@ -1261,11 +1319,24 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
1261
1319
|
await this.handleAudio(Buffer.from(chunk));
|
|
1262
1320
|
}
|
|
1263
1321
|
|
|
1264
|
-
async close(
|
|
1322
|
+
async close(reason: LiveVoiceSessionCloseReason): Promise<void> {
|
|
1265
1323
|
if (this.isClosed) {
|
|
1266
1324
|
return;
|
|
1267
1325
|
}
|
|
1268
1326
|
|
|
1327
|
+
// Recorded first, and independently of `shouldEmitSessionEndMetrics`
|
|
1328
|
+
// below: that flag governs the client-facing `metrics` frame, and a
|
|
1329
|
+
// failed session (which suppresses the frame) is the one whose end
|
|
1330
|
+
// telemetry matters most. Session duration downstream is this row's
|
|
1331
|
+
// `recorded_at` minus the started row's, so it must be written before the
|
|
1332
|
+
// teardown below, which awaits a pending continuation and can run long.
|
|
1333
|
+
const failed = this.state === "failed";
|
|
1334
|
+
recordLiveVoiceSessionEnded({
|
|
1335
|
+
sessionId: this.context.sessionId,
|
|
1336
|
+
screen: liveVoiceEndScreen(reason, failed ? this.failureCode : null),
|
|
1337
|
+
outcome: failed ? "failed" : "completed",
|
|
1338
|
+
});
|
|
1339
|
+
|
|
1269
1340
|
const shouldEmitSessionEndMetrics = this.state !== "failed";
|
|
1270
1341
|
this.state = "closed";
|
|
1271
1342
|
// Retire the island before the teardown below starts awaiting things. A
|
|
@@ -2864,8 +2935,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
2864
2935
|
): void {
|
|
2865
2936
|
turn.speculativePending = false;
|
|
2866
2937
|
// The dispatch is being unwound, so the barge-in merge note, the finished
|
|
2867
|
-
// continuation's answer,
|
|
2868
|
-
//
|
|
2938
|
+
// continuation's answer, its queued announcement, and any photos it claimed
|
|
2939
|
+
// all go back to the session. The turn that would have delivered them is
|
|
2940
|
+
// gone, and the utterance they belong to is about to be sent by another.
|
|
2869
2941
|
this.restorePendingTurnContext(turn);
|
|
2870
2942
|
// Latched before the handle check: when the discard beats the bridge
|
|
2871
2943
|
// handle's resolution (startVoiceTurn still persisting), the handle's
|
|
@@ -3807,8 +3879,20 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
3807
3879
|
voiceSessionId: this.context.sessionId,
|
|
3808
3880
|
userMessageChannel: "vellum",
|
|
3809
3881
|
assistantMessageChannel: "vellum",
|
|
3882
|
+
// Fixed, and NOT the originating client: this pair resolves the turn's
|
|
3883
|
+
// channel capabilities, where `macos` is what grants a live-voice turn
|
|
3884
|
+
// desktop UI and dynamic surfaces. Reporting the true client here would
|
|
3885
|
+
// strip `supportsDynamicUi` from every iOS session: a behavior change
|
|
3886
|
+
// wearing an attribution fix's clothes. The originating client travels
|
|
3887
|
+
// as telemetry instead, on `voiceTelemetry` below.
|
|
3810
3888
|
userMessageInterface: "macos",
|
|
3811
3889
|
assistantMessageInterface: "macos",
|
|
3890
|
+
voiceTelemetry: {
|
|
3891
|
+
sessionId: this.context.sessionId,
|
|
3892
|
+
...(this.context.startFrame.client
|
|
3893
|
+
? { client: this.context.startFrame.client }
|
|
3894
|
+
: {}),
|
|
3895
|
+
},
|
|
3812
3896
|
voiceControlPrompt: buildVoiceControlPrompt(activeTurn, {
|
|
3813
3897
|
...(leg.frontDoor !== undefined ? { frontDoor: leg.frontDoor } : {}),
|
|
3814
3898
|
}),
|
|
@@ -5506,6 +5590,13 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
|
|
|
5506
5590
|
// method, and a per-call-site hook would drift from it on the first frame
|
|
5507
5591
|
// anyone added. Fire-and-forget by contract — see the reporter.
|
|
5508
5592
|
this.liveActivityReporter.report(frame);
|
|
5593
|
+
// Latched here for the same reason: both fatal paths mark the session
|
|
5594
|
+
// `failed` and then send their error frame through this method, so the
|
|
5595
|
+
// code that ended the session is readable at close without either path
|
|
5596
|
+
// having to remember to stash it.
|
|
5597
|
+
if (frame.type === "error" && this.state === "failed") {
|
|
5598
|
+
this.failureCode ??= frame.code;
|
|
5599
|
+
}
|
|
5509
5600
|
let sent = false;
|
|
5510
5601
|
this.outboundFrames = this.outboundFrames
|
|
5511
5602
|
.catch(() => {})
|
|
@@ -5675,6 +5766,12 @@ export async function defaultSpawnBackgroundContinuation(args: {
|
|
|
5675
5766
|
objective: args.objective,
|
|
5676
5767
|
fork: true,
|
|
5677
5768
|
sendResultToUser: false,
|
|
5769
|
+
// Distinct spawn mode so this unattended continuation is separable in
|
|
5770
|
+
// cost telemetry from a tool-initiated fork. Mechanically both are
|
|
5771
|
+
// forks, but they come from different call sites with different cost
|
|
5772
|
+
// profiles, and lumping them together is what made delegated spend
|
|
5773
|
+
// opaque in the first place.
|
|
5774
|
+
spawnMode: "voice_continuation",
|
|
5678
5775
|
// Full subagent abilities: the continuation runs like any other
|
|
5679
5776
|
// background subagent, so it can genuinely finish build-shaped work
|
|
5680
5777
|
// (JARVIS-1354). Side effects are governed by the standard
|
|
@@ -5735,7 +5832,14 @@ async function defaultStartVoiceTurn(
|
|
|
5735
5832
|
// inside `startVoiceTurn` trips `FOREIGN KEY constraint failed`. Ensure it
|
|
5736
5833
|
// exists (idempotent) before persisting. Lives in the production wiring, not
|
|
5737
5834
|
// the session state machine, so session unit tests stay DB-free.
|
|
5738
|
-
|
|
5835
|
+
// Native: this is the local live-voice session, which adopts a conversation
|
|
5836
|
+
// id the app supplied. Its trust context resolves through
|
|
5837
|
+
// `resolveLocalLiveVoiceTrustContext`, and a phone call reaches the assistant
|
|
5838
|
+
// through the telephony path rather than here.
|
|
5839
|
+
const createdConversation = ensureConversationExists(
|
|
5840
|
+
options.conversationId,
|
|
5841
|
+
"vellum",
|
|
5842
|
+
);
|
|
5739
5843
|
if (createdConversation) {
|
|
5740
5844
|
// The row was created outside the normal send-message route, which is where
|
|
5741
5845
|
// sibling clients/sidebars learn about a new conversation. Emit the same
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { type ClientOs, parseClientOs } from "../channels/types.js";
|
|
2
|
+
|
|
1
3
|
const LIVE_VOICE_CLIENT_FRAME_TYPES = [
|
|
2
4
|
"start",
|
|
3
5
|
"audio",
|
|
@@ -5,6 +7,7 @@ const LIVE_VOICE_CLIENT_FRAME_TYPES = [
|
|
|
5
7
|
"interrupt",
|
|
6
8
|
"end",
|
|
7
9
|
"update_config",
|
|
10
|
+
"attach_image",
|
|
8
11
|
] as const;
|
|
9
12
|
|
|
10
13
|
type LiveVoiceClientFrameType = (typeof LIVE_VOICE_CLIENT_FRAME_TYPES)[number];
|
|
@@ -98,6 +101,21 @@ export interface LiveVoiceClientStartFrame {
|
|
|
98
101
|
* [{@link MIN_BARGE_IN_MIN_SPEECH_MS}, {@link MAX_BARGE_IN_MIN_SPEECH_MS}].
|
|
99
102
|
*/
|
|
100
103
|
readonly bargeInMinSpeechMs?: number;
|
|
104
|
+
/**
|
|
105
|
+
* Which client opened the session. Absent from clients that predate the
|
|
106
|
+
* field, in which case the originating client is simply unknown.
|
|
107
|
+
*
|
|
108
|
+
* A {@link ClientOs}, not an `InterfaceId`, and deliberately so: the iOS and
|
|
109
|
+
* macOS apps run the same web bundle over the same transport, so the thing
|
|
110
|
+
* that actually differs between them is the OS surface. `ClientOs` is also
|
|
111
|
+
* the vocabulary that is barred by contract from answering transport
|
|
112
|
+
* questions (see `channels/types.ts`), which is the invariant wanted here:
|
|
113
|
+
* this value is **analytics only**. It rides the voice turn's telemetry
|
|
114
|
+
* `client` bag (see `voice-session-bridge.ts`) and never reaches
|
|
115
|
+
* `userMessageInterface`, which feeds `resolveChannelCapabilities` and is
|
|
116
|
+
* load-bearing for what a voice turn may do.
|
|
117
|
+
*/
|
|
118
|
+
readonly client?: ClientOs;
|
|
101
119
|
}
|
|
102
120
|
|
|
103
121
|
/**
|
|
@@ -141,13 +159,35 @@ export interface LiveVoiceClientUpdateConfigFrame {
|
|
|
141
159
|
readonly bargeInMinSpeechMs?: number;
|
|
142
160
|
}
|
|
143
161
|
|
|
162
|
+
/**
|
|
163
|
+
* A photo the user took mid-call, already uploaded over the normal attachment
|
|
164
|
+
* route (`POST /v1/assistants/{id}/attachments`). Only the resulting id
|
|
165
|
+
* travels here.
|
|
166
|
+
*
|
|
167
|
+
* The bytes deliberately do not: the upload endpoint already normalizes HEIF,
|
|
168
|
+
* caps size, and stores the blob, and routing an image through this socket
|
|
169
|
+
* would duplicate all of it on a transport tuned for 50 ms audio frames.
|
|
170
|
+
*
|
|
171
|
+
* The id is *parked*, not dispatched. It rides the next turn's own user
|
|
172
|
+
* message (see {@link LiveVoiceSession}'s pending-attachment handling), so the
|
|
173
|
+
* photo and the words spoken about it are one message rather than two, which
|
|
174
|
+
* is what lets "what's this?" resolve, and what keeps the image attached to the
|
|
175
|
+
* newest user message, the only one a context-overflow retry preserves media on
|
|
176
|
+
* (`conversation-media-retry.ts`).
|
|
177
|
+
*/
|
|
178
|
+
export interface LiveVoiceClientAttachImageFrame {
|
|
179
|
+
readonly type: "attach_image";
|
|
180
|
+
readonly attachmentId: string;
|
|
181
|
+
}
|
|
182
|
+
|
|
144
183
|
export type LiveVoiceClientFrame =
|
|
145
184
|
| LiveVoiceClientStartFrame
|
|
146
185
|
| LiveVoiceClientAudioFrame
|
|
147
186
|
| LiveVoiceClientPttReleaseFrame
|
|
148
187
|
| LiveVoiceClientInterruptFrame
|
|
149
188
|
| LiveVoiceClientEndFrame
|
|
150
|
-
| LiveVoiceClientUpdateConfigFrame
|
|
189
|
+
| LiveVoiceClientUpdateConfigFrame
|
|
190
|
+
| LiveVoiceClientAttachImageFrame;
|
|
151
191
|
|
|
152
192
|
interface LiveVoiceBinaryAudioFrame {
|
|
153
193
|
readonly type: "binary_audio";
|
|
@@ -337,6 +377,19 @@ export interface LiveVoiceErrorServerFrame extends LiveVoiceServerFrameBase {
|
|
|
337
377
|
readonly type: "error";
|
|
338
378
|
readonly code: LiveVoiceProtocolErrorCode;
|
|
339
379
|
readonly message: string;
|
|
380
|
+
/**
|
|
381
|
+
* The client frame this error is about, when the failure was a parse or
|
|
382
|
+
* validation failure of a specific frame. Absent otherwise, and absent from
|
|
383
|
+
* daemons predating the field.
|
|
384
|
+
*
|
|
385
|
+
* It exists so an `unknown_type` is attributable. A client that sends more
|
|
386
|
+
* than one optional frame (today: `update_config` and `attach_image`) gets
|
|
387
|
+
* the same code for either, and without this has to assume which one was
|
|
388
|
+
* refused. The wrong assumption is silent in both directions: settings stop
|
|
389
|
+
* applying for a session, or a photo the user watched themselves take is
|
|
390
|
+
* dropped with nothing said.
|
|
391
|
+
*/
|
|
392
|
+
readonly frameType?: string;
|
|
340
393
|
/**
|
|
341
394
|
* True when the session continues past the error (e.g. a transient
|
|
342
395
|
* transcriber blip or one failed TTS segment). Absent (including on frames
|
|
@@ -469,9 +522,38 @@ export function validateLiveVoiceClientFrame(
|
|
|
469
522
|
return { ok: true, frame: { type: "end" } };
|
|
470
523
|
case "update_config":
|
|
471
524
|
return validateUpdateConfigFrame(value);
|
|
525
|
+
case "attach_image":
|
|
526
|
+
return validateAttachImageFrame(value);
|
|
472
527
|
}
|
|
473
528
|
}
|
|
474
529
|
|
|
530
|
+
function validateAttachImageFrame(
|
|
531
|
+
value: Record<string, unknown>,
|
|
532
|
+
): LiveVoiceParseResult<LiveVoiceClientAttachImageFrame> {
|
|
533
|
+
if (!("attachmentId" in value)) {
|
|
534
|
+
return protocolError(
|
|
535
|
+
"missing_required_field",
|
|
536
|
+
"attach_image frame is missing required field attachmentId",
|
|
537
|
+
"attachmentId",
|
|
538
|
+
"attach_image",
|
|
539
|
+
);
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
if (!isNonEmptyString(value.attachmentId)) {
|
|
543
|
+
return protocolError(
|
|
544
|
+
"invalid_field",
|
|
545
|
+
"attach_image frame field attachmentId must be a non-empty string",
|
|
546
|
+
"attachmentId",
|
|
547
|
+
"attach_image",
|
|
548
|
+
);
|
|
549
|
+
}
|
|
550
|
+
|
|
551
|
+
return {
|
|
552
|
+
ok: true,
|
|
553
|
+
frame: { type: "attach_image", attachmentId: value.attachmentId },
|
|
554
|
+
};
|
|
555
|
+
}
|
|
556
|
+
|
|
475
557
|
function validateUpdateConfigFrame(
|
|
476
558
|
value: Record<string, unknown>,
|
|
477
559
|
): LiveVoiceParseResult<LiveVoiceClientUpdateConfigFrame> {
|
|
@@ -642,6 +724,11 @@ function validateStartFrame(
|
|
|
642
724
|
);
|
|
643
725
|
}
|
|
644
726
|
|
|
727
|
+
// An unrecognized client is dropped rather than rejected: the field is an
|
|
728
|
+
// analytics dimension, and failing a session's startup over it would trade a
|
|
729
|
+
// gap in a chart for a user who cannot talk to their assistant.
|
|
730
|
+
const client = parseClientOs(value.client);
|
|
731
|
+
|
|
645
732
|
return {
|
|
646
733
|
ok: true,
|
|
647
734
|
frame: {
|
|
@@ -649,6 +736,7 @@ function validateStartFrame(
|
|
|
649
736
|
...(typeof value.conversationId === "string"
|
|
650
737
|
? { conversationId: value.conversationId }
|
|
651
738
|
: {}),
|
|
739
|
+
...(client ? { client } : {}),
|
|
652
740
|
audio: audioConfig.frame,
|
|
653
741
|
...(isLiveVoiceTurnDetectionMode(value.turnDetection)
|
|
654
742
|
? { turnDetection: value.turnDetection }
|
package/src/mcp/client.ts
CHANGED
|
@@ -3,6 +3,7 @@ import { Client } from "@modelcontextprotocol/sdk/client/index.js";
|
|
|
3
3
|
import { SSEClientTransport } from "@modelcontextprotocol/sdk/client/sse.js";
|
|
4
4
|
import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
|
|
5
5
|
import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
|
|
6
|
+
import type { ToolAnnotations } from "@modelcontextprotocol/sdk/types.js";
|
|
6
7
|
|
|
7
8
|
import { getIsPlatform } from "../config/env-registry.js";
|
|
8
9
|
import type { McpTransport } from "../config/schemas/mcp.js";
|
|
@@ -33,10 +34,18 @@ function normalizeInputSchema(
|
|
|
33
34
|
return { type: "object", properties: {} };
|
|
34
35
|
}
|
|
35
36
|
|
|
37
|
+
/**
|
|
38
|
+
* Behavioral hints a server may attach to a tool (MCP `ToolAnnotations`).
|
|
39
|
+
* Self-reported by the server, so consumers treat them as hints rather than
|
|
40
|
+
* guarantees.
|
|
41
|
+
*/
|
|
42
|
+
export type McpToolAnnotations = ToolAnnotations;
|
|
43
|
+
|
|
36
44
|
export interface McpToolInfo {
|
|
37
45
|
name: string;
|
|
38
46
|
description: string;
|
|
39
47
|
inputSchema: Record<string, unknown>;
|
|
48
|
+
annotations?: McpToolAnnotations;
|
|
40
49
|
}
|
|
41
50
|
|
|
42
51
|
export interface McpCallResult {
|
|
@@ -197,6 +206,7 @@ export class McpClient {
|
|
|
197
206
|
name: tool.name,
|
|
198
207
|
description: tool.description ?? "",
|
|
199
208
|
inputSchema: normalizeInputSchema(tool.inputSchema, tool.name),
|
|
209
|
+
annotations: tool.annotations,
|
|
200
210
|
}));
|
|
201
211
|
}
|
|
202
212
|
|
|
@@ -10,11 +10,14 @@ describe("channelForCallback", () => {
|
|
|
10
10
|
expect(channelForCallback("http://gw/deliver/telegram")).toBe("telegram");
|
|
11
11
|
expect(channelForCallback("http://gw/deliver/whatsapp")).toBe("whatsapp");
|
|
12
12
|
expect(channelForCallback("http://gw/deliver/a2a?taskId=t1")).toBe("a2a");
|
|
13
|
+
expect(channelForCallback("http://gw/deliver/discord?threadId=T1")).toBe(
|
|
14
|
+
"discord",
|
|
15
|
+
);
|
|
13
16
|
});
|
|
14
17
|
|
|
15
18
|
test("returns undefined for channels not delivered directly", () => {
|
|
16
|
-
expect(channelForCallback("http://gw/deliver/discord")).toBeUndefined();
|
|
17
19
|
expect(channelForCallback("http://gw/deliver/phone")).toBeUndefined();
|
|
20
|
+
expect(channelForCallback("http://gw/deliver/email")).toBeUndefined();
|
|
18
21
|
});
|
|
19
22
|
|
|
20
23
|
test("returns undefined for non-delivery paths", () => {
|
|
@@ -41,5 +44,6 @@ describe("channelForCallback", () => {
|
|
|
41
44
|
expect(channelForCallback("/deliver/slack")).toBe("slack");
|
|
42
45
|
expect(channelForCallback("/deliver/telegram")).toBe("telegram");
|
|
43
46
|
expect(channelForCallback("/deliver/whatsapp")).toBe("whatsapp");
|
|
47
|
+
expect(channelForCallback("/deliver/discord")).toBe("discord");
|
|
44
48
|
});
|
|
45
49
|
});
|