@vellumai/assistant 0.12.2 → 0.12.3-staging.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +2 -0
- package/AGENTS.md +1 -1
- package/ARCHITECTURE.md +43 -29
- package/Dockerfile +63 -0
- package/docs/architecture/integrations.md +7 -0
- package/docs/architecture/memory.md +17 -2
- package/docs/architecture/security.md +10 -5
- package/docs/architecture/turn-actor.md +2 -0
- package/docs/credential-execution-service.md +1 -1
- package/docs/desktop-browser-cli.md +50 -5
- package/docs/live-voice-task-outcomes.md +100 -0
- package/docs/mode-sessions.md +183 -0
- package/docs/voice-input-diagnostics.md +58 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/package.json +2 -1
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +1 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/secret-detection.ts +4 -1
- package/node_modules/@vellumai/environments/src/shell.test.ts +21 -0
- package/node_modules/@vellumai/environments/src/shell.ts +24 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/package.json +2 -1
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +1 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/secret-detection.ts +4 -1
- package/node_modules/@vellumai/gateway-client/src/gateway-ipc-contracts.ts +20 -0
- package/node_modules/@vellumai/gateway-client/src/inbound-contract.ts +8 -2
- package/node_modules/@vellumai/gateway-client/src/index.ts +4 -1
- package/node_modules/@vellumai/gateway-client/src/outbound-contract.ts +38 -7
- package/node_modules/@vellumai/gateway-client/src/trust-verdict-contract.ts +5 -6
- package/node_modules/@vellumai/service-contracts/package.json +2 -1
- package/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +1 -0
- package/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
- package/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
- package/node_modules/@vellumai/service-contracts/src/secret-detection.ts +4 -1
- package/node_modules/@vellumai/slack-text/src/index.ts +4 -8
- package/openapi.yaml +716 -16
- package/package.json +1 -1
- package/scripts/smoke-desktop-browser-cli.ts +11 -26
- package/scripts/smoke-desktop-window-theme.ts +2 -3
- package/scripts/sync-llm-catalog.ts +3 -0
- package/scripts/voice-ttft-spike.ts +2 -2
- package/src/__tests__/agent-loop-output-hooks.test.ts +150 -1
- package/src/__tests__/agent-loop-provider-error-recording.test.ts +87 -19
- package/src/__tests__/agent-loop.test.ts +259 -0
- package/src/__tests__/agent-wake-delegation-prompt.test.ts +64 -2
- package/src/__tests__/annotate-activity-metadata.test.ts +3 -1
- package/src/__tests__/annotate-risk-options.test.ts +2 -0
- package/src/__tests__/answered-question-persistence.test.ts +2 -0
- package/src/__tests__/anthropic-gateway-shared.test.ts +22 -0
- package/src/__tests__/anthropic-provider.test.ts +55 -0
- package/src/__tests__/approval-interception-trust-gates.test.ts +40 -0
- package/src/__tests__/approval-message-composer.test.ts +1 -118
- package/src/__tests__/approval-routes-http.test.ts +3 -0
- package/src/__tests__/assistant-event-hub-connection-history.test.ts +111 -0
- package/src/__tests__/assistant-event-hub-targeted.test.ts +50 -0
- package/src/__tests__/assistant-event-targeting.test.ts +186 -0
- package/src/__tests__/assistant-stream-state.test.ts +29 -9
- package/src/__tests__/attachments-store.test.ts +22 -3
- package/src/__tests__/browser-fill-credential.test.ts +11 -22
- package/src/__tests__/btw-routes.test.ts +4 -1
- package/src/__tests__/build-persisted-content.test.ts +1 -1
- package/src/__tests__/call-conversation-messages.test.ts +5 -0
- package/src/__tests__/call-recovery.test.ts +7 -2
- package/src/__tests__/call-routes-http.test.ts +12 -0
- package/src/__tests__/call-store.test.ts +7 -2
- package/src/__tests__/channel-approval.test.ts +9 -14
- package/src/__tests__/channel-reply-delivery.test.ts +56 -0
- package/src/__tests__/channel-retry-sweep.test.ts +9 -0
- package/src/__tests__/chat-credential-redaction.test.ts +23 -0
- package/src/__tests__/chat-reveal-guard-priming.test.ts +2 -0
- package/src/__tests__/checker.test.ts +11 -3
- package/src/__tests__/client-os-metadata-persistence.test.ts +35 -1
- package/src/__tests__/computer-use-screenshot-attachments.test.ts +411 -0
- package/src/__tests__/computer-use-screenshot-selection.test.ts +50 -0
- package/src/__tests__/computer-use-skill-manifest-regression.test.ts +0 -25
- package/src/__tests__/computer-use-tools.test.ts +1 -233
- package/src/__tests__/config-model-image-provider.test.ts +39 -0
- package/src/__tests__/config-schema.test.ts +16 -0
- package/src/__tests__/config-watcher.test.ts +24 -0
- package/src/__tests__/context-search-agent-runner.test.ts +260 -1
- package/src/__tests__/context-search-fanout.test.ts +1 -1
- package/src/__tests__/context-search-types.test.ts +7 -7
- package/src/__tests__/conversation-agent-loop-disk-pressure.test.ts +8 -2
- package/src/__tests__/conversation-agent-loop-fatal-cleanup.test.ts +2 -0
- package/src/__tests__/conversation-agent-loop-inference-profile.test.ts +2 -0
- package/src/__tests__/conversation-agent-loop-overflow.test.ts +2 -0
- package/src/__tests__/conversation-agent-loop.test.ts +797 -16
- package/src/__tests__/conversation-app-control-lifecycle.test.ts +53 -1
- package/src/__tests__/conversation-attachments.test.ts +106 -0
- package/src/__tests__/conversation-evictor.test.ts +62 -37
- package/src/__tests__/conversation-fork-crud.test.ts +191 -0
- package/src/__tests__/conversation-history-boundary-repair-failure.test.ts +174 -0
- package/src/__tests__/conversation-loop-db-lock-resilience.test.ts +2 -0
- package/src/__tests__/conversation-process-app-control-preactivation.test.ts +2 -0
- package/src/__tests__/conversation-rendered-delegation-state.test.ts +276 -0
- package/src/__tests__/conversation-retry-route.test.ts +22 -0
- package/src/__tests__/conversation-routes-disk-view.test.ts +2 -0
- package/src/__tests__/conversation-routes-enabled-plugins.test.ts +2 -0
- package/src/__tests__/conversation-routes-guardian-reply.test.ts +10 -0
- package/src/__tests__/conversation-routes-hidden-queue.test.ts +2 -0
- package/src/__tests__/conversation-routes-slash-commands.test.ts +2 -0
- package/src/__tests__/conversation-runtime-assembly.test.ts +29 -0
- package/src/__tests__/conversation-surface-snapshots.test.ts +543 -0
- package/src/__tests__/conversation-surfaces-action-delivery.test.ts +151 -0
- package/src/__tests__/conversation-surfaces-point-at-budget.test.ts +63 -1
- package/src/__tests__/conversation-tool-setup-attribution.test.ts +23 -0
- package/src/__tests__/conversation-undo.test.ts +61 -0
- package/src/__tests__/credential-execution-client.test.ts +72 -55
- package/src/__tests__/credential-prompt-route.test.ts +35 -3
- package/src/__tests__/cu-unified-flow.test.ts +82 -1
- package/src/__tests__/db-conversation-tool-surface.test.ts +221 -0
- package/src/__tests__/db-mode-sessions-migration.test.ts +175 -0
- package/src/__tests__/deterministic-verification-control-plane.test.ts +2 -0
- package/src/__tests__/dm-backfill.test.ts +91 -1
- package/src/__tests__/document-create-route.test.ts +88 -0
- package/src/__tests__/document-sync-tags.test.ts +27 -0
- package/src/__tests__/emit-signal-routing-intent.test.ts +142 -2
- package/src/__tests__/events-tail-route.test.ts +58 -0
- package/src/__tests__/evict-conversations-for-reload.test.ts +57 -8
- package/src/__tests__/guardian-action-sweep.test.ts +16 -0
- package/src/__tests__/guardian-dispatch.test.ts +13 -0
- package/src/__tests__/headless-browser-interactions.test.ts +64 -20
- package/src/__tests__/heartbeat-service.test.ts +2 -59
- package/src/__tests__/helpers/mock-conversation.ts +21 -0
- package/src/__tests__/helpers/native-web-search-harness.ts +2 -0
- package/src/__tests__/helpers/verification-sessions-ipc-sim.ts +5 -0
- package/src/__tests__/history-repair.test.ts +53 -0
- package/src/__tests__/host-cu-proxy.test.ts +46 -2
- package/src/__tests__/http-user-message-parity.test.ts +2 -0
- package/src/__tests__/image-credentials.test.ts +49 -0
- package/src/__tests__/image-service-dispatcher.test.ts +45 -0
- package/src/__tests__/inference-profile-session-handler.test.ts +26 -0
- package/src/__tests__/interrupt-on-send.test.ts +57 -43
- package/src/__tests__/interrupt-turn-note.test.ts +5 -6
- package/src/__tests__/list-messages-tool-merge.test.ts +85 -1
- package/src/__tests__/llm-catalog-parity.test.ts +27 -4
- package/src/__tests__/media-generate-image.test.ts +49 -1
- package/src/__tests__/message-provenance-contact-id.test.ts +69 -0
- package/src/__tests__/migration-export-to-gcs.test.ts +142 -0
- package/src/__tests__/mode-session-history.test.ts +245 -0
- package/src/__tests__/mode-session-recovery.test.ts +222 -0
- package/src/__tests__/mode-session-wire.test.ts +141 -0
- package/src/__tests__/notification-guardian-path.test.ts +7 -0
- package/src/__tests__/notification-telegram-adapter.test.ts +2 -2
- package/src/__tests__/notification-vellum-adapter.test.ts +34 -0
- package/src/__tests__/oauth-apps-routes.test.ts +1 -0
- package/src/__tests__/oauth-commands-routes.test.ts +114 -101
- package/src/__tests__/oauth-connect-orchestrator.test.ts +2 -0
- package/src/__tests__/oauth-provider-serializer.test.ts +13 -0
- package/src/__tests__/oauth-providers-routes.test.ts +3 -0
- package/src/__tests__/outbound-slack-persistence.test.ts +2 -0
- package/src/__tests__/persist-media-references.test.ts +50 -0
- package/src/__tests__/persistence-secret-redaction.test.ts +2 -0
- package/src/__tests__/plugin-api-provider.test.ts +5 -0
- package/src/__tests__/plugin-api-resolve-credential.test.ts +97 -0
- package/src/__tests__/plugin-import-boundary-guard.test.ts +9 -1
- package/src/__tests__/process-message-display-content.test.ts +56 -0
- package/src/__tests__/reaction-persistence.test.ts +5 -0
- package/src/__tests__/request-secret-standalone-channel-gate.test.ts +25 -9
- package/src/__tests__/run-conversation-turn-persistence.test.ts +138 -1
- package/src/__tests__/runtime-events-sse-reconnect.test.ts +1 -1
- package/src/__tests__/schedule-routes.test.ts +20 -0
- package/src/__tests__/scheduler-result-notification.test.ts +23 -4
- package/src/__tests__/script-proxy-certs.test.ts +1 -1
- package/src/__tests__/scripted-turn-metadata-persistence.test.ts +73 -0
- package/src/__tests__/secret-ingress-http.test.ts +2 -0
- package/src/__tests__/secret-routes-platform-proxy.test.ts +86 -31
- package/src/__tests__/secret-routes-scrub.test.ts +22 -27
- package/src/__tests__/secret-scanner.test.ts +20 -0
- package/src/__tests__/secure-keys-managed-failover.test.ts +8 -0
- package/src/__tests__/secure-keys.test.ts +50 -4
- package/src/__tests__/send-endpoint-busy.test.ts +163 -1
- package/src/__tests__/server-history-render.test.ts +91 -1
- package/src/__tests__/server-tool-pairing.test.ts +107 -0
- package/src/__tests__/settings-routes.test.ts +50 -1
- package/src/__tests__/skill-script-runner-sandbox.test.ts +32 -0
- package/src/__tests__/skills.test.ts +5 -4
- package/src/__tests__/subagent-fork-notifications.test.ts +1 -2
- package/src/__tests__/subagent-manager-notify.test.ts +1 -2
- package/src/__tests__/subagent-notify-parent.test.ts +146 -39
- package/src/__tests__/subagent-notify-stale-rebuild.test.ts +2 -4
- package/src/__tests__/subagent-tool-gate-mode.test.ts +226 -0
- package/src/__tests__/surface-completion-in-flight-snapshot.test.ts +2 -0
- package/src/__tests__/terminal-tools.test.ts +8 -0
- package/src/__tests__/thread-backfill.test.ts +53 -0
- package/src/__tests__/tool-preview-lifecycle.test.ts +2 -0
- package/src/__tests__/tool-result-follow-up.test.ts +99 -0
- package/src/__tests__/tool-result-metadata-plumbing.test.ts +66 -1
- package/src/__tests__/tool-start-timestamp.test.ts +2 -0
- package/src/__tests__/twilio-routes.test.ts +1 -0
- package/src/__tests__/unicode.test.ts +36 -0
- package/src/__tests__/web-search-backend-failure.test.ts +1 -1
- package/src/activation/progress-store.test.ts +12 -7
- package/src/agent/loop.ts +136 -53
- package/src/agent/tool-result-follow-up.ts +70 -0
- package/src/api/attachment-provenance.test.ts +68 -0
- package/src/api/computer-use-tool.test.ts +55 -0
- package/src/api/computer-use-tool.ts +27 -0
- package/src/api/events/assistant-outbound-attachment.ts +3 -0
- package/src/api/events/assistant-turn-start.ts +3 -0
- package/src/api/events/desktop-activity-changed.ts +10 -0
- package/src/api/events/generation-handoff.ts +2 -0
- package/src/api/events/message-complete.ts +2 -0
- package/src/api/events/notification-conversation-created.ts +14 -16
- package/src/api/events/notification-intent.ts +4 -2
- package/src/api/events/question-request.ts +1 -0
- package/src/api/events/tool-result.ts +85 -0
- package/src/api/events/tool-use-start.ts +3 -0
- package/src/api/events/user-message-echo.ts +3 -0
- package/src/api/index.ts +42 -6
- package/src/api/mode-session.test.ts +131 -0
- package/src/api/mode-session.ts +148 -0
- package/src/api/remember-tool.ts +15 -0
- package/src/api/responses/conversation-message.ts +9 -0
- package/src/approvals/AGENTS.md +4 -2
- package/src/approvals/approval-primitive.ts +5 -2
- package/src/approvals/scoped-approval-grants.ts +6 -2
- package/src/browser/__tests__/operations.test.ts +29 -0
- package/src/browser/operation-meta.ts +2 -1
- package/src/browser/operations.ts +29 -0
- package/src/browser/virtual-desktop-target.ts +2 -9
- package/src/bundler/bundle-signer.ts +1 -3
- package/src/calls/__tests__/barge-in-guard.test.ts +3 -0
- package/src/calls/__tests__/call-controller.test.ts +94 -0
- package/src/calls/__tests__/call-funnel-telemetry.test.ts +204 -0
- package/src/calls/__tests__/voice-control-protocol.test.ts +15 -0
- package/src/calls/__tests__/voice-escalation-judge.test.ts +209 -0
- package/src/calls/__tests__/voice-leg-coordinator.test.ts +168 -0
- package/src/{live-voice/__tests__/live-voice-metrics.test.ts → calls/__tests__/voice-metrics.test.ts} +32 -38
- package/src/calls/__tests__/voice-session-bridge.test.ts +576 -122
- package/src/calls/barge-in-guard.ts +5 -0
- package/src/calls/call-controller.ts +196 -1
- package/src/calls/call-domain.ts +4 -0
- package/src/calls/call-store.ts +87 -2
- package/src/calls/guardian-action-sweep.ts +22 -8
- package/src/calls/media-stream-audio-transcode.ts +1 -3
- package/src/calls/typesafe-noul.ts +156 -0
- package/src/calls/voice-control-protocol.ts +6 -0
- package/src/calls/voice-escalation-judge.ts +145 -0
- package/src/calls/voice-escalation-target.ts +10 -0
- package/src/calls/voice-leg-coordinator.ts +121 -3
- package/src/{live-voice/live-voice-metrics.ts → calls/voice-metrics.ts} +94 -87
- package/src/calls/voice-session-bridge.ts +256 -112
- package/src/calls/voice-triage-escalate.ts +7 -0
- package/src/cli/__tests__/roadmap-cli.test.ts +4 -41
- package/src/cli/commands/__tests__/cli-test-harness.ts +21 -3
- package/src/cli/commands/__tests__/notifications.test.ts +35 -10
- package/src/cli/commands/__tests__/plugins.test.ts +20 -1
- package/src/cli/commands/__tests__/schedules.test.ts +14 -0
- package/src/cli/commands/bash.help.ts +4 -3
- package/src/cli/commands/browser.help.ts +9 -6
- package/src/cli/commands/browser.ts +0 -1
- package/src/cli/commands/clients.help.ts +46 -0
- package/src/cli/commands/clients.ts +116 -0
- package/src/cli/commands/credentials.help.ts +9 -4
- package/src/cli/commands/notifications.help.ts +16 -2
- package/src/cli/commands/notifications.ts +30 -8
- package/src/cli/commands/oauth/index.help.ts +10 -0
- package/src/cli/commands/oauth/providers.ts +12 -0
- package/src/cli/commands/oauth/request.test.ts +179 -2
- package/src/cli/commands/oauth/request.ts +41 -10
- package/src/cli/commands/plugins.ts +12 -5
- package/src/cli/commands/roadmap.help.ts +18 -21
- package/src/cli/commands/roadmap.ts +4 -21
- package/src/cli/commands/schedules.ts +2 -0
- package/src/cli/lib/__tests__/inspect-plugin.test.ts +54 -0
- package/src/cli/lib/__tests__/install-from-github.test.ts +41 -0
- package/src/cli/lib/__tests__/list-installed-plugins.test.ts +5 -2
- package/src/cli/lib/__tests__/local-plugin-upgrade.test.ts +133 -51
- package/src/cli/lib/__tests__/plugin-catalog-cache.test.ts +8 -2
- package/src/cli/lib/bundled-marketplace.json +906 -1
- package/src/cli/lib/bundled-plugin-packages.json +783 -1
- package/src/cli/lib/inspect-plugin.ts +11 -4
- package/src/cli/lib/plugin-marketplace.ts +8 -0
- package/src/cli/lib/publish-plugin.ts +1 -2
- package/src/cli/lib/upgrade-plugin.ts +19 -11
- package/src/cli/output.ts +18 -4
- package/src/config/__tests__/default-provider.test.ts +6 -0
- package/src/config/__tests__/memory-retrospective-schema.test.ts +14 -1
- package/src/config/bundled-skills/computer-use/SKILL.md +1 -1
- package/src/config/bundled-skills/computer-use/TOOLS.json +89 -1
- package/src/config/bundled-skills/image-studio/SKILL.md +17 -2
- package/src/config/bundled-skills/image-studio/tools/media-generate-image.ts +7 -11
- package/src/config/bundled-skills/schedule/SKILL.md +6 -6
- package/src/config/bundled-skills/sequences/TOOLS.json +1 -5
- package/src/config/call-site-defaults.ts +9 -0
- package/src/config/feature-flag-registry.json +16 -0
- package/src/config/loader.ts +9 -17
- package/src/config/platform-identity.ts +8 -8
- package/src/config/profile-text-generation.test.ts +51 -0
- package/src/config/profile-text-generation.ts +51 -0
- package/src/config/schemas/__tests__/memory-v3.test.ts +12 -0
- package/src/config/schemas/__tests__/tools.test.ts +31 -0
- package/src/config/schemas/call-site-catalog.ts +14 -0
- package/src/config/schemas/channels.ts +0 -18
- package/src/config/schemas/llm.ts +9 -2
- package/src/config/schemas/mcp.ts +6 -1
- package/src/config/schemas/memory-retrospective.ts +11 -2
- package/src/config/schemas/memory-v3.ts +12 -0
- package/src/config/schemas/services.ts +15 -6
- package/src/config/schemas/tools.ts +10 -0
- package/src/config/session-groups-gate.ts +21 -0
- package/src/config/skills.ts +2 -6
- package/src/conversations/__tests__/message-consolidation.test.ts +149 -0
- package/src/conversations/message-consolidation.ts +102 -5
- package/src/credential-execution/ces-connect.test.ts +69 -0
- package/src/credential-execution/ces-connect.ts +115 -0
- package/src/credential-execution/ces-runtime.ts +55 -93
- package/src/credential-execution/executable-discovery.ts +34 -142
- package/src/credential-execution/process-manager.test.ts +20 -24
- package/src/credential-execution/process-manager.ts +24 -19
- package/src/daemon/__tests__/conversation-surfaces-launch.test.ts +10 -1
- package/src/daemon/__tests__/conversation-tool-setup.test.ts +43 -0
- package/src/daemon/__tests__/inference-profile-notification.test.ts +2 -0
- package/src/daemon/__tests__/turn-tail-deleted-conversation.test.ts +36 -0
- package/src/daemon/assistant-attachments.ts +20 -12
- package/src/daemon/browser-mode-session.test.ts +402 -0
- package/src/daemon/browser-mode-session.ts +186 -0
- package/src/daemon/chat-credential-redaction.ts +16 -1
- package/src/daemon/computer-use-mode-session.test.ts +210 -0
- package/src/daemon/computer-use-mode-session.ts +105 -0
- package/src/daemon/config-watcher.ts +10 -0
- package/src/daemon/conversation-agent-loop-handlers.ts +132 -13
- package/src/daemon/conversation-agent-loop.ts +231 -25
- package/src/daemon/conversation-attachments.ts +93 -12
- package/src/daemon/conversation-evictor.ts +5 -24
- package/src/daemon/conversation-history.ts +10 -0
- package/src/daemon/conversation-interrupt-repair.ts +4 -43
- package/src/daemon/conversation-interrupt.ts +7 -11
- package/src/daemon/conversation-messaging.ts +58 -5
- package/src/daemon/conversation-mode-session.test.ts +1667 -0
- package/src/daemon/conversation-mode-session.ts +1008 -0
- package/src/daemon/conversation-process.ts +5 -0
- package/src/daemon/conversation-runtime-assembly.ts +10 -20
- package/src/daemon/conversation-skill-tools.ts +1 -0
- package/src/daemon/conversation-store.ts +4 -4
- package/src/daemon/conversation-surface-snapshots.ts +135 -0
- package/src/daemon/conversation-surfaces.ts +113 -5
- package/src/daemon/conversation-tool-setup.ts +79 -6
- package/src/daemon/conversation-turn-finalize.ts +26 -13
- package/src/daemon/conversation.ts +215 -58
- package/src/daemon/daemon-control.ts +3 -7
- package/src/daemon/handlers/config-model.ts +7 -5
- package/src/daemon/handlers/shared.ts +55 -16
- package/src/daemon/host-cu-proxy.ts +51 -22
- package/src/daemon/lifecycle.ts +30 -13
- package/src/daemon/mcp-reload-service.ts +1 -1
- package/src/daemon/message-protocol.ts +0 -1
- package/src/daemon/message-provenance.ts +28 -4
- package/src/daemon/message-types/shared.ts +1 -0
- package/src/daemon/mode-session-startup-recovery.ts +23 -0
- package/src/daemon/mode-session-tracking.ts +54 -0
- package/src/daemon/orphan-reaper.ts +4 -3
- package/src/daemon/persist-media-references.ts +20 -3
- package/src/daemon/process-message.ts +13 -4
- package/src/daemon/tool-setup-types.ts +6 -0
- package/src/daemon/virtual-desktop-context.ts +21 -0
- package/src/daemon/wake-conversation-ops.ts +50 -15
- package/src/desktop/__tests__/fake-desktop.ts +1 -0
- package/src/desktop/desktop-accessibility-script.ts +88 -0
- package/src/desktop/desktop-accessibility.test.ts +160 -0
- package/src/desktop/desktop-accessibility.ts +169 -0
- package/src/desktop/desktop-automation-lease.test.ts +302 -64
- package/src/desktop/desktop-automation-lease.ts +129 -54
- package/src/desktop/desktop-browser-endpoint.ts +4 -0
- package/src/desktop/desktop-computer-use-routing.test.ts +330 -0
- package/src/desktop/desktop-computer-use.test.ts +449 -0
- package/src/desktop/desktop-computer-use.ts +502 -0
- package/src/desktop/desktop-dependencies.ts +47 -293
- package/src/desktop/desktop-display.ts +2 -2
- package/src/desktop/desktop-help.ts +15 -0
- package/src/desktop/desktop-panel-config.test.ts +13 -3
- package/src/desktop/desktop-panel-config.ts +109 -15
- package/src/desktop/desktop-session-manager.test.ts +69 -21
- package/src/desktop/desktop-session-manager.ts +76 -22
- package/src/desktop/desktop-stream-bridge.test.ts +76 -132
- package/src/desktop/desktop-stream-bridge.ts +9 -11
- package/src/desktop/desktop-wallpaper-renderer.ts +132 -0
- package/src/desktop/desktop-wallpaper-worker.ts +17 -0
- package/src/desktop/desktop-wallpaper.test.ts +48 -7
- package/src/desktop/desktop-wallpaper.ts +34 -102
- package/src/desktop/desktop-window-theme.ts +5 -25
- package/src/desktop/virtual-desktop-feature.ts +53 -3
- package/src/desktop/virtual-desktop-platform.test.ts +5 -9
- package/src/documents/document-store.ts +23 -0
- package/src/heartbeat/__tests__/heartbeat-service.test.ts +22 -26
- package/src/heartbeat/heartbeat-service.ts +5 -33
- package/src/ipc/__tests__/browser-ipc.test.ts +215 -6
- package/src/ipc/assistant-server.ts +2 -0
- package/src/ipc/routes/__tests__/channel-reply-ipc-routes.test.ts +111 -0
- package/src/ipc/routes/channel-reply-ipc-routes.ts +73 -0
- package/src/live-voice/__tests__/continuation-judge.test.ts +151 -0
- package/src/live-voice/__tests__/live-activity-reporter.test.ts +154 -15
- package/src/live-voice/__tests__/live-voice-flux-turn-end.test.ts +743 -31
- package/src/live-voice/__tests__/live-voice-look-follow-up.test.ts +191 -10
- package/src/live-voice/__tests__/live-voice-session-groups-gate.test.ts +87 -0
- package/src/live-voice/__tests__/live-voice-session-manager.test.ts +5 -1
- package/src/live-voice/__tests__/live-voice-sight-frame-echo.test.ts +41 -0
- package/src/live-voice/__tests__/live-voice-sight-frame.test.ts +1017 -9
- package/src/live-voice/__tests__/live-voice-triage-escalate.test.ts +224 -1
- package/src/live-voice/__tests__/live-voice-tts-session.test.ts +101 -0
- package/src/live-voice/__tests__/live-voice-tts.test.ts +44 -0
- package/src/live-voice/__tests__/live-voice-vad.test.ts +939 -75
- package/src/live-voice/__tests__/microphone-idle-gate.test.ts +87 -0
- package/src/live-voice/__tests__/session-controls.test.ts +23 -0
- package/src/live-voice/__tests__/task-outcomes.test.ts +117 -0
- package/src/live-voice/__tests__/voice-input-diagnostics.test.ts +117 -0
- package/src/live-voice/camera-mode-session.test.ts +265 -0
- package/src/live-voice/camera-mode-session.ts +277 -0
- package/src/live-voice/continuation-judge.ts +123 -0
- package/src/live-voice/live-activity-reporter.ts +115 -6
- package/src/live-voice/live-voice-manager.ts +36 -1
- package/src/live-voice/live-voice-photo.ts +88 -7
- package/src/live-voice/live-voice-session-manager.ts +15 -0
- package/src/live-voice/live-voice-session.ts +1443 -747
- package/src/live-voice/live-voice-tts.ts +22 -4
- package/src/live-voice/microphone-idle-gate.ts +56 -0
- package/src/live-voice/protocol.ts +171 -4
- package/src/live-voice/session-controls.ts +11 -3
- package/src/live-voice/task-outcomes.ts +131 -0
- package/src/live-voice/voice-input-diagnostics.ts +117 -0
- package/src/mcp/__tests__/manager-tool-caps.test.ts +156 -0
- package/src/mcp/__tests__/startup.test.ts +30 -7
- package/src/mcp/__tests__/tool-caps.test.ts +134 -0
- package/src/mcp/manager.ts +171 -101
- package/src/mcp/startup.ts +33 -13
- package/src/mcp/tool-caps.ts +184 -0
- package/src/media/__tests__/image-models.test.ts +58 -0
- package/src/media/image-credentials.ts +5 -0
- package/src/media/image-models.ts +52 -0
- package/src/media/image-service.ts +8 -0
- package/src/media/openrouter-image-service.test.ts +133 -0
- package/src/media/openrouter-image-service.ts +153 -0
- package/src/media/types.ts +1 -1
- package/src/messaging/provider-message-metadata.ts +3 -3
- package/src/messaging/providers/__tests__/transport-dispatch.test.ts +65 -11
- package/src/messaging/providers/discord/send.ts +4 -17
- package/src/messaging/providers/discord/transport.ts +2 -2
- package/src/messaging/providers/outlook/client.ts +16 -22
- package/src/messaging/providers/telegram-bot/send.test.ts +236 -1
- package/src/messaging/providers/telegram-bot/send.ts +10 -7
- package/src/messaging/providers/telegram-bot/transport.ts +3 -10
- package/src/messaging/providers/whatsapp/send.test.ts +169 -4
- package/src/monitoring/__tests__/control.test.ts +17 -5
- package/src/monitoring/control.ts +15 -5
- package/src/notifications/AGENTS.md +1 -1
- package/src/notifications/README.md +83 -99
- package/src/notifications/__tests__/broadcaster.test.ts +1 -1
- package/src/notifications/__tests__/channel-allowlist.test.ts +39 -0
- package/src/notifications/__tests__/copy-composer.test.ts +71 -1
- package/src/notifications/__tests__/decision-engine.test.ts +79 -2
- package/src/notifications/__tests__/deterministic-checks.test.ts +10 -10
- package/src/notifications/__tests__/home-feed-side-effect.test.ts +93 -4
- package/src/notifications/__tests__/notification-utils.test.ts +18 -0
- package/src/notifications/adapters/macos.ts +37 -46
- package/src/notifications/broadcaster.ts +5 -6
- package/src/notifications/channel-allowlist.ts +44 -0
- package/src/notifications/copy-composer.ts +11 -26
- package/src/notifications/decision-engine.ts +24 -14
- package/src/notifications/deterministic-checks.ts +1 -1
- package/src/notifications/emit-signal.ts +123 -37
- package/src/notifications/home-feed-side-effect.ts +9 -7
- package/src/notifications/notification-utils.ts +19 -0
- package/src/notifications/schedule-result-producer.ts +4 -6
- package/src/notifications/signal.ts +4 -19
- package/src/notifications/types.ts +15 -0
- package/src/oauth/AGENTS.md +2 -0
- package/src/oauth/__tests__/identity-verifier.test.ts +36 -1
- package/src/oauth/connection-resolver.ts +1 -2
- package/src/oauth/identity-verifier.ts +24 -0
- package/src/oauth/oauth-store.ts +12 -4
- package/src/oauth/provider-serializer.ts +8 -0
- package/src/oauth/seed-providers.ts +42 -0
- package/src/onboarding/onboarding-events-store.ts +99 -12
- package/src/permissions/confirmation-guardian-request.test.ts +16 -0
- package/src/permissions/confirmation-guardian-request.ts +2 -3
- package/src/permissions/question-prompter.test.ts +31 -0
- package/src/permissions/question-prompter.ts +2 -0
- package/src/persistence/__tests__/client-connection-events-store.test.ts +322 -0
- package/src/persistence/attachments-store.ts +84 -44
- package/src/persistence/client-connection-events-store.ts +377 -0
- package/src/persistence/conversation-crud.ts +166 -26
- package/src/persistence/conversation-mode-sessions.test.ts +424 -0
- package/src/persistence/conversation-mode-sessions.ts +460 -0
- package/src/persistence/conversation-plugin-facade.ts +30 -0
- package/src/persistence/conversation-queries.ts +6 -19
- package/src/persistence/conversation-tool-surface.ts +135 -0
- package/src/persistence/conversation-types.test.ts +32 -0
- package/src/persistence/conversation-types.ts +26 -4
- package/src/persistence/llm-usage-store.ts +3 -9
- package/src/persistence/message-metadata.ts +25 -0
- package/src/persistence/migrations/378-create-conversation-tool-surfaces.test.ts +78 -0
- package/src/persistence/migrations/378-create-conversation-tool-surfaces.ts +29 -0
- package/src/persistence/migrations/379-oauth-providers-response-ok-field.test.ts +92 -0
- package/src/persistence/migrations/379-oauth-providers-response-ok-field.ts +32 -0
- package/src/persistence/migrations/380-conversation-tool-surfaces-delegate-independent-tasks.test.ts +90 -0
- package/src/persistence/migrations/380-conversation-tool-surfaces-delegate-independent-tasks.ts +34 -0
- package/src/persistence/migrations/381-create-conversation-mode-sessions.ts +41 -0
- package/src/persistence/migrations/382-create-client-connection-events.ts +40 -0
- package/src/persistence/migrations/__tests__/382-create-client-connection-events.test.ts +56 -0
- package/src/persistence/schema/conversation-tool-surfaces.ts +32 -0
- package/src/persistence/schema/conversations.ts +67 -0
- package/src/persistence/schema/index.ts +1 -0
- package/src/persistence/schema/infrastructure.ts +29 -0
- package/src/persistence/schema/oauth.ts +1 -0
- package/src/persistence/steps.ts +41 -3
- package/src/plugin-api/__tests__/plugin-name-env.test.ts +249 -0
- package/src/plugin-api/conversation-turn.ts +31 -7
- package/src/plugin-api/effective-context-window.ts +33 -0
- package/src/plugin-api/index.ts +35 -6
- package/src/plugin-api/plugin-channel-turn-trust.test.ts +133 -0
- package/src/plugin-api/plugin-channel-turn-trust.ts +71 -0
- package/src/plugin-api/plugin-name-env.ts +125 -0
- package/src/plugin-api/resolve-credential.ts +79 -20
- package/src/plugin-api/vision-support.test.ts +22 -0
- package/src/plugin-api/vision-support.ts +17 -5
- package/src/plugins/__tests__/active-plugin-env.test.ts +136 -0
- package/src/plugins/active-plugin-env.ts +51 -0
- package/src/plugins/defaults/index.ts +24 -0
- package/src/plugins/defaults/memory/AGENTS.md +39 -5
- package/src/plugins/defaults/memory/__tests__/archive-removal-framing.test.ts +64 -0
- package/src/plugins/defaults/memory/__tests__/buffer-file.test.ts +365 -0
- package/src/plugins/defaults/memory/__tests__/buffer-format.test.ts +43 -0
- package/src/plugins/defaults/memory/__tests__/conversation-memory-purge.test.ts +1 -0
- package/src/plugins/defaults/memory/__tests__/db-memory-attach.test.ts +2 -0
- package/src/plugins/defaults/memory/__tests__/fixtures/buffer-appender.ts +17 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-job.test.ts +74 -0
- package/src/plugins/defaults/memory/__tests__/memory-run-evidence.test.ts +161 -0
- package/src/plugins/defaults/memory/__tests__/relocated-memory-test-rows.ts +10 -0
- package/src/plugins/defaults/memory/__tests__/worker-control.test.ts +19 -6
- package/src/plugins/defaults/memory/buffer-file.ts +354 -0
- package/src/plugins/defaults/memory/buffer-format.ts +40 -0
- package/src/plugins/defaults/memory/context-search/agent-protocol.ts +9 -12
- package/src/plugins/defaults/memory/context-search/agent-runner.ts +77 -22
- package/src/plugins/defaults/memory/context-search/format.ts +53 -8
- package/src/plugins/defaults/memory/context-search/limits.ts +10 -12
- package/src/plugins/defaults/memory/context-search/recall-input.ts +38 -0
- package/src/plugins/defaults/memory/context-search/search.ts +8 -14
- package/src/plugins/defaults/memory/context-search/sources/conversations.ts +1 -0
- package/src/plugins/defaults/memory/context-search/sources/memory-v2.ts +2 -1
- package/src/plugins/defaults/memory/context-search/sources/workspace.ts +2 -1
- package/src/plugins/defaults/memory/context-search/types.ts +1 -11
- package/src/plugins/defaults/memory/conversation-memory-purge.ts +4 -0
- package/src/plugins/defaults/memory/graph/__tests__/handle-remember-v2.test.ts +11 -4
- package/src/plugins/defaults/memory/graph/capability-seed.ts +1 -2
- package/src/plugins/defaults/memory/graph/tool-handlers.ts +13 -63
- package/src/plugins/defaults/memory/graph/tools.ts +35 -64
- package/src/plugins/defaults/memory/host-utils.ts +0 -10
- package/src/plugins/defaults/memory/injectors.ts +4 -3
- package/src/plugins/defaults/memory/memory-retrospective-job.ts +72 -183
- package/src/plugins/defaults/memory/memory-run-evidence.ts +213 -0
- package/src/plugins/defaults/memory/src/memory-eval-routes.ts +2 -3
- package/src/plugins/defaults/memory/src/memory-item-routes.test.ts +6 -1
- package/src/plugins/defaults/memory/src/memory-item-routes.ts +2 -1
- package/src/plugins/defaults/memory/substrate/__tests__/consolidation-job.test.ts +407 -99
- package/src/plugins/defaults/memory/substrate/__tests__/consolidation-prompt-flag-gating-guard.test.ts +10 -0
- package/src/plugins/defaults/memory/substrate/__tests__/prompts-consolidation.test.ts +107 -7
- package/src/plugins/defaults/memory/substrate/consolidation-job.ts +307 -86
- package/src/plugins/defaults/memory/substrate/consolidation-tool-surface.ts +34 -0
- package/src/plugins/defaults/memory/substrate/page-index.ts +2 -1
- package/src/plugins/defaults/memory/substrate/prompts/consolidation.ts +91 -51
- package/src/plugins/defaults/memory/substrate/sweep-job.ts +1 -1
- package/src/plugins/defaults/memory/tools.test.ts +124 -9
- package/src/plugins/defaults/memory/tools.ts +38 -11
- package/src/plugins/defaults/memory/v1/graph/consolidation.ts +2 -2
- package/src/plugins/defaults/memory/v1/graph/extraction.ts +2 -1
- package/src/plugins/defaults/memory/v1/graph/retriever.ts +1 -1
- package/src/plugins/defaults/memory/v2/__tests__/migration.test.ts +5 -0
- package/src/plugins/defaults/memory/v2/__tests__/reranker.test.ts +5 -2
- package/src/plugins/defaults/memory/v2/reranker.ts +2 -1
- package/src/plugins/defaults/memory/v3/__tests__/injection.test.ts +81 -1
- package/src/plugins/defaults/memory/v3/__tests__/orchestrate.test.ts +87 -0
- package/src/plugins/defaults/memory/v3/__tests__/plugin-schema.test.ts +12 -0
- package/src/plugins/defaults/memory/v3/__tests__/pool-log-store.test.ts +332 -15
- package/src/plugins/defaults/memory/v3/__tests__/pool-select.test.ts +626 -27
- package/src/plugins/defaults/memory/v3/__tests__/shadow-plugin.test.ts +148 -28
- package/src/plugins/defaults/memory/v3/card.ts +2 -1
- package/src/plugins/defaults/memory/v3/injector.ts +212 -178
- package/src/plugins/defaults/memory/v3/orchestrate.ts +159 -32
- package/src/plugins/defaults/memory/v3/plugin-schema.ts +54 -4
- package/src/plugins/defaults/memory/v3/pool-log-store.ts +275 -37
- package/src/plugins/defaults/memory/v3/pool-select.test.ts +47 -5
- package/src/plugins/defaults/memory/v3/pool-select.ts +710 -59
- package/src/plugins/defaults/memory/v3/sections.ts +2 -1
- package/src/plugins/defaults/memory/v3/shadow-plugin.ts +41 -14
- package/src/plugins/defaults/memory/worker-control.ts +15 -5
- package/src/plugins/defaults/task-progress/__tests__/registration.test.ts +70 -0
- package/src/plugins/defaults/task-progress/__tests__/task-progress-context.test.ts +380 -0
- package/src/plugins/defaults/task-progress/hooks/post-compact.ts +19 -0
- package/src/plugins/defaults/task-progress/hooks/user-prompt-submit.ts +24 -0
- package/src/plugins/defaults/task-progress/package.json +14 -0
- package/src/plugins/defaults/task-progress/src/apply-task-progress-context.ts +93 -0
- package/src/plugins/defaults/task-progress/src/format-task-progress-context.ts +67 -0
- package/src/plugins/defaults/tool-error/hooks/post-tool-use.ts +4 -1
- package/src/plugins/defaults/tool-result-truncate/terminal.ts +1 -46
- package/src/prompts/__tests__/parallel-tasks-section.test.ts +25 -0
- package/src/prompts/delegation-gate.ts +57 -0
- package/src/prompts/system-prompt.ts +14 -32
- package/src/providers/__tests__/context-overflow-error.test.ts +10 -0
- package/src/providers/__tests__/search-provider-catalog.test.ts +18 -0
- package/src/providers/anthropic/client.ts +10 -1
- package/src/providers/anthropic-gateway-shared.ts +6 -0
- package/src/providers/fetch-provider-catalog.ts +11 -0
- package/src/providers/gemini/client.ts +25 -8
- package/src/providers/inference/adapter-factory.ts +6 -0
- package/src/providers/jev/client.test.ts +270 -0
- package/src/providers/jev/client.ts +534 -0
- package/src/providers/model-catalog.ts +59 -3
- package/src/providers/openai/__tests__/chat-completions-extra-body.test.ts +94 -1
- package/src/providers/openai/__tests__/chat-completions-provider-reasoning.test.ts +106 -1
- package/src/providers/openai/__tests__/chat-template-flatten.test.ts +24 -0
- package/src/providers/openai/__tests__/google-thought-signature.test.ts +1 -2
- package/src/providers/openai/chat-completions-provider.ts +100 -29
- package/src/providers/openai/google-thought-signature.ts +0 -6
- package/src/providers/openai/responses-provider.ts +23 -1
- package/src/providers/ratelimit.ts +1 -3
- package/src/providers/search-provider-catalog.ts +24 -0
- package/src/providers/server-tool-pairing.ts +16 -7
- package/src/providers/speech-to-text/__tests__/deepgram-flux-frames.test.ts +24 -0
- package/src/providers/speech-to-text/__tests__/deepgram-flux-realtime.test.ts +7 -1
- package/src/providers/speech-to-text/__tests__/vellum-managed-realtime.test.ts +50 -1
- package/src/providers/speech-to-text/deepgram-flux-frames.ts +11 -0
- package/src/providers/speech-to-text/vellum-managed-flux-realtime.ts +13 -0
- package/src/providers/types.ts +5 -0
- package/src/routes/control.ts +7 -5
- package/src/runtime/AGENTS.md +6 -4
- package/src/runtime/__tests__/agent-wake.test.ts +94 -1
- package/src/runtime/actor-trust-resolver.ts +3 -3
- package/src/runtime/agent-wake.ts +32 -5
- package/src/runtime/approval-message-composer.ts +3 -69
- package/src/runtime/assistant-event-hub.ts +73 -75
- package/src/runtime/assistant-event-publish-options.ts +7 -0
- package/src/runtime/assistant-event-targeting.ts +79 -0
- package/src/runtime/assistant-stream-state.ts +13 -100
- package/src/runtime/background-job-runner.ts +5 -5
- package/src/runtime/channel-retry-sweep.ts +1 -0
- package/src/runtime/guardian-action-message-composer.ts +8 -161
- package/src/runtime/guardian-action-service.ts +2 -17
- package/src/runtime/guardian-reply-router.ts +29 -40
- package/src/runtime/http-server.ts +2 -2
- package/src/runtime/http-types.ts +6 -0
- package/src/runtime/message-composer-types.ts +3 -61
- package/src/runtime/migrations/__tests__/vbundle-import-parity.test.ts +67 -1
- package/src/runtime/migrations/__tests__/vbundle-import-policy.test.ts +89 -0
- package/src/runtime/migrations/vbundle-builder.ts +24 -1
- package/src/runtime/migrations/vbundle-import-analyzer.ts +13 -4
- package/src/runtime/migrations/vbundle-import-policy.ts +43 -11
- package/src/runtime/migrations/vbundle-importer.ts +4 -1
- package/src/runtime/migrations/vbundle-streaming-importer.ts +4 -1
- package/src/runtime/migrations/vbundle-validator.ts +1 -0
- package/src/runtime/pending-interactions-mode-session.test.ts +62 -0
- package/src/runtime/pending-interactions.ts +29 -0
- package/src/runtime/routes/__tests__/client-history-routes.test.ts +104 -0
- package/src/runtime/routes/__tests__/image-generation-routes.test.ts +97 -0
- package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +24 -2
- package/src/runtime/routes/__tests__/roadmap-routes.test.ts +29 -6
- package/src/runtime/routes/browser-context.ts +31 -11
- package/src/runtime/routes/browser-routes.ts +43 -4
- package/src/runtime/routes/btw-routes.ts +9 -4
- package/src/runtime/routes/channel-route-shared.ts +1 -9
- package/src/runtime/routes/client-routes.ts +108 -0
- package/src/runtime/routes/conversation-management-routes.ts +4 -1
- package/src/runtime/routes/conversation-routes.ts +125 -14
- package/src/runtime/routes/credential-prompt-routes.ts +4 -3
- package/src/runtime/routes/credential-routes.ts +1 -2
- package/src/runtime/routes/desktop-setup-routes.test.ts +9 -18
- package/src/runtime/routes/desktop-setup-routes.ts +8 -7
- package/src/runtime/routes/documents-routes.ts +51 -0
- package/src/runtime/routes/email-routes.ts +3 -7
- package/src/runtime/routes/events-routes.ts +75 -54
- package/src/runtime/routes/guardian-approval-interception.ts +24 -0
- package/src/runtime/routes/identity-routes.ts +1 -5
- package/src/runtime/routes/image-generation-routes.ts +8 -14
- package/src/runtime/routes/inbound-message-handler.ts +37 -41
- package/src/runtime/routes/inbound-stages/backfill-sender-provenance.test.ts +168 -0
- package/src/runtime/routes/inbound-stages/backfill-sender-provenance.ts +96 -0
- package/src/runtime/routes/inbound-stages/background-dispatch.test.ts +39 -10
- package/src/runtime/routes/inbound-stages/background-dispatch.ts +9 -9
- package/src/runtime/routes/inbound-stages/reaction-intercept.ts +2 -0
- package/src/runtime/routes/inference-profile-session-handler.ts +11 -1
- package/src/runtime/routes/inference-profiles-routes.ts +9 -0
- package/src/runtime/routes/llm-context-normalization.ts +1 -7
- package/src/runtime/routes/migration-routes.ts +73 -2
- package/src/runtime/routes/notification-routes.ts +10 -0
- package/src/runtime/routes/oauth-commands-routes.ts +57 -54
- package/src/runtime/routes/oauth-providers.ts +11 -0
- package/src/runtime/routes/oauth-request-hints.test.ts +246 -0
- package/src/runtime/routes/oauth-request-hints.ts +149 -0
- package/src/runtime/routes/plugins-routes.ts +6 -0
- package/src/runtime/routes/roadmap-routes.ts +47 -8
- package/src/runtime/routes/schedule-routes.ts +2 -0
- package/src/runtime/routes/secret-routes.ts +99 -119
- package/src/runtime/routes/settings-routes.ts +10 -4
- package/src/runtime/routes/watcher-routes.ts +1 -3
- package/src/runtime/trust-class.test.ts +7 -6
- package/src/runtime/trust-class.ts +9 -17
- package/src/schedule/__tests__/worker-control.test.ts +36 -5
- package/src/schedule/__tests__/worker-mcp-bootstrap.test.ts +107 -0
- package/src/schedule/__tests__/worker-mcp-readiness.test.ts +173 -0
- package/src/schedule/__tests__/worker-mcp-tools.test.ts +13 -47
- package/src/schedule/run-script.ts +2 -2
- package/src/schedule/scheduler.ts +47 -11
- package/src/schedule/tool-surface-readiness.ts +65 -0
- package/src/schedule/worker-control.ts +15 -5
- package/src/schedule/worker-mcp.ts +85 -0
- package/src/schedule/worker.ts +9 -31
- package/src/security/secure-keys.ts +74 -112
- package/src/skills/clawhub.ts +3 -8
- package/src/skills/managed-store.ts +1 -5
- package/src/stt/types.ts +4 -0
- package/src/subagent/notify.ts +54 -34
- package/src/subagent/parent-notification.ts +6 -0
- package/src/telemetry/__tests__/phone-call-funnel.test.ts +72 -0
- package/src/telemetry/phone-call-funnel.ts +144 -0
- package/src/tools/__tests__/tool-schema-root-combinator-guard.test.ts +81 -0
- package/src/tools/ask-question/ask-question-tool.test.ts +167 -5
- package/src/tools/ask-question/ask-question-tool.ts +120 -22
- package/src/tools/browser/browser-execution.ts +29 -60
- package/src/tools/browser/cdp-client/__tests__/cdp-dom-helpers.test.ts +561 -39
- package/src/tools/browser/cdp-client/cdp-dom-helpers.ts +435 -39
- package/src/tools/browser/cdp-client/cdp-inspect-client.ts +1 -8
- package/src/tools/client-os.ts +15 -0
- package/src/tools/computer-use/skill-proxy-bridge.ts +1 -1
- package/src/tools/computer-use/target.ts +42 -0
- package/src/tools/document/document-tool.ts +13 -18
- package/src/tools/execution-target.ts +14 -16
- package/src/tools/host-terminal/host-shell.ts +16 -7
- package/src/tools/network/__tests__/web-fetch-tinyfish.test.ts +250 -0
- package/src/tools/network/__tests__/web-search.test.ts +203 -0
- package/src/tools/network/web-fetch.ts +274 -7
- package/src/tools/network/web-search.ts +468 -6
- package/src/tools/permission-checker.ts +3 -3
- package/src/tools/policy-context.ts +3 -1
- package/src/tools/shared/filesystem/file-ops-service.ts +1 -31
- package/src/tools/shared/shell-output.test.ts +10 -0
- package/src/tools/shared/shell-output.ts +14 -2
- package/src/tools/skills/load.ts +2 -1
- package/src/tools/skills/sandbox-runner.ts +24 -4
- package/src/tools/skills/scaffold-managed.ts +2 -1
- package/src/tools/skills/skill-script-runner.ts +7 -0
- package/src/tools/skills/skill-tool-factory.ts +15 -1
- package/src/tools/terminal/__tests__/safe-env.test.ts +44 -4
- package/src/tools/terminal/__tests__/sanitized-bash.test.ts +0 -14
- package/src/tools/terminal/safe-env.ts +39 -24
- package/src/tools/terminal/sanitized-bash.ts +15 -2
- package/src/tools/terminal/shell-launch.test.ts +162 -0
- package/src/tools/terminal/shell.test.ts +29 -0
- package/src/tools/terminal/shell.ts +15 -7
- package/src/tools/tool-approval-handler.ts +5 -1
- package/src/tools/tool-types.ts +3 -3
- package/src/tools/types.ts +8 -2
- package/src/util/__tests__/worker-process-command.test.ts +5 -4
- package/src/util/abort-reasons.ts +18 -26
- package/src/util/browser-human-verification.ts +14 -0
- package/src/util/canonicalize-identity.test.ts +21 -0
- package/src/util/canonicalize-identity.ts +16 -0
- package/src/util/errors.ts +13 -3
- package/src/util/host-process.test.ts +17 -1
- package/src/util/host-process.ts +24 -0
- package/src/util/provider-error-patterns.ts +3 -0
- package/src/util/unicode.ts +29 -0
- package/src/util/worker-process.ts +77 -35
- package/src/workspace/migrations/157-add-desktop-files-launcher.ts +94 -0
- package/src/workspace/migrations/__tests__/156-extract-workspace-mcp-json.test.ts +4 -10
- package/src/workspace/migrations/__tests__/157-add-desktop-files-launcher.test.ts +113 -0
- package/src/workspace/migrations/registry.ts +2 -0
- package/src/__tests__/computer-use-window-schema.test.ts +0 -24
- package/src/api/events/heartbeat-alert.ts +0 -20
- package/src/daemon/message-types/web-activity.ts +0 -65
- package/src/desktop/desktop-dependencies.test.ts +0 -131
- package/src/tools/computer-use/definitions.ts +0 -607
|
@@ -57,7 +57,8 @@ function contentBlocks(msg: ServerToolPairingMessage): ReadonlyArray<unknown> {
|
|
|
57
57
|
* `tool_use`, no result emitted) and executes it on the next request, placing
|
|
58
58
|
* the web_search_tool_result at the head of the next assistant message. Both
|
|
59
59
|
* the deferred tail and the resulting split pair must be sent back verbatim
|
|
60
|
-
* for the provider to pair and execute them
|
|
60
|
+
* for the provider to pair and execute them, and the user message answering
|
|
61
|
+
* a deferred tail must hold tool_result blocks alone.
|
|
61
62
|
*/
|
|
62
63
|
export interface ServerToolPairing {
|
|
63
64
|
/**
|
|
@@ -67,8 +68,12 @@ export interface ServerToolPairing {
|
|
|
67
68
|
resolvedPairIds: Set<string>;
|
|
68
69
|
/**
|
|
69
70
|
* Unanswered use ids in the final assistant message of an active tool-loop
|
|
70
|
-
* continuation
|
|
71
|
-
*
|
|
71
|
+
* continuation: nothing follows it, or only user messages made solely of
|
|
72
|
+
* tool_result blocks. The provider executes these on this request. Any
|
|
73
|
+
* other trailing block (guidance text after the results, a queued user
|
|
74
|
+
* message) closes the assistant turn on the provider side, which then
|
|
75
|
+
* rejects the unanswered use as unpaired, so such a use is an orphan to
|
|
76
|
+
* repair rather than a deferral to preserve.
|
|
72
77
|
*/
|
|
73
78
|
deferredUseIds: Set<string>;
|
|
74
79
|
/** Ids whose use/result pair spans two messages. */
|
|
@@ -118,10 +123,14 @@ export function analyzeServerToolPairing(
|
|
|
118
123
|
if (lastAssistantIndex >= 0) {
|
|
119
124
|
const trailing = messages.slice(lastAssistantIndex + 1);
|
|
120
125
|
const trailingAllUser = trailing.every((m) => m.role === "user");
|
|
121
|
-
const
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
126
|
+
const trailingBlocks = trailing.flatMap((m) => [...contentBlocks(m)]);
|
|
127
|
+
const trailingIsToolResultsOnly =
|
|
128
|
+
trailingBlocks.length > 0 &&
|
|
129
|
+
trailingBlocks.every(isToolResultShapedBlock);
|
|
130
|
+
if (
|
|
131
|
+
trailingAllUser &&
|
|
132
|
+
(trailing.length === 0 || trailingIsToolResultsOnly)
|
|
133
|
+
) {
|
|
125
134
|
for (const block of contentBlocks(messages[lastAssistantIndex])) {
|
|
126
135
|
if (isServerToolUseBlock(block) && !resolvedPairIds.has(block.id)) {
|
|
127
136
|
deferredUseIds.add(block.id);
|
|
@@ -129,6 +129,30 @@ describe("parseFluxFrame", () => {
|
|
|
129
129
|
]);
|
|
130
130
|
});
|
|
131
131
|
|
|
132
|
+
test.each(["model", "timeout", "manual"])(
|
|
133
|
+
"retains the %s turn-end trigger and decoded audio position",
|
|
134
|
+
(trigger) => {
|
|
135
|
+
expect(
|
|
136
|
+
parseFluxFrame({ ...END_OF_TURN, trigger, audio_window_end: 12.5 })[1],
|
|
137
|
+
).toMatchObject({
|
|
138
|
+
type: "turn-end",
|
|
139
|
+
confidence: 0.88,
|
|
140
|
+
trigger,
|
|
141
|
+
audioWindowEndSeconds: 12.5,
|
|
142
|
+
});
|
|
143
|
+
},
|
|
144
|
+
);
|
|
145
|
+
|
|
146
|
+
test("omits unknown triggers and malformed audio positions", () => {
|
|
147
|
+
expect(
|
|
148
|
+
parseFluxFrame({
|
|
149
|
+
...END_OF_TURN,
|
|
150
|
+
trigger: "future-trigger",
|
|
151
|
+
audio_window_end: "invalid",
|
|
152
|
+
}),
|
|
153
|
+
).toEqual(parseFluxFrame(END_OF_TURN));
|
|
154
|
+
});
|
|
155
|
+
|
|
132
156
|
test("flat frames without a nested event field are accepted", () => {
|
|
133
157
|
expect(parseFluxFrame({ type: "StartOfTurn" })).toEqual([
|
|
134
158
|
{ type: "turn-start" },
|
|
@@ -334,6 +334,7 @@ describe("DeepgramFluxRealtimeTranscriber", () => {
|
|
|
334
334
|
type: "turn-end",
|
|
335
335
|
text: "what is the weather",
|
|
336
336
|
confidence: 0.91,
|
|
337
|
+
audioWindowEndSeconds: 1.5,
|
|
337
338
|
turnIndex: 0,
|
|
338
339
|
},
|
|
339
340
|
]);
|
|
@@ -382,7 +383,12 @@ describe("DeepgramFluxRealtimeTranscriber", () => {
|
|
|
382
383
|
);
|
|
383
384
|
expect(events).toEqual([
|
|
384
385
|
{ type: "final", text: "still here" },
|
|
385
|
-
{
|
|
386
|
+
{
|
|
387
|
+
type: "turn-end",
|
|
388
|
+
text: "still here",
|
|
389
|
+
turnIndex: 0,
|
|
390
|
+
audioWindowEndSeconds: 1.5,
|
|
391
|
+
},
|
|
386
392
|
]);
|
|
387
393
|
});
|
|
388
394
|
});
|
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
import { afterEach, beforeEach, describe, expect, test } from "bun:test";
|
|
9
9
|
|
|
10
10
|
import { SttError, type SttStreamServerEvent } from "../../../stt/types.js";
|
|
11
|
+
import { VellumManagedFluxRealtimeTranscriber } from "../vellum-managed-flux-realtime.js";
|
|
11
12
|
import { VellumManagedRealtimeTranscriber } from "../vellum-managed-realtime.js";
|
|
12
13
|
import type { SpeechRelayConnection } from "../vellum-speech-relay-connection.js";
|
|
13
14
|
|
|
@@ -99,7 +100,7 @@ const CONNECTION: SpeechRelayConnection = {
|
|
|
99
100
|
|
|
100
101
|
const tick = () => new Promise<void>((r) => setTimeout(r, 0));
|
|
101
102
|
|
|
102
|
-
describe("
|
|
103
|
+
describe("Vellum managed realtime transcribers", () => {
|
|
103
104
|
let sockets: MockWebSocket[];
|
|
104
105
|
let originalWebSocket: unknown;
|
|
105
106
|
let originalFetch: unknown;
|
|
@@ -262,6 +263,54 @@ describe("VellumManagedRealtimeTranscriber", () => {
|
|
|
262
263
|
expect(new URL(sockets[1]!.url).searchParams.get("key")).toBe("vk-secret");
|
|
263
264
|
});
|
|
264
265
|
|
|
266
|
+
test("Flux turn indices stay in one sequence across relay reconnects", async () => {
|
|
267
|
+
const adapter = new VellumManagedFluxRealtimeTranscriber(CONNECTION);
|
|
268
|
+
const { events, onEvent } = collector();
|
|
269
|
+
const started = adapter.start(onEvent);
|
|
270
|
+
sockets[0]!.simulateOpen();
|
|
271
|
+
await started;
|
|
272
|
+
try {
|
|
273
|
+
for (let generation = 0; generation < 3; generation += 1) {
|
|
274
|
+
const socket = sockets[generation]!;
|
|
275
|
+
const turnIndex = generation === 0 ? 5 : 0;
|
|
276
|
+
const startFrame = JSON.stringify({
|
|
277
|
+
type: "TurnInfo",
|
|
278
|
+
event: "StartOfTurn",
|
|
279
|
+
turn_index: turnIndex,
|
|
280
|
+
});
|
|
281
|
+
socket.simulateMessage(startFrame);
|
|
282
|
+
socket.simulateMessage(startFrame);
|
|
283
|
+
socket.simulateMessage(
|
|
284
|
+
JSON.stringify({
|
|
285
|
+
type: "TurnInfo",
|
|
286
|
+
event: "EndOfTurn",
|
|
287
|
+
// A later end without its start still advances the sequence.
|
|
288
|
+
turn_index: generation === 1 ? 2 : turnIndex,
|
|
289
|
+
transcript: "hello",
|
|
290
|
+
}),
|
|
291
|
+
);
|
|
292
|
+
if (generation < 2) {
|
|
293
|
+
socket.simulateMessage(velayErrorFrame("session_duration_exceeded"));
|
|
294
|
+
socket.simulateClose(1000, "session_duration_exceeded");
|
|
295
|
+
await tick();
|
|
296
|
+
sockets[generation + 1]!.simulateOpen();
|
|
297
|
+
await tick();
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
expect(
|
|
301
|
+
events.flatMap((event) =>
|
|
302
|
+
event.type === "turn-start" || event.type === "turn-end"
|
|
303
|
+
? [event.turnIndex]
|
|
304
|
+
: [],
|
|
305
|
+
),
|
|
306
|
+
).toEqual([5, 5, 5, 6, 6, 8, 9, 9, 9]);
|
|
307
|
+
expect(events.some((event) => event.type === "closed")).toBe(false);
|
|
308
|
+
} finally {
|
|
309
|
+
adapter.stop();
|
|
310
|
+
sockets.at(-1)!.simulateClose();
|
|
311
|
+
}
|
|
312
|
+
});
|
|
313
|
+
|
|
265
314
|
test("finalize during a re-dial settles immediately", async () => {
|
|
266
315
|
const { adapter, events } = await startAdapter();
|
|
267
316
|
|
|
@@ -71,6 +71,7 @@ export interface FluxTurnFrame {
|
|
|
71
71
|
transcript?: string;
|
|
72
72
|
/** Flux's end-of-turn confidence in [0, 1]. */
|
|
73
73
|
end_of_turn_confidence?: number;
|
|
74
|
+
trigger?: "model" | "manual" | "timeout";
|
|
74
75
|
}
|
|
75
76
|
|
|
76
77
|
/**
|
|
@@ -153,6 +154,8 @@ export function parseFluxFrame(raw: unknown): SttStreamServerEvent[] {
|
|
|
153
154
|
case "EndOfTurn": {
|
|
154
155
|
const text = readTranscript(frame);
|
|
155
156
|
const confidence = readNumber(frame.end_of_turn_confidence);
|
|
157
|
+
const trigger = frame.trigger;
|
|
158
|
+
const audioWindowEndSeconds = readNumber(frame.audio_window_end);
|
|
156
159
|
return [
|
|
157
160
|
{ type: "final", text },
|
|
158
161
|
{
|
|
@@ -160,6 +163,14 @@ export function parseFluxFrame(raw: unknown): SttStreamServerEvent[] {
|
|
|
160
163
|
text,
|
|
161
164
|
...(confidence !== undefined ? { confidence } : {}),
|
|
162
165
|
...(turnIndex !== undefined ? { turnIndex } : {}),
|
|
166
|
+
...(trigger === "model" ||
|
|
167
|
+
trigger === "manual" ||
|
|
168
|
+
trigger === "timeout"
|
|
169
|
+
? { trigger }
|
|
170
|
+
: {}),
|
|
171
|
+
...(audioWindowEndSeconds !== undefined
|
|
172
|
+
? { audioWindowEndSeconds }
|
|
173
|
+
: {}),
|
|
163
174
|
},
|
|
164
175
|
];
|
|
165
176
|
}
|
|
@@ -79,6 +79,9 @@ export class VellumManagedFluxRealtimeTranscriber implements StreamingTranscribe
|
|
|
79
79
|
private redialOnClose = false;
|
|
80
80
|
private stopping = false;
|
|
81
81
|
private closedEmitted = false;
|
|
82
|
+
// Consumers see one stream across relay reconnects, with one turn sequence.
|
|
83
|
+
private turnIndexOffset = 0;
|
|
84
|
+
private latestTurnIndex = -1;
|
|
82
85
|
|
|
83
86
|
constructor(
|
|
84
87
|
connection: SpeechRelayConnection,
|
|
@@ -225,6 +228,15 @@ export class VellumManagedFluxRealtimeTranscriber implements StreamingTranscribe
|
|
|
225
228
|
return;
|
|
226
229
|
}
|
|
227
230
|
|
|
231
|
+
if (
|
|
232
|
+
(event.type === "turn-start" || event.type === "turn-end") &&
|
|
233
|
+
event.turnIndex !== undefined
|
|
234
|
+
) {
|
|
235
|
+
const turnIndex = this.turnIndexOffset + event.turnIndex;
|
|
236
|
+
this.latestTurnIndex = Math.max(this.latestTurnIndex, turnIndex);
|
|
237
|
+
this.emit({ ...event, turnIndex });
|
|
238
|
+
return;
|
|
239
|
+
}
|
|
228
240
|
this.emit(event);
|
|
229
241
|
}
|
|
230
242
|
|
|
@@ -236,6 +248,7 @@ export class VellumManagedFluxRealtimeTranscriber implements StreamingTranscribe
|
|
|
236
248
|
private beginRedial(): void {
|
|
237
249
|
log.info("Managed Flux relay session hit velay's duration cap, re-dialing");
|
|
238
250
|
this.inner = null;
|
|
251
|
+
this.turnIndexOffset = this.latestTurnIndex + 1;
|
|
239
252
|
void (async () => {
|
|
240
253
|
try {
|
|
241
254
|
const inner = await this.dial();
|
package/src/providers/types.ts
CHANGED
|
@@ -551,6 +551,8 @@ export interface ContextOverflowErrorOptions {
|
|
|
551
551
|
cause?: unknown;
|
|
552
552
|
/** Semantic reason override; defaults to `context_overflow`. */
|
|
553
553
|
reason?: ProviderErrorReason;
|
|
554
|
+
/** Inspectable SDK/wire request payload for the overflowing call. */
|
|
555
|
+
rawRequest?: unknown;
|
|
554
556
|
}
|
|
555
557
|
|
|
556
558
|
/**
|
|
@@ -581,6 +583,9 @@ export class ContextOverflowError extends ProviderError {
|
|
|
581
583
|
super(message, provider, options.statusCode ?? 400, {
|
|
582
584
|
reason: options.reason ?? "context_overflow",
|
|
583
585
|
...(options.cause !== undefined ? { cause: options.cause } : {}),
|
|
586
|
+
...(options.rawRequest !== undefined
|
|
587
|
+
? { rawRequest: options.rawRequest }
|
|
588
|
+
: {}),
|
|
584
589
|
});
|
|
585
590
|
this.name = "ContextOverflowError";
|
|
586
591
|
this.actualTokens = options.actualTokens;
|
package/src/routes/control.ts
CHANGED
|
@@ -44,13 +44,15 @@ function routeHostPidPath(): string {
|
|
|
44
44
|
return getProcPidPath(ROUTE_HOST_PROC_NAME);
|
|
45
45
|
}
|
|
46
46
|
|
|
47
|
+
const ROUTE_HOST_ENTRY = new URL("./worker.ts", import.meta.url);
|
|
48
|
+
|
|
47
49
|
/**
|
|
48
50
|
* Read the PID file and report liveness. A missing or malformed file reports
|
|
49
|
-
* not_running; a file pointing at a dead process
|
|
50
|
-
* not_running.
|
|
51
|
+
* not_running; a file pointing at a dead process, or at a live process that
|
|
52
|
+
* is not this worker, is cleaned up and reported as not_running.
|
|
51
53
|
*/
|
|
52
54
|
export function probeRouteHostWorker(): WorkerProcessStatus {
|
|
53
|
-
return probeWorkerPidFile(routeHostPidPath());
|
|
55
|
+
return probeWorkerPidFile(routeHostPidPath(), ROUTE_HOST_ENTRY, "routes");
|
|
54
56
|
}
|
|
55
57
|
|
|
56
58
|
export class RouteHostSpawnError extends WorkerProcessSpawnError {}
|
|
@@ -67,7 +69,7 @@ export async function spawnRouteHostWorkerProcess(
|
|
|
67
69
|
try {
|
|
68
70
|
return await spawnWorkerProcess({
|
|
69
71
|
pidPath: routeHostPidPath(),
|
|
70
|
-
entry:
|
|
72
|
+
entry: ROUTE_HOST_ENTRY,
|
|
71
73
|
packagedEntry: "routes",
|
|
72
74
|
workerLabel: "Route host",
|
|
73
75
|
options: opts,
|
|
@@ -86,7 +88,7 @@ export async function spawnRouteHostWorkerProcess(
|
|
|
86
88
|
* (e.g. EPERM) — a not-running host is a no-op.
|
|
87
89
|
*/
|
|
88
90
|
export function stopRouteHostWorkerProcess(): WorkerProcessStatus {
|
|
89
|
-
return stopWorkerProcess(routeHostPidPath());
|
|
91
|
+
return stopWorkerProcess(routeHostPidPath(), ROUTE_HOST_ENTRY, "routes");
|
|
90
92
|
}
|
|
91
93
|
|
|
92
94
|
/**
|
package/src/runtime/AGENTS.md
CHANGED
|
@@ -24,7 +24,7 @@ Queueing remains the fallback, not a legacy path, and every non-`released` outco
|
|
|
24
24
|
|
|
25
25
|
A repair that fell back to the queue carries its durable requirement onto the drain that runs the queued message, and `pendingInterruptRepair` is what carries it: the drain reads that flag and asks `repairInterruptedToolUseBlocks` for `requireDurable`, so a second persist failure leaves the flag armed and the message queued instead of persisting the prompt behind an unmatched durable `tool_use`. Only a steered drain settles for an in-memory repair, because it has no row of its own to persist behind it.
|
|
26
26
|
|
|
27
|
-
**The
|
|
27
|
+
**The two interrupt texts are split by job, and the behavior instruction lives in exactly one of them.** A synthetic `tool_result` (`PREEMPTED_TOOL_RESULT_TEXT`) states the fact about the one call it answers: the call stopped early and may have taken effect anyway. It is repeated once per stopped call in a parallel batch, so it holds no instruction. Every completed handover arms `pendingInterruptNote`, whether or not a tool call was stopped. The persist of the interrupting user message consumes that flag exactly once: it stamps `interruptedPriorTurn` on the row and appends `INTERRUPTED_TURN_NOTE_TEXT` to the message's LLM-facing content only, so the persisted row stays what the user typed and no client renders the note. That note is the single place the behavior after an interrupt is written, it leads with the short acknowledgement the user is waiting for, and it sits after the user's words instead of before them. `reinjectInterruptTurnNote` rebuilds the identical block at history load, next to the attachment annotations and for the same reason: a reload that rebuilds it differently breaks provider prefix-cache parity. Do not add a third annotation, and do not put instructions back in the tool result.
|
|
28
28
|
|
|
29
29
|
Two ordering rules a change here must not break. The synthetic `tool_result` is persisted BEFORE the caller writes the interrupting user row, because a user message between a `tool_use` and its result is a sequence every provider rejects. And the `thinking` / `message_interrupted` activity state is emitted by the agent loop at the head of the replacement turn, never at the handover: activity states are cached in `lastActivityStateMsg` and replayed to reconnecting clients, so a send that fails before its turn starts would otherwise strand every client on a busy conversation that is running nothing.
|
|
30
30
|
|
|
@@ -163,7 +163,7 @@ In the CDP factory the bridge is the internal `"host-bridge"` candidate kind (`I
|
|
|
163
163
|
|
|
164
164
|
### Browser CLI surface defaults
|
|
165
165
|
|
|
166
|
-
The browser execute and tab routes share `browser/virtual-desktop-target.ts`. Platform-hosted web guardian conversations use
|
|
166
|
+
The browser execute and tab routes share `browser/virtual-desktop-target.ts`. Platform-hosted web guardian conversations use image-provided virtual desktop Chrome by default when enabled. The shared `isVirtualDesktopEnabled` gate requires both `IS_PLATFORM` and `IS_CONTAINERIZED` alongside the `assistant-desktop` flag. Native renderer turns also use the `web` transport, so the frozen turn `clientOs` excludes native apps from automatic streamed-browser selection. Native apps retain their existing backend selection and fallback behavior. Explicit desktop/backend/client targets and existing personal-browser sessions override the surface default. Disabled desktop support retains the existing browser path. Missing image components fail the managed operation without installing dependencies or falling back to a personal browser.
|
|
167
167
|
|
|
168
168
|
### Per-tool `browser_mode` override
|
|
169
169
|
|
|
@@ -277,6 +277,8 @@ Verification SESSION state (sessions, secrets, rate limits, validate+consume) AN
|
|
|
277
277
|
|
|
278
278
|
The daemon relays session lifecycle operations over the `verification_sessions_*` IPC routes via `assistant/src/channels/gateway-verification-sessions.ts` and keeps what is presentation: message composition and channel delivery (`channel-verification-routes.ts`, `verification-outbound-actions.ts`). `channel-verification-service.ts` retains only guardian-delivery reads (`getGuardianBinding`, `isGuardian`, `isGuardianBoundForChannel`).
|
|
279
279
|
|
|
280
|
+
The gateway answers a code (or an invite) it consumed at ingress with a reply it composes itself, and delivers it through the daemon's IPC-only `deliver_gateway_reply` method (`ipc/routes/channel-reply-ipc-routes.ts`), which hands it to the channel transport the inbound message's callback URL names. The gateway never sends to a provider itself: the transports are the one outbound path per channel, whoever composed the text. The reply is not recorded in a conversation, since the message it answers never entered one.
|
|
281
|
+
|
|
280
282
|
The verified outcome is written in-process by the gateway: the HTTP guardian-attest handler calls `ContactStore.markChannelVerified` directly (verifiedVia "manual"); the code-match paths (text and the `verification_sessions_validate_consume` engine route) apply role side effects in-engine — guardian phone binding commits in the same gateway transaction as the consume. The revoke/downgrade outcome is relayed from the daemon via `ipcCallPersistent("mark_channel_revoked", …)` to `ContactStore.markChannelRevoked`.
|
|
281
283
|
|
|
282
284
|
## Rate Limiting & Diagnostics
|
|
@@ -301,6 +303,6 @@ The provider-level rate limiter (`providers/ratelimit.ts`) also logs warnings (m
|
|
|
301
303
|
|
|
302
304
|
## HTTP-Only Transport
|
|
303
305
|
|
|
304
|
-
HTTP is the sole transport for
|
|
306
|
+
HTTP is the sole transport for browser and desktop clients; the CLI and the gateway reach the daemon over the IPC socket (see `assistant/AGENTS.md`). The runtime HTTP server (`assistant/src/runtime/http-server.ts`) is the canonical API surface. Clients connect via HTTP for request/response operations and SSE (`GET /v1/events`) for streaming server-to-client events.
|
|
305
307
|
|
|
306
|
-
|
|
308
|
+
A skill never calls the runtime HTTP API directly: every request from a skill targets the gateway, and `gateway-only-guard.test.ts` fails CI on a runtime-port URL in a skill. For configuration reads, use the CLI (`assistant config get`); for control-plane writes with no CLI verb, use `$INTERNAL_GATEWAY_BASE_URL`. See "Gateway-Only API Consumption" in `gateway/AGENTS.md` for the rule and its exceptions.
|
|
@@ -18,9 +18,10 @@
|
|
|
18
18
|
* route captured calls back to the originating conversation's probe arrays.
|
|
19
19
|
*/
|
|
20
20
|
|
|
21
|
-
import { beforeEach, describe, expect, mock, test } from "bun:test";
|
|
21
|
+
import { beforeEach, describe, expect, mock, spyOn, test } from "bun:test";
|
|
22
22
|
|
|
23
23
|
import type { DiskPressureStatus } from "../../daemon/disk-pressure-guard.js";
|
|
24
|
+
import { desktopAutomationLease } from "../../desktop/desktop-automation-lease.js";
|
|
24
25
|
|
|
25
26
|
// ── Per-conversation capture registry ────────────────────────────────
|
|
26
27
|
//
|
|
@@ -501,6 +502,13 @@ function makeWakeConversation(options: {
|
|
|
501
502
|
`tools:${snapshotAllowedTools()?.join(",") ?? "all"}`,
|
|
502
503
|
);
|
|
503
504
|
},
|
|
505
|
+
// Mirrors Conversation.setPreactivatedSkillIds; the wake's tool-scope
|
|
506
|
+
// restore calls it, and a double without it throws inside the restore
|
|
507
|
+
// closure, leaving every field restored after it untouched.
|
|
508
|
+
preactivatedSkillIds: undefined as string[] | undefined,
|
|
509
|
+
setPreactivatedSkillIds(ids: string[] | undefined) {
|
|
510
|
+
this.preactivatedSkillIds = ids;
|
|
511
|
+
},
|
|
504
512
|
get wakePersonaOverride() {
|
|
505
513
|
return wakePersonaOverride;
|
|
506
514
|
},
|
|
@@ -1357,6 +1365,50 @@ describe("wakeAgentForOpportunity", () => {
|
|
|
1357
1365
|
expect(conversation.toolContextPin).toBeUndefined();
|
|
1358
1366
|
});
|
|
1359
1367
|
|
|
1368
|
+
test("applies wireToolDefinitions and the delegation state alongside the allowlist and restores both after the wake", async () => {
|
|
1369
|
+
const replay = [
|
|
1370
|
+
{ name: "remember", description: "Save", input_schema: {} },
|
|
1371
|
+
{ name: "bell_jingle", description: "Ring", input_schema: {} },
|
|
1372
|
+
];
|
|
1373
|
+
let replayDuringRun: unknown;
|
|
1374
|
+
let delegationDuringRun: unknown;
|
|
1375
|
+
const conversation = makeWakeConversation({
|
|
1376
|
+
runImpl: async (input) => {
|
|
1377
|
+
replayDuringRun = conversation.wireToolReplay;
|
|
1378
|
+
delegationDuringRun = conversation.delegateIndependentTasksReplay;
|
|
1379
|
+
return runResult([
|
|
1380
|
+
...input,
|
|
1381
|
+
{ role: "assistant", content: [{ type: "text", text: "Saved." }] },
|
|
1382
|
+
]);
|
|
1383
|
+
},
|
|
1384
|
+
});
|
|
1385
|
+
|
|
1386
|
+
const result = await wakeAgentForOpportunity(
|
|
1387
|
+
{
|
|
1388
|
+
conversationId: conversation.conversationId,
|
|
1389
|
+
hint: "review for memories",
|
|
1390
|
+
source: "memory-retrospective",
|
|
1391
|
+
allowedTools: ["remember"],
|
|
1392
|
+
toolGateMode: "execution",
|
|
1393
|
+
wireToolDefinitions: replay,
|
|
1394
|
+
delegateIndependentTasks: true,
|
|
1395
|
+
},
|
|
1396
|
+
{ resolveTarget: async () => conversation },
|
|
1397
|
+
);
|
|
1398
|
+
|
|
1399
|
+
expect(result).toEqual({ invoked: true, producedToolCalls: false });
|
|
1400
|
+
// The replay array is live on the conversation for the duration of the
|
|
1401
|
+
// run (the tool resolver returns it as the wire array), as is the
|
|
1402
|
+
// delegation-section state the source's prompt rendered (the prompt
|
|
1403
|
+
// build reads it)...
|
|
1404
|
+
expect(replayDuringRun).toEqual(replay);
|
|
1405
|
+
expect(delegationDuringRun).toBe(true);
|
|
1406
|
+
// ...and both are cleared alongside the allowlist + gate mode after the
|
|
1407
|
+
// wake.
|
|
1408
|
+
expect(conversation.wireToolReplay).toBeUndefined();
|
|
1409
|
+
expect(conversation.delegateIndependentTasksReplay).toBeUndefined();
|
|
1410
|
+
});
|
|
1411
|
+
|
|
1360
1412
|
test("defaults to the wire gate mode when toolGateMode is absent", async () => {
|
|
1361
1413
|
let gateModeDuringRun: string | undefined;
|
|
1362
1414
|
const conversation = makeWakeConversation({
|
|
@@ -1517,6 +1569,47 @@ describe("wakeAgentForOpportunity", () => {
|
|
|
1517
1569
|
expect(conversation.pushedMessages[2]).toEqual(followupAssistant);
|
|
1518
1570
|
});
|
|
1519
1571
|
|
|
1572
|
+
for (const fails of [false, true]) {
|
|
1573
|
+
test(`releases desktop control before a ${fails ? "failed" : "completed"} wake drains the queue`, async () => {
|
|
1574
|
+
const conversation = makeWakeConversation({
|
|
1575
|
+
scriptedAssistant: {
|
|
1576
|
+
role: "assistant",
|
|
1577
|
+
content: [{ type: "text", text: "reply" }],
|
|
1578
|
+
},
|
|
1579
|
+
...(fails
|
|
1580
|
+
? {
|
|
1581
|
+
runImpl: async () => {
|
|
1582
|
+
throw new Error("wake failed");
|
|
1583
|
+
},
|
|
1584
|
+
}
|
|
1585
|
+
: {}),
|
|
1586
|
+
});
|
|
1587
|
+
const release = spyOn(desktopAutomationLease, "releaseForConversation");
|
|
1588
|
+
const setProcessing = conversation.setProcessing.bind(conversation);
|
|
1589
|
+
conversation.setProcessing = (processing) => {
|
|
1590
|
+
if (!processing) {
|
|
1591
|
+
expect(release).toHaveBeenCalledWith(conversation.conversationId);
|
|
1592
|
+
}
|
|
1593
|
+
setProcessing(processing);
|
|
1594
|
+
};
|
|
1595
|
+
try {
|
|
1596
|
+
const result = await wakeAgentForOpportunity(
|
|
1597
|
+
{
|
|
1598
|
+
conversationId: conversation.conversationId,
|
|
1599
|
+
hint: "x",
|
|
1600
|
+
source: "unit-test",
|
|
1601
|
+
},
|
|
1602
|
+
{ resolveTarget: async () => conversation },
|
|
1603
|
+
);
|
|
1604
|
+
expect(result.invoked).toBe(!fails);
|
|
1605
|
+
expect(release).toHaveBeenCalledTimes(1);
|
|
1606
|
+
expect(conversation.drainQueueCalls).toBe(1);
|
|
1607
|
+
} finally {
|
|
1608
|
+
release.mockRestore();
|
|
1609
|
+
}
|
|
1610
|
+
});
|
|
1611
|
+
}
|
|
1612
|
+
|
|
1520
1613
|
test("marks processing true during the run and false afterwards", async () => {
|
|
1521
1614
|
const conversation = makeWakeConversation({
|
|
1522
1615
|
baseline: [{ role: "user", content: [{ type: "text", text: "hi" }] }],
|
|
@@ -42,9 +42,9 @@ const log = getLogger("actor-trust-resolver");
|
|
|
42
42
|
// ---------------------------------------------------------------------------
|
|
43
43
|
|
|
44
44
|
/**
|
|
45
|
-
* Trust classification for an inbound actor.
|
|
46
|
-
*
|
|
47
|
-
*
|
|
45
|
+
* Trust classification for an inbound actor. Declared in
|
|
46
|
+
* `@vellumai/gateway-client` and re-exported here for the resolver's
|
|
47
|
+
* consumers.
|
|
48
48
|
*/
|
|
49
49
|
export type { TrustClass };
|
|
50
50
|
|
|
@@ -107,6 +107,7 @@ import {
|
|
|
107
107
|
persistWakeTriggerMessage,
|
|
108
108
|
scopeWakeAllowedTools,
|
|
109
109
|
} from "../daemon/wake-conversation-ops.js";
|
|
110
|
+
import { desktopAutomationLease } from "../desktop/desktop-automation-lease.js";
|
|
110
111
|
import {
|
|
111
112
|
recordCompactionEndBestEffort,
|
|
112
113
|
recordCompactionStartBestEffort,
|
|
@@ -118,7 +119,7 @@ import {
|
|
|
118
119
|
setAgentLoopExitReasonOnLatestLog,
|
|
119
120
|
} from "../persistence/llm-request-log-store.js";
|
|
120
121
|
import type { SystemPromptPersonaOverride } from "../prompts/system-prompt.js";
|
|
121
|
-
import type { Message } from "../providers/types.js";
|
|
122
|
+
import type { Message, ToolDefinition } from "../providers/types.js";
|
|
122
123
|
import {
|
|
123
124
|
type UntrustedContentSource,
|
|
124
125
|
wrapUntrustedContent,
|
|
@@ -350,6 +351,27 @@ export interface WakeOptions {
|
|
|
350
351
|
* skill-management authoring tools callable directly.
|
|
351
352
|
*/
|
|
352
353
|
preactivateSkillIds?: readonly string[];
|
|
354
|
+
/**
|
|
355
|
+
* Tool definitions to send verbatim as the wake's wire tool array, in place
|
|
356
|
+
* of the ones the conversation would resolve for itself. Used by fork-based
|
|
357
|
+
* memory retrospectives to replay the SOURCE conversation's recorded surface
|
|
358
|
+
* (`getRecordedConversationToolSurface`) so the provider prompt-cache prefix matches
|
|
359
|
+
* the source's live turns byte for byte. Definitions only: what may execute
|
|
360
|
+
* is still decided by `allowedTools` and the turn's own active set. Applied
|
|
361
|
+
* and restored alongside `allowedTools`; ignored when `allowedTools` is
|
|
362
|
+
* absent.
|
|
363
|
+
*/
|
|
364
|
+
wireToolDefinitions?: readonly ToolDefinition[];
|
|
365
|
+
/**
|
|
366
|
+
* Whether the source's recorded surface rendered the system prompt's
|
|
367
|
+
* parallel-delegation section, replayed into the wake's prompt so that
|
|
368
|
+
* tier of the provider cache prefix matches the source's too (the wake's
|
|
369
|
+
* own scope cannot spawn, so deriving it would render the section off where
|
|
370
|
+
* an interactive source rendered it on). Prompt only: a spawn attempt is
|
|
371
|
+
* still rejected at execution. Applied and restored alongside
|
|
372
|
+
* `allowedTools`; ignored when `allowedTools` is absent.
|
|
373
|
+
*/
|
|
374
|
+
delegateIndependentTasks?: boolean;
|
|
353
375
|
/**
|
|
354
376
|
* Explicit persona/channel slugs for the wake's system-prompt build,
|
|
355
377
|
* applied to the conversation for the duration of the run and restored
|
|
@@ -1090,7 +1112,7 @@ export async function wakeAgentForOpportunity(
|
|
|
1090
1112
|
try {
|
|
1091
1113
|
recordRequestLog(
|
|
1092
1114
|
conversationId,
|
|
1093
|
-
JSON.stringify(record.rawRequest),
|
|
1115
|
+
JSON.stringify(record.rawRequest ?? null),
|
|
1094
1116
|
JSON.stringify(record.rawResponse),
|
|
1095
1117
|
undefined,
|
|
1096
1118
|
record.provider,
|
|
@@ -1410,6 +1432,7 @@ export async function wakeAgentForOpportunity(
|
|
|
1410
1432
|
* is one function rather than a rebuild bolted onto either half.
|
|
1411
1433
|
*/
|
|
1412
1434
|
const restoreWakeTurnScope = (): void => {
|
|
1435
|
+
desktopAutomationLease.releaseForConversation(conversationId);
|
|
1413
1436
|
restoreWakeAllowedTools();
|
|
1414
1437
|
clearWakePersonaOverride();
|
|
1415
1438
|
syncWakeLoopSystemPrompt();
|
|
@@ -1422,9 +1445,13 @@ export async function wakeAgentForOpportunity(
|
|
|
1422
1445
|
restoreWakeToolScope = scopeWakeAllowedTools(
|
|
1423
1446
|
conversation,
|
|
1424
1447
|
new Set(opts.allowedTools),
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
|
|
1448
|
+
{
|
|
1449
|
+
gateMode: opts.toolGateMode,
|
|
1450
|
+
toolContextPin: opts.toolContextPin,
|
|
1451
|
+
preactivateSkillIds: opts.preactivateSkillIds,
|
|
1452
|
+
wireToolDefinitions: opts.wireToolDefinitions,
|
|
1453
|
+
delegateIndependentTasks: opts.delegateIndependentTasks,
|
|
1454
|
+
},
|
|
1428
1455
|
);
|
|
1429
1456
|
return true;
|
|
1430
1457
|
} catch (err) {
|