@vellumai/assistant 0.12.2 → 0.12.3-staging.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +2 -0
- package/AGENTS.md +1 -1
- package/ARCHITECTURE.md +43 -29
- package/Dockerfile +63 -0
- package/docs/architecture/integrations.md +7 -0
- package/docs/architecture/memory.md +17 -2
- package/docs/architecture/security.md +10 -5
- package/docs/architecture/turn-actor.md +2 -0
- package/docs/credential-execution-service.md +1 -1
- package/docs/desktop-browser-cli.md +50 -5
- package/docs/live-voice-task-outcomes.md +100 -0
- package/docs/mode-sessions.md +183 -0
- package/docs/voice-input-diagnostics.md +58 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/package.json +2 -1
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +1 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/secret-detection.ts +4 -1
- package/node_modules/@vellumai/environments/src/shell.test.ts +21 -0
- package/node_modules/@vellumai/environments/src/shell.ts +24 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/package.json +2 -1
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +1 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/secret-detection.ts +4 -1
- package/node_modules/@vellumai/gateway-client/src/gateway-ipc-contracts.ts +20 -0
- package/node_modules/@vellumai/gateway-client/src/inbound-contract.ts +8 -2
- package/node_modules/@vellumai/gateway-client/src/index.ts +4 -1
- package/node_modules/@vellumai/gateway-client/src/outbound-contract.ts +38 -7
- package/node_modules/@vellumai/gateway-client/src/trust-verdict-contract.ts +5 -6
- package/node_modules/@vellumai/service-contracts/package.json +2 -1
- package/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +1 -0
- package/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
- package/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
- package/node_modules/@vellumai/service-contracts/src/secret-detection.ts +4 -1
- package/node_modules/@vellumai/slack-text/src/index.ts +4 -8
- package/openapi.yaml +716 -16
- package/package.json +1 -1
- package/scripts/smoke-desktop-browser-cli.ts +11 -26
- package/scripts/smoke-desktop-window-theme.ts +2 -3
- package/scripts/sync-llm-catalog.ts +3 -0
- package/scripts/voice-ttft-spike.ts +2 -2
- package/src/__tests__/agent-loop-output-hooks.test.ts +150 -1
- package/src/__tests__/agent-loop-provider-error-recording.test.ts +87 -19
- package/src/__tests__/agent-loop.test.ts +259 -0
- package/src/__tests__/agent-wake-delegation-prompt.test.ts +64 -2
- package/src/__tests__/annotate-activity-metadata.test.ts +3 -1
- package/src/__tests__/annotate-risk-options.test.ts +2 -0
- package/src/__tests__/answered-question-persistence.test.ts +2 -0
- package/src/__tests__/anthropic-gateway-shared.test.ts +22 -0
- package/src/__tests__/anthropic-provider.test.ts +55 -0
- package/src/__tests__/approval-interception-trust-gates.test.ts +40 -0
- package/src/__tests__/approval-message-composer.test.ts +1 -118
- package/src/__tests__/approval-routes-http.test.ts +3 -0
- package/src/__tests__/assistant-event-hub-connection-history.test.ts +111 -0
- package/src/__tests__/assistant-event-hub-targeted.test.ts +50 -0
- package/src/__tests__/assistant-event-targeting.test.ts +186 -0
- package/src/__tests__/assistant-stream-state.test.ts +29 -9
- package/src/__tests__/attachments-store.test.ts +22 -3
- package/src/__tests__/browser-fill-credential.test.ts +11 -22
- package/src/__tests__/btw-routes.test.ts +4 -1
- package/src/__tests__/build-persisted-content.test.ts +1 -1
- package/src/__tests__/call-conversation-messages.test.ts +5 -0
- package/src/__tests__/call-recovery.test.ts +7 -2
- package/src/__tests__/call-routes-http.test.ts +12 -0
- package/src/__tests__/call-store.test.ts +7 -2
- package/src/__tests__/channel-approval.test.ts +9 -14
- package/src/__tests__/channel-reply-delivery.test.ts +56 -0
- package/src/__tests__/channel-retry-sweep.test.ts +9 -0
- package/src/__tests__/chat-credential-redaction.test.ts +23 -0
- package/src/__tests__/chat-reveal-guard-priming.test.ts +2 -0
- package/src/__tests__/checker.test.ts +11 -3
- package/src/__tests__/client-os-metadata-persistence.test.ts +35 -1
- package/src/__tests__/computer-use-screenshot-attachments.test.ts +411 -0
- package/src/__tests__/computer-use-screenshot-selection.test.ts +50 -0
- package/src/__tests__/computer-use-skill-manifest-regression.test.ts +0 -25
- package/src/__tests__/computer-use-tools.test.ts +1 -233
- package/src/__tests__/config-model-image-provider.test.ts +39 -0
- package/src/__tests__/config-schema.test.ts +16 -0
- package/src/__tests__/config-watcher.test.ts +24 -0
- package/src/__tests__/context-search-agent-runner.test.ts +260 -1
- package/src/__tests__/context-search-fanout.test.ts +1 -1
- package/src/__tests__/context-search-types.test.ts +7 -7
- package/src/__tests__/conversation-agent-loop-disk-pressure.test.ts +8 -2
- package/src/__tests__/conversation-agent-loop-fatal-cleanup.test.ts +2 -0
- package/src/__tests__/conversation-agent-loop-inference-profile.test.ts +2 -0
- package/src/__tests__/conversation-agent-loop-overflow.test.ts +2 -0
- package/src/__tests__/conversation-agent-loop.test.ts +797 -16
- package/src/__tests__/conversation-app-control-lifecycle.test.ts +53 -1
- package/src/__tests__/conversation-attachments.test.ts +106 -0
- package/src/__tests__/conversation-evictor.test.ts +62 -37
- package/src/__tests__/conversation-fork-crud.test.ts +191 -0
- package/src/__tests__/conversation-history-boundary-repair-failure.test.ts +174 -0
- package/src/__tests__/conversation-loop-db-lock-resilience.test.ts +2 -0
- package/src/__tests__/conversation-process-app-control-preactivation.test.ts +2 -0
- package/src/__tests__/conversation-rendered-delegation-state.test.ts +276 -0
- package/src/__tests__/conversation-retry-route.test.ts +22 -0
- package/src/__tests__/conversation-routes-disk-view.test.ts +2 -0
- package/src/__tests__/conversation-routes-enabled-plugins.test.ts +2 -0
- package/src/__tests__/conversation-routes-guardian-reply.test.ts +10 -0
- package/src/__tests__/conversation-routes-hidden-queue.test.ts +2 -0
- package/src/__tests__/conversation-routes-slash-commands.test.ts +2 -0
- package/src/__tests__/conversation-runtime-assembly.test.ts +29 -0
- package/src/__tests__/conversation-surface-snapshots.test.ts +543 -0
- package/src/__tests__/conversation-surfaces-action-delivery.test.ts +151 -0
- package/src/__tests__/conversation-surfaces-point-at-budget.test.ts +63 -1
- package/src/__tests__/conversation-tool-setup-attribution.test.ts +23 -0
- package/src/__tests__/conversation-undo.test.ts +61 -0
- package/src/__tests__/credential-execution-client.test.ts +72 -55
- package/src/__tests__/credential-prompt-route.test.ts +35 -3
- package/src/__tests__/cu-unified-flow.test.ts +82 -1
- package/src/__tests__/db-conversation-tool-surface.test.ts +221 -0
- package/src/__tests__/db-mode-sessions-migration.test.ts +175 -0
- package/src/__tests__/deterministic-verification-control-plane.test.ts +2 -0
- package/src/__tests__/dm-backfill.test.ts +91 -1
- package/src/__tests__/document-create-route.test.ts +88 -0
- package/src/__tests__/document-sync-tags.test.ts +27 -0
- package/src/__tests__/emit-signal-routing-intent.test.ts +142 -2
- package/src/__tests__/events-tail-route.test.ts +58 -0
- package/src/__tests__/evict-conversations-for-reload.test.ts +57 -8
- package/src/__tests__/guardian-action-sweep.test.ts +16 -0
- package/src/__tests__/guardian-dispatch.test.ts +13 -0
- package/src/__tests__/headless-browser-interactions.test.ts +64 -20
- package/src/__tests__/heartbeat-service.test.ts +2 -59
- package/src/__tests__/helpers/mock-conversation.ts +21 -0
- package/src/__tests__/helpers/native-web-search-harness.ts +2 -0
- package/src/__tests__/helpers/verification-sessions-ipc-sim.ts +5 -0
- package/src/__tests__/history-repair.test.ts +53 -0
- package/src/__tests__/host-cu-proxy.test.ts +46 -2
- package/src/__tests__/http-user-message-parity.test.ts +2 -0
- package/src/__tests__/image-credentials.test.ts +49 -0
- package/src/__tests__/image-service-dispatcher.test.ts +45 -0
- package/src/__tests__/inference-profile-session-handler.test.ts +26 -0
- package/src/__tests__/interrupt-on-send.test.ts +57 -43
- package/src/__tests__/interrupt-turn-note.test.ts +5 -6
- package/src/__tests__/list-messages-tool-merge.test.ts +85 -1
- package/src/__tests__/llm-catalog-parity.test.ts +27 -4
- package/src/__tests__/media-generate-image.test.ts +49 -1
- package/src/__tests__/message-provenance-contact-id.test.ts +69 -0
- package/src/__tests__/migration-export-to-gcs.test.ts +142 -0
- package/src/__tests__/mode-session-history.test.ts +245 -0
- package/src/__tests__/mode-session-recovery.test.ts +222 -0
- package/src/__tests__/mode-session-wire.test.ts +141 -0
- package/src/__tests__/notification-guardian-path.test.ts +7 -0
- package/src/__tests__/notification-telegram-adapter.test.ts +2 -2
- package/src/__tests__/notification-vellum-adapter.test.ts +34 -0
- package/src/__tests__/oauth-apps-routes.test.ts +1 -0
- package/src/__tests__/oauth-commands-routes.test.ts +114 -101
- package/src/__tests__/oauth-connect-orchestrator.test.ts +2 -0
- package/src/__tests__/oauth-provider-serializer.test.ts +13 -0
- package/src/__tests__/oauth-providers-routes.test.ts +3 -0
- package/src/__tests__/outbound-slack-persistence.test.ts +2 -0
- package/src/__tests__/persist-media-references.test.ts +50 -0
- package/src/__tests__/persistence-secret-redaction.test.ts +2 -0
- package/src/__tests__/plugin-api-provider.test.ts +5 -0
- package/src/__tests__/plugin-api-resolve-credential.test.ts +97 -0
- package/src/__tests__/plugin-import-boundary-guard.test.ts +9 -1
- package/src/__tests__/process-message-display-content.test.ts +56 -0
- package/src/__tests__/reaction-persistence.test.ts +5 -0
- package/src/__tests__/request-secret-standalone-channel-gate.test.ts +25 -9
- package/src/__tests__/run-conversation-turn-persistence.test.ts +138 -1
- package/src/__tests__/runtime-events-sse-reconnect.test.ts +1 -1
- package/src/__tests__/schedule-routes.test.ts +20 -0
- package/src/__tests__/scheduler-result-notification.test.ts +23 -4
- package/src/__tests__/script-proxy-certs.test.ts +1 -1
- package/src/__tests__/scripted-turn-metadata-persistence.test.ts +73 -0
- package/src/__tests__/secret-ingress-http.test.ts +2 -0
- package/src/__tests__/secret-routes-platform-proxy.test.ts +86 -31
- package/src/__tests__/secret-routes-scrub.test.ts +22 -27
- package/src/__tests__/secret-scanner.test.ts +20 -0
- package/src/__tests__/secure-keys-managed-failover.test.ts +8 -0
- package/src/__tests__/secure-keys.test.ts +50 -4
- package/src/__tests__/send-endpoint-busy.test.ts +163 -1
- package/src/__tests__/server-history-render.test.ts +91 -1
- package/src/__tests__/server-tool-pairing.test.ts +107 -0
- package/src/__tests__/settings-routes.test.ts +50 -1
- package/src/__tests__/skill-script-runner-sandbox.test.ts +32 -0
- package/src/__tests__/skills.test.ts +5 -4
- package/src/__tests__/subagent-fork-notifications.test.ts +1 -2
- package/src/__tests__/subagent-manager-notify.test.ts +1 -2
- package/src/__tests__/subagent-notify-parent.test.ts +146 -39
- package/src/__tests__/subagent-notify-stale-rebuild.test.ts +2 -4
- package/src/__tests__/subagent-tool-gate-mode.test.ts +226 -0
- package/src/__tests__/surface-completion-in-flight-snapshot.test.ts +2 -0
- package/src/__tests__/terminal-tools.test.ts +8 -0
- package/src/__tests__/thread-backfill.test.ts +53 -0
- package/src/__tests__/tool-preview-lifecycle.test.ts +2 -0
- package/src/__tests__/tool-result-follow-up.test.ts +99 -0
- package/src/__tests__/tool-result-metadata-plumbing.test.ts +66 -1
- package/src/__tests__/tool-start-timestamp.test.ts +2 -0
- package/src/__tests__/twilio-routes.test.ts +1 -0
- package/src/__tests__/unicode.test.ts +36 -0
- package/src/__tests__/web-search-backend-failure.test.ts +1 -1
- package/src/activation/progress-store.test.ts +12 -7
- package/src/agent/loop.ts +136 -53
- package/src/agent/tool-result-follow-up.ts +70 -0
- package/src/api/attachment-provenance.test.ts +68 -0
- package/src/api/computer-use-tool.test.ts +55 -0
- package/src/api/computer-use-tool.ts +27 -0
- package/src/api/events/assistant-outbound-attachment.ts +3 -0
- package/src/api/events/assistant-turn-start.ts +3 -0
- package/src/api/events/desktop-activity-changed.ts +10 -0
- package/src/api/events/generation-handoff.ts +2 -0
- package/src/api/events/message-complete.ts +2 -0
- package/src/api/events/notification-conversation-created.ts +14 -16
- package/src/api/events/notification-intent.ts +4 -2
- package/src/api/events/question-request.ts +1 -0
- package/src/api/events/tool-result.ts +85 -0
- package/src/api/events/tool-use-start.ts +3 -0
- package/src/api/events/user-message-echo.ts +3 -0
- package/src/api/index.ts +42 -6
- package/src/api/mode-session.test.ts +131 -0
- package/src/api/mode-session.ts +148 -0
- package/src/api/remember-tool.ts +15 -0
- package/src/api/responses/conversation-message.ts +9 -0
- package/src/approvals/AGENTS.md +4 -2
- package/src/approvals/approval-primitive.ts +5 -2
- package/src/approvals/scoped-approval-grants.ts +6 -2
- package/src/browser/__tests__/operations.test.ts +29 -0
- package/src/browser/operation-meta.ts +2 -1
- package/src/browser/operations.ts +29 -0
- package/src/browser/virtual-desktop-target.ts +2 -9
- package/src/bundler/bundle-signer.ts +1 -3
- package/src/calls/__tests__/barge-in-guard.test.ts +3 -0
- package/src/calls/__tests__/call-controller.test.ts +94 -0
- package/src/calls/__tests__/call-funnel-telemetry.test.ts +204 -0
- package/src/calls/__tests__/voice-control-protocol.test.ts +15 -0
- package/src/calls/__tests__/voice-escalation-judge.test.ts +209 -0
- package/src/calls/__tests__/voice-leg-coordinator.test.ts +168 -0
- package/src/{live-voice/__tests__/live-voice-metrics.test.ts → calls/__tests__/voice-metrics.test.ts} +32 -38
- package/src/calls/__tests__/voice-session-bridge.test.ts +576 -122
- package/src/calls/barge-in-guard.ts +5 -0
- package/src/calls/call-controller.ts +196 -1
- package/src/calls/call-domain.ts +4 -0
- package/src/calls/call-store.ts +87 -2
- package/src/calls/guardian-action-sweep.ts +22 -8
- package/src/calls/media-stream-audio-transcode.ts +1 -3
- package/src/calls/typesafe-noul.ts +156 -0
- package/src/calls/voice-control-protocol.ts +6 -0
- package/src/calls/voice-escalation-judge.ts +145 -0
- package/src/calls/voice-escalation-target.ts +10 -0
- package/src/calls/voice-leg-coordinator.ts +121 -3
- package/src/{live-voice/live-voice-metrics.ts → calls/voice-metrics.ts} +94 -87
- package/src/calls/voice-session-bridge.ts +256 -112
- package/src/calls/voice-triage-escalate.ts +7 -0
- package/src/cli/__tests__/roadmap-cli.test.ts +4 -41
- package/src/cli/commands/__tests__/cli-test-harness.ts +21 -3
- package/src/cli/commands/__tests__/notifications.test.ts +35 -10
- package/src/cli/commands/__tests__/plugins.test.ts +20 -1
- package/src/cli/commands/__tests__/schedules.test.ts +14 -0
- package/src/cli/commands/bash.help.ts +4 -3
- package/src/cli/commands/browser.help.ts +9 -6
- package/src/cli/commands/browser.ts +0 -1
- package/src/cli/commands/clients.help.ts +46 -0
- package/src/cli/commands/clients.ts +116 -0
- package/src/cli/commands/credentials.help.ts +9 -4
- package/src/cli/commands/notifications.help.ts +16 -2
- package/src/cli/commands/notifications.ts +30 -8
- package/src/cli/commands/oauth/index.help.ts +10 -0
- package/src/cli/commands/oauth/providers.ts +12 -0
- package/src/cli/commands/oauth/request.test.ts +179 -2
- package/src/cli/commands/oauth/request.ts +41 -10
- package/src/cli/commands/plugins.ts +12 -5
- package/src/cli/commands/roadmap.help.ts +18 -21
- package/src/cli/commands/roadmap.ts +4 -21
- package/src/cli/commands/schedules.ts +2 -0
- package/src/cli/lib/__tests__/inspect-plugin.test.ts +54 -0
- package/src/cli/lib/__tests__/install-from-github.test.ts +41 -0
- package/src/cli/lib/__tests__/list-installed-plugins.test.ts +5 -2
- package/src/cli/lib/__tests__/local-plugin-upgrade.test.ts +133 -51
- package/src/cli/lib/__tests__/plugin-catalog-cache.test.ts +8 -2
- package/src/cli/lib/bundled-marketplace.json +906 -1
- package/src/cli/lib/bundled-plugin-packages.json +783 -1
- package/src/cli/lib/inspect-plugin.ts +11 -4
- package/src/cli/lib/plugin-marketplace.ts +8 -0
- package/src/cli/lib/publish-plugin.ts +1 -2
- package/src/cli/lib/upgrade-plugin.ts +19 -11
- package/src/cli/output.ts +18 -4
- package/src/config/__tests__/default-provider.test.ts +6 -0
- package/src/config/__tests__/memory-retrospective-schema.test.ts +14 -1
- package/src/config/bundled-skills/computer-use/SKILL.md +1 -1
- package/src/config/bundled-skills/computer-use/TOOLS.json +89 -1
- package/src/config/bundled-skills/image-studio/SKILL.md +17 -2
- package/src/config/bundled-skills/image-studio/tools/media-generate-image.ts +7 -11
- package/src/config/bundled-skills/schedule/SKILL.md +6 -6
- package/src/config/bundled-skills/sequences/TOOLS.json +1 -5
- package/src/config/call-site-defaults.ts +9 -0
- package/src/config/feature-flag-registry.json +16 -0
- package/src/config/loader.ts +9 -17
- package/src/config/platform-identity.ts +8 -8
- package/src/config/profile-text-generation.test.ts +51 -0
- package/src/config/profile-text-generation.ts +51 -0
- package/src/config/schemas/__tests__/memory-v3.test.ts +12 -0
- package/src/config/schemas/__tests__/tools.test.ts +31 -0
- package/src/config/schemas/call-site-catalog.ts +14 -0
- package/src/config/schemas/channels.ts +0 -18
- package/src/config/schemas/llm.ts +9 -2
- package/src/config/schemas/mcp.ts +6 -1
- package/src/config/schemas/memory-retrospective.ts +11 -2
- package/src/config/schemas/memory-v3.ts +12 -0
- package/src/config/schemas/services.ts +15 -6
- package/src/config/schemas/tools.ts +10 -0
- package/src/config/session-groups-gate.ts +21 -0
- package/src/config/skills.ts +2 -6
- package/src/conversations/__tests__/message-consolidation.test.ts +149 -0
- package/src/conversations/message-consolidation.ts +102 -5
- package/src/credential-execution/ces-connect.test.ts +69 -0
- package/src/credential-execution/ces-connect.ts +115 -0
- package/src/credential-execution/ces-runtime.ts +55 -93
- package/src/credential-execution/executable-discovery.ts +34 -142
- package/src/credential-execution/process-manager.test.ts +20 -24
- package/src/credential-execution/process-manager.ts +24 -19
- package/src/daemon/__tests__/conversation-surfaces-launch.test.ts +10 -1
- package/src/daemon/__tests__/conversation-tool-setup.test.ts +43 -0
- package/src/daemon/__tests__/inference-profile-notification.test.ts +2 -0
- package/src/daemon/__tests__/turn-tail-deleted-conversation.test.ts +36 -0
- package/src/daemon/assistant-attachments.ts +20 -12
- package/src/daemon/browser-mode-session.test.ts +402 -0
- package/src/daemon/browser-mode-session.ts +186 -0
- package/src/daemon/chat-credential-redaction.ts +16 -1
- package/src/daemon/computer-use-mode-session.test.ts +210 -0
- package/src/daemon/computer-use-mode-session.ts +105 -0
- package/src/daemon/config-watcher.ts +10 -0
- package/src/daemon/conversation-agent-loop-handlers.ts +132 -13
- package/src/daemon/conversation-agent-loop.ts +231 -25
- package/src/daemon/conversation-attachments.ts +93 -12
- package/src/daemon/conversation-evictor.ts +5 -24
- package/src/daemon/conversation-history.ts +10 -0
- package/src/daemon/conversation-interrupt-repair.ts +4 -43
- package/src/daemon/conversation-interrupt.ts +7 -11
- package/src/daemon/conversation-messaging.ts +58 -5
- package/src/daemon/conversation-mode-session.test.ts +1667 -0
- package/src/daemon/conversation-mode-session.ts +1008 -0
- package/src/daemon/conversation-process.ts +5 -0
- package/src/daemon/conversation-runtime-assembly.ts +10 -20
- package/src/daemon/conversation-skill-tools.ts +1 -0
- package/src/daemon/conversation-store.ts +4 -4
- package/src/daemon/conversation-surface-snapshots.ts +135 -0
- package/src/daemon/conversation-surfaces.ts +113 -5
- package/src/daemon/conversation-tool-setup.ts +79 -6
- package/src/daemon/conversation-turn-finalize.ts +26 -13
- package/src/daemon/conversation.ts +215 -58
- package/src/daemon/daemon-control.ts +3 -7
- package/src/daemon/handlers/config-model.ts +7 -5
- package/src/daemon/handlers/shared.ts +55 -16
- package/src/daemon/host-cu-proxy.ts +51 -22
- package/src/daemon/lifecycle.ts +30 -13
- package/src/daemon/mcp-reload-service.ts +1 -1
- package/src/daemon/message-protocol.ts +0 -1
- package/src/daemon/message-provenance.ts +28 -4
- package/src/daemon/message-types/shared.ts +1 -0
- package/src/daemon/mode-session-startup-recovery.ts +23 -0
- package/src/daemon/mode-session-tracking.ts +54 -0
- package/src/daemon/orphan-reaper.ts +4 -3
- package/src/daemon/persist-media-references.ts +20 -3
- package/src/daemon/process-message.ts +13 -4
- package/src/daemon/tool-setup-types.ts +6 -0
- package/src/daemon/virtual-desktop-context.ts +21 -0
- package/src/daemon/wake-conversation-ops.ts +50 -15
- package/src/desktop/__tests__/fake-desktop.ts +1 -0
- package/src/desktop/desktop-accessibility-script.ts +88 -0
- package/src/desktop/desktop-accessibility.test.ts +160 -0
- package/src/desktop/desktop-accessibility.ts +169 -0
- package/src/desktop/desktop-automation-lease.test.ts +302 -64
- package/src/desktop/desktop-automation-lease.ts +129 -54
- package/src/desktop/desktop-browser-endpoint.ts +4 -0
- package/src/desktop/desktop-computer-use-routing.test.ts +330 -0
- package/src/desktop/desktop-computer-use.test.ts +449 -0
- package/src/desktop/desktop-computer-use.ts +502 -0
- package/src/desktop/desktop-dependencies.ts +47 -293
- package/src/desktop/desktop-display.ts +2 -2
- package/src/desktop/desktop-help.ts +15 -0
- package/src/desktop/desktop-panel-config.test.ts +13 -3
- package/src/desktop/desktop-panel-config.ts +109 -15
- package/src/desktop/desktop-session-manager.test.ts +69 -21
- package/src/desktop/desktop-session-manager.ts +76 -22
- package/src/desktop/desktop-stream-bridge.test.ts +76 -132
- package/src/desktop/desktop-stream-bridge.ts +9 -11
- package/src/desktop/desktop-wallpaper-renderer.ts +132 -0
- package/src/desktop/desktop-wallpaper-worker.ts +17 -0
- package/src/desktop/desktop-wallpaper.test.ts +48 -7
- package/src/desktop/desktop-wallpaper.ts +34 -102
- package/src/desktop/desktop-window-theme.ts +5 -25
- package/src/desktop/virtual-desktop-feature.ts +53 -3
- package/src/desktop/virtual-desktop-platform.test.ts +5 -9
- package/src/documents/document-store.ts +23 -0
- package/src/heartbeat/__tests__/heartbeat-service.test.ts +22 -26
- package/src/heartbeat/heartbeat-service.ts +5 -33
- package/src/ipc/__tests__/browser-ipc.test.ts +215 -6
- package/src/ipc/assistant-server.ts +2 -0
- package/src/ipc/routes/__tests__/channel-reply-ipc-routes.test.ts +111 -0
- package/src/ipc/routes/channel-reply-ipc-routes.ts +73 -0
- package/src/live-voice/__tests__/continuation-judge.test.ts +151 -0
- package/src/live-voice/__tests__/live-activity-reporter.test.ts +154 -15
- package/src/live-voice/__tests__/live-voice-flux-turn-end.test.ts +743 -31
- package/src/live-voice/__tests__/live-voice-look-follow-up.test.ts +191 -10
- package/src/live-voice/__tests__/live-voice-session-groups-gate.test.ts +87 -0
- package/src/live-voice/__tests__/live-voice-session-manager.test.ts +5 -1
- package/src/live-voice/__tests__/live-voice-sight-frame-echo.test.ts +41 -0
- package/src/live-voice/__tests__/live-voice-sight-frame.test.ts +1017 -9
- package/src/live-voice/__tests__/live-voice-triage-escalate.test.ts +224 -1
- package/src/live-voice/__tests__/live-voice-tts-session.test.ts +101 -0
- package/src/live-voice/__tests__/live-voice-tts.test.ts +44 -0
- package/src/live-voice/__tests__/live-voice-vad.test.ts +939 -75
- package/src/live-voice/__tests__/microphone-idle-gate.test.ts +87 -0
- package/src/live-voice/__tests__/session-controls.test.ts +23 -0
- package/src/live-voice/__tests__/task-outcomes.test.ts +117 -0
- package/src/live-voice/__tests__/voice-input-diagnostics.test.ts +117 -0
- package/src/live-voice/camera-mode-session.test.ts +265 -0
- package/src/live-voice/camera-mode-session.ts +277 -0
- package/src/live-voice/continuation-judge.ts +123 -0
- package/src/live-voice/live-activity-reporter.ts +115 -6
- package/src/live-voice/live-voice-manager.ts +36 -1
- package/src/live-voice/live-voice-photo.ts +88 -7
- package/src/live-voice/live-voice-session-manager.ts +15 -0
- package/src/live-voice/live-voice-session.ts +1443 -747
- package/src/live-voice/live-voice-tts.ts +22 -4
- package/src/live-voice/microphone-idle-gate.ts +56 -0
- package/src/live-voice/protocol.ts +171 -4
- package/src/live-voice/session-controls.ts +11 -3
- package/src/live-voice/task-outcomes.ts +131 -0
- package/src/live-voice/voice-input-diagnostics.ts +117 -0
- package/src/mcp/__tests__/manager-tool-caps.test.ts +156 -0
- package/src/mcp/__tests__/startup.test.ts +30 -7
- package/src/mcp/__tests__/tool-caps.test.ts +134 -0
- package/src/mcp/manager.ts +171 -101
- package/src/mcp/startup.ts +33 -13
- package/src/mcp/tool-caps.ts +184 -0
- package/src/media/__tests__/image-models.test.ts +58 -0
- package/src/media/image-credentials.ts +5 -0
- package/src/media/image-models.ts +52 -0
- package/src/media/image-service.ts +8 -0
- package/src/media/openrouter-image-service.test.ts +133 -0
- package/src/media/openrouter-image-service.ts +153 -0
- package/src/media/types.ts +1 -1
- package/src/messaging/provider-message-metadata.ts +3 -3
- package/src/messaging/providers/__tests__/transport-dispatch.test.ts +65 -11
- package/src/messaging/providers/discord/send.ts +4 -17
- package/src/messaging/providers/discord/transport.ts +2 -2
- package/src/messaging/providers/outlook/client.ts +16 -22
- package/src/messaging/providers/telegram-bot/send.test.ts +236 -1
- package/src/messaging/providers/telegram-bot/send.ts +10 -7
- package/src/messaging/providers/telegram-bot/transport.ts +3 -10
- package/src/messaging/providers/whatsapp/send.test.ts +169 -4
- package/src/monitoring/__tests__/control.test.ts +17 -5
- package/src/monitoring/control.ts +15 -5
- package/src/notifications/AGENTS.md +1 -1
- package/src/notifications/README.md +83 -99
- package/src/notifications/__tests__/broadcaster.test.ts +1 -1
- package/src/notifications/__tests__/channel-allowlist.test.ts +39 -0
- package/src/notifications/__tests__/copy-composer.test.ts +71 -1
- package/src/notifications/__tests__/decision-engine.test.ts +79 -2
- package/src/notifications/__tests__/deterministic-checks.test.ts +10 -10
- package/src/notifications/__tests__/home-feed-side-effect.test.ts +93 -4
- package/src/notifications/__tests__/notification-utils.test.ts +18 -0
- package/src/notifications/adapters/macos.ts +37 -46
- package/src/notifications/broadcaster.ts +5 -6
- package/src/notifications/channel-allowlist.ts +44 -0
- package/src/notifications/copy-composer.ts +11 -26
- package/src/notifications/decision-engine.ts +24 -14
- package/src/notifications/deterministic-checks.ts +1 -1
- package/src/notifications/emit-signal.ts +123 -37
- package/src/notifications/home-feed-side-effect.ts +9 -7
- package/src/notifications/notification-utils.ts +19 -0
- package/src/notifications/schedule-result-producer.ts +4 -6
- package/src/notifications/signal.ts +4 -19
- package/src/notifications/types.ts +15 -0
- package/src/oauth/AGENTS.md +2 -0
- package/src/oauth/__tests__/identity-verifier.test.ts +36 -1
- package/src/oauth/connection-resolver.ts +1 -2
- package/src/oauth/identity-verifier.ts +24 -0
- package/src/oauth/oauth-store.ts +12 -4
- package/src/oauth/provider-serializer.ts +8 -0
- package/src/oauth/seed-providers.ts +42 -0
- package/src/onboarding/onboarding-events-store.ts +99 -12
- package/src/permissions/confirmation-guardian-request.test.ts +16 -0
- package/src/permissions/confirmation-guardian-request.ts +2 -3
- package/src/permissions/question-prompter.test.ts +31 -0
- package/src/permissions/question-prompter.ts +2 -0
- package/src/persistence/__tests__/client-connection-events-store.test.ts +322 -0
- package/src/persistence/attachments-store.ts +84 -44
- package/src/persistence/client-connection-events-store.ts +377 -0
- package/src/persistence/conversation-crud.ts +166 -26
- package/src/persistence/conversation-mode-sessions.test.ts +424 -0
- package/src/persistence/conversation-mode-sessions.ts +460 -0
- package/src/persistence/conversation-plugin-facade.ts +30 -0
- package/src/persistence/conversation-queries.ts +6 -19
- package/src/persistence/conversation-tool-surface.ts +135 -0
- package/src/persistence/conversation-types.test.ts +32 -0
- package/src/persistence/conversation-types.ts +26 -4
- package/src/persistence/llm-usage-store.ts +3 -9
- package/src/persistence/message-metadata.ts +25 -0
- package/src/persistence/migrations/378-create-conversation-tool-surfaces.test.ts +78 -0
- package/src/persistence/migrations/378-create-conversation-tool-surfaces.ts +29 -0
- package/src/persistence/migrations/379-oauth-providers-response-ok-field.test.ts +92 -0
- package/src/persistence/migrations/379-oauth-providers-response-ok-field.ts +32 -0
- package/src/persistence/migrations/380-conversation-tool-surfaces-delegate-independent-tasks.test.ts +90 -0
- package/src/persistence/migrations/380-conversation-tool-surfaces-delegate-independent-tasks.ts +34 -0
- package/src/persistence/migrations/381-create-conversation-mode-sessions.ts +41 -0
- package/src/persistence/migrations/382-create-client-connection-events.ts +40 -0
- package/src/persistence/migrations/__tests__/382-create-client-connection-events.test.ts +56 -0
- package/src/persistence/schema/conversation-tool-surfaces.ts +32 -0
- package/src/persistence/schema/conversations.ts +67 -0
- package/src/persistence/schema/index.ts +1 -0
- package/src/persistence/schema/infrastructure.ts +29 -0
- package/src/persistence/schema/oauth.ts +1 -0
- package/src/persistence/steps.ts +41 -3
- package/src/plugin-api/__tests__/plugin-name-env.test.ts +249 -0
- package/src/plugin-api/conversation-turn.ts +31 -7
- package/src/plugin-api/effective-context-window.ts +33 -0
- package/src/plugin-api/index.ts +35 -6
- package/src/plugin-api/plugin-channel-turn-trust.test.ts +133 -0
- package/src/plugin-api/plugin-channel-turn-trust.ts +71 -0
- package/src/plugin-api/plugin-name-env.ts +125 -0
- package/src/plugin-api/resolve-credential.ts +79 -20
- package/src/plugin-api/vision-support.test.ts +22 -0
- package/src/plugin-api/vision-support.ts +17 -5
- package/src/plugins/__tests__/active-plugin-env.test.ts +136 -0
- package/src/plugins/active-plugin-env.ts +51 -0
- package/src/plugins/defaults/index.ts +24 -0
- package/src/plugins/defaults/memory/AGENTS.md +39 -5
- package/src/plugins/defaults/memory/__tests__/archive-removal-framing.test.ts +64 -0
- package/src/plugins/defaults/memory/__tests__/buffer-file.test.ts +365 -0
- package/src/plugins/defaults/memory/__tests__/buffer-format.test.ts +43 -0
- package/src/plugins/defaults/memory/__tests__/conversation-memory-purge.test.ts +1 -0
- package/src/plugins/defaults/memory/__tests__/db-memory-attach.test.ts +2 -0
- package/src/plugins/defaults/memory/__tests__/fixtures/buffer-appender.ts +17 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-job.test.ts +74 -0
- package/src/plugins/defaults/memory/__tests__/memory-run-evidence.test.ts +161 -0
- package/src/plugins/defaults/memory/__tests__/relocated-memory-test-rows.ts +10 -0
- package/src/plugins/defaults/memory/__tests__/worker-control.test.ts +19 -6
- package/src/plugins/defaults/memory/buffer-file.ts +354 -0
- package/src/plugins/defaults/memory/buffer-format.ts +40 -0
- package/src/plugins/defaults/memory/context-search/agent-protocol.ts +9 -12
- package/src/plugins/defaults/memory/context-search/agent-runner.ts +77 -22
- package/src/plugins/defaults/memory/context-search/format.ts +53 -8
- package/src/plugins/defaults/memory/context-search/limits.ts +10 -12
- package/src/plugins/defaults/memory/context-search/recall-input.ts +38 -0
- package/src/plugins/defaults/memory/context-search/search.ts +8 -14
- package/src/plugins/defaults/memory/context-search/sources/conversations.ts +1 -0
- package/src/plugins/defaults/memory/context-search/sources/memory-v2.ts +2 -1
- package/src/plugins/defaults/memory/context-search/sources/workspace.ts +2 -1
- package/src/plugins/defaults/memory/context-search/types.ts +1 -11
- package/src/plugins/defaults/memory/conversation-memory-purge.ts +4 -0
- package/src/plugins/defaults/memory/graph/__tests__/handle-remember-v2.test.ts +11 -4
- package/src/plugins/defaults/memory/graph/capability-seed.ts +1 -2
- package/src/plugins/defaults/memory/graph/tool-handlers.ts +13 -63
- package/src/plugins/defaults/memory/graph/tools.ts +35 -64
- package/src/plugins/defaults/memory/host-utils.ts +0 -10
- package/src/plugins/defaults/memory/injectors.ts +4 -3
- package/src/plugins/defaults/memory/memory-retrospective-job.ts +72 -183
- package/src/plugins/defaults/memory/memory-run-evidence.ts +213 -0
- package/src/plugins/defaults/memory/src/memory-eval-routes.ts +2 -3
- package/src/plugins/defaults/memory/src/memory-item-routes.test.ts +6 -1
- package/src/plugins/defaults/memory/src/memory-item-routes.ts +2 -1
- package/src/plugins/defaults/memory/substrate/__tests__/consolidation-job.test.ts +407 -99
- package/src/plugins/defaults/memory/substrate/__tests__/consolidation-prompt-flag-gating-guard.test.ts +10 -0
- package/src/plugins/defaults/memory/substrate/__tests__/prompts-consolidation.test.ts +107 -7
- package/src/plugins/defaults/memory/substrate/consolidation-job.ts +307 -86
- package/src/plugins/defaults/memory/substrate/consolidation-tool-surface.ts +34 -0
- package/src/plugins/defaults/memory/substrate/page-index.ts +2 -1
- package/src/plugins/defaults/memory/substrate/prompts/consolidation.ts +91 -51
- package/src/plugins/defaults/memory/substrate/sweep-job.ts +1 -1
- package/src/plugins/defaults/memory/tools.test.ts +124 -9
- package/src/plugins/defaults/memory/tools.ts +38 -11
- package/src/plugins/defaults/memory/v1/graph/consolidation.ts +2 -2
- package/src/plugins/defaults/memory/v1/graph/extraction.ts +2 -1
- package/src/plugins/defaults/memory/v1/graph/retriever.ts +1 -1
- package/src/plugins/defaults/memory/v2/__tests__/migration.test.ts +5 -0
- package/src/plugins/defaults/memory/v2/__tests__/reranker.test.ts +5 -2
- package/src/plugins/defaults/memory/v2/reranker.ts +2 -1
- package/src/plugins/defaults/memory/v3/__tests__/injection.test.ts +81 -1
- package/src/plugins/defaults/memory/v3/__tests__/orchestrate.test.ts +87 -0
- package/src/plugins/defaults/memory/v3/__tests__/plugin-schema.test.ts +12 -0
- package/src/plugins/defaults/memory/v3/__tests__/pool-log-store.test.ts +332 -15
- package/src/plugins/defaults/memory/v3/__tests__/pool-select.test.ts +626 -27
- package/src/plugins/defaults/memory/v3/__tests__/shadow-plugin.test.ts +148 -28
- package/src/plugins/defaults/memory/v3/card.ts +2 -1
- package/src/plugins/defaults/memory/v3/injector.ts +212 -178
- package/src/plugins/defaults/memory/v3/orchestrate.ts +159 -32
- package/src/plugins/defaults/memory/v3/plugin-schema.ts +54 -4
- package/src/plugins/defaults/memory/v3/pool-log-store.ts +275 -37
- package/src/plugins/defaults/memory/v3/pool-select.test.ts +47 -5
- package/src/plugins/defaults/memory/v3/pool-select.ts +710 -59
- package/src/plugins/defaults/memory/v3/sections.ts +2 -1
- package/src/plugins/defaults/memory/v3/shadow-plugin.ts +41 -14
- package/src/plugins/defaults/memory/worker-control.ts +15 -5
- package/src/plugins/defaults/task-progress/__tests__/registration.test.ts +70 -0
- package/src/plugins/defaults/task-progress/__tests__/task-progress-context.test.ts +380 -0
- package/src/plugins/defaults/task-progress/hooks/post-compact.ts +19 -0
- package/src/plugins/defaults/task-progress/hooks/user-prompt-submit.ts +24 -0
- package/src/plugins/defaults/task-progress/package.json +14 -0
- package/src/plugins/defaults/task-progress/src/apply-task-progress-context.ts +93 -0
- package/src/plugins/defaults/task-progress/src/format-task-progress-context.ts +67 -0
- package/src/plugins/defaults/tool-error/hooks/post-tool-use.ts +4 -1
- package/src/plugins/defaults/tool-result-truncate/terminal.ts +1 -46
- package/src/prompts/__tests__/parallel-tasks-section.test.ts +25 -0
- package/src/prompts/delegation-gate.ts +57 -0
- package/src/prompts/system-prompt.ts +14 -32
- package/src/providers/__tests__/context-overflow-error.test.ts +10 -0
- package/src/providers/__tests__/search-provider-catalog.test.ts +18 -0
- package/src/providers/anthropic/client.ts +10 -1
- package/src/providers/anthropic-gateway-shared.ts +6 -0
- package/src/providers/fetch-provider-catalog.ts +11 -0
- package/src/providers/gemini/client.ts +25 -8
- package/src/providers/inference/adapter-factory.ts +6 -0
- package/src/providers/jev/client.test.ts +270 -0
- package/src/providers/jev/client.ts +534 -0
- package/src/providers/model-catalog.ts +59 -3
- package/src/providers/openai/__tests__/chat-completions-extra-body.test.ts +94 -1
- package/src/providers/openai/__tests__/chat-completions-provider-reasoning.test.ts +106 -1
- package/src/providers/openai/__tests__/chat-template-flatten.test.ts +24 -0
- package/src/providers/openai/__tests__/google-thought-signature.test.ts +1 -2
- package/src/providers/openai/chat-completions-provider.ts +100 -29
- package/src/providers/openai/google-thought-signature.ts +0 -6
- package/src/providers/openai/responses-provider.ts +23 -1
- package/src/providers/ratelimit.ts +1 -3
- package/src/providers/search-provider-catalog.ts +24 -0
- package/src/providers/server-tool-pairing.ts +16 -7
- package/src/providers/speech-to-text/__tests__/deepgram-flux-frames.test.ts +24 -0
- package/src/providers/speech-to-text/__tests__/deepgram-flux-realtime.test.ts +7 -1
- package/src/providers/speech-to-text/__tests__/vellum-managed-realtime.test.ts +50 -1
- package/src/providers/speech-to-text/deepgram-flux-frames.ts +11 -0
- package/src/providers/speech-to-text/vellum-managed-flux-realtime.ts +13 -0
- package/src/providers/types.ts +5 -0
- package/src/routes/control.ts +7 -5
- package/src/runtime/AGENTS.md +6 -4
- package/src/runtime/__tests__/agent-wake.test.ts +94 -1
- package/src/runtime/actor-trust-resolver.ts +3 -3
- package/src/runtime/agent-wake.ts +32 -5
- package/src/runtime/approval-message-composer.ts +3 -69
- package/src/runtime/assistant-event-hub.ts +73 -75
- package/src/runtime/assistant-event-publish-options.ts +7 -0
- package/src/runtime/assistant-event-targeting.ts +79 -0
- package/src/runtime/assistant-stream-state.ts +13 -100
- package/src/runtime/background-job-runner.ts +5 -5
- package/src/runtime/channel-retry-sweep.ts +1 -0
- package/src/runtime/guardian-action-message-composer.ts +8 -161
- package/src/runtime/guardian-action-service.ts +2 -17
- package/src/runtime/guardian-reply-router.ts +29 -40
- package/src/runtime/http-server.ts +2 -2
- package/src/runtime/http-types.ts +6 -0
- package/src/runtime/message-composer-types.ts +3 -61
- package/src/runtime/migrations/__tests__/vbundle-import-parity.test.ts +67 -1
- package/src/runtime/migrations/__tests__/vbundle-import-policy.test.ts +89 -0
- package/src/runtime/migrations/vbundle-builder.ts +24 -1
- package/src/runtime/migrations/vbundle-import-analyzer.ts +13 -4
- package/src/runtime/migrations/vbundle-import-policy.ts +43 -11
- package/src/runtime/migrations/vbundle-importer.ts +4 -1
- package/src/runtime/migrations/vbundle-streaming-importer.ts +4 -1
- package/src/runtime/migrations/vbundle-validator.ts +1 -0
- package/src/runtime/pending-interactions-mode-session.test.ts +62 -0
- package/src/runtime/pending-interactions.ts +29 -0
- package/src/runtime/routes/__tests__/client-history-routes.test.ts +104 -0
- package/src/runtime/routes/__tests__/image-generation-routes.test.ts +97 -0
- package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +24 -2
- package/src/runtime/routes/__tests__/roadmap-routes.test.ts +29 -6
- package/src/runtime/routes/browser-context.ts +31 -11
- package/src/runtime/routes/browser-routes.ts +43 -4
- package/src/runtime/routes/btw-routes.ts +9 -4
- package/src/runtime/routes/channel-route-shared.ts +1 -9
- package/src/runtime/routes/client-routes.ts +108 -0
- package/src/runtime/routes/conversation-management-routes.ts +4 -1
- package/src/runtime/routes/conversation-routes.ts +125 -14
- package/src/runtime/routes/credential-prompt-routes.ts +4 -3
- package/src/runtime/routes/credential-routes.ts +1 -2
- package/src/runtime/routes/desktop-setup-routes.test.ts +9 -18
- package/src/runtime/routes/desktop-setup-routes.ts +8 -7
- package/src/runtime/routes/documents-routes.ts +51 -0
- package/src/runtime/routes/email-routes.ts +3 -7
- package/src/runtime/routes/events-routes.ts +75 -54
- package/src/runtime/routes/guardian-approval-interception.ts +24 -0
- package/src/runtime/routes/identity-routes.ts +1 -5
- package/src/runtime/routes/image-generation-routes.ts +8 -14
- package/src/runtime/routes/inbound-message-handler.ts +37 -41
- package/src/runtime/routes/inbound-stages/backfill-sender-provenance.test.ts +168 -0
- package/src/runtime/routes/inbound-stages/backfill-sender-provenance.ts +96 -0
- package/src/runtime/routes/inbound-stages/background-dispatch.test.ts +39 -10
- package/src/runtime/routes/inbound-stages/background-dispatch.ts +9 -9
- package/src/runtime/routes/inbound-stages/reaction-intercept.ts +2 -0
- package/src/runtime/routes/inference-profile-session-handler.ts +11 -1
- package/src/runtime/routes/inference-profiles-routes.ts +9 -0
- package/src/runtime/routes/llm-context-normalization.ts +1 -7
- package/src/runtime/routes/migration-routes.ts +73 -2
- package/src/runtime/routes/notification-routes.ts +10 -0
- package/src/runtime/routes/oauth-commands-routes.ts +57 -54
- package/src/runtime/routes/oauth-providers.ts +11 -0
- package/src/runtime/routes/oauth-request-hints.test.ts +246 -0
- package/src/runtime/routes/oauth-request-hints.ts +149 -0
- package/src/runtime/routes/plugins-routes.ts +6 -0
- package/src/runtime/routes/roadmap-routes.ts +47 -8
- package/src/runtime/routes/schedule-routes.ts +2 -0
- package/src/runtime/routes/secret-routes.ts +99 -119
- package/src/runtime/routes/settings-routes.ts +10 -4
- package/src/runtime/routes/watcher-routes.ts +1 -3
- package/src/runtime/trust-class.test.ts +7 -6
- package/src/runtime/trust-class.ts +9 -17
- package/src/schedule/__tests__/worker-control.test.ts +36 -5
- package/src/schedule/__tests__/worker-mcp-bootstrap.test.ts +107 -0
- package/src/schedule/__tests__/worker-mcp-readiness.test.ts +173 -0
- package/src/schedule/__tests__/worker-mcp-tools.test.ts +13 -47
- package/src/schedule/run-script.ts +2 -2
- package/src/schedule/scheduler.ts +47 -11
- package/src/schedule/tool-surface-readiness.ts +65 -0
- package/src/schedule/worker-control.ts +15 -5
- package/src/schedule/worker-mcp.ts +85 -0
- package/src/schedule/worker.ts +9 -31
- package/src/security/secure-keys.ts +74 -112
- package/src/skills/clawhub.ts +3 -8
- package/src/skills/managed-store.ts +1 -5
- package/src/stt/types.ts +4 -0
- package/src/subagent/notify.ts +54 -34
- package/src/subagent/parent-notification.ts +6 -0
- package/src/telemetry/__tests__/phone-call-funnel.test.ts +72 -0
- package/src/telemetry/phone-call-funnel.ts +144 -0
- package/src/tools/__tests__/tool-schema-root-combinator-guard.test.ts +81 -0
- package/src/tools/ask-question/ask-question-tool.test.ts +167 -5
- package/src/tools/ask-question/ask-question-tool.ts +120 -22
- package/src/tools/browser/browser-execution.ts +29 -60
- package/src/tools/browser/cdp-client/__tests__/cdp-dom-helpers.test.ts +561 -39
- package/src/tools/browser/cdp-client/cdp-dom-helpers.ts +435 -39
- package/src/tools/browser/cdp-client/cdp-inspect-client.ts +1 -8
- package/src/tools/client-os.ts +15 -0
- package/src/tools/computer-use/skill-proxy-bridge.ts +1 -1
- package/src/tools/computer-use/target.ts +42 -0
- package/src/tools/document/document-tool.ts +13 -18
- package/src/tools/execution-target.ts +14 -16
- package/src/tools/host-terminal/host-shell.ts +16 -7
- package/src/tools/network/__tests__/web-fetch-tinyfish.test.ts +250 -0
- package/src/tools/network/__tests__/web-search.test.ts +203 -0
- package/src/tools/network/web-fetch.ts +274 -7
- package/src/tools/network/web-search.ts +468 -6
- package/src/tools/permission-checker.ts +3 -3
- package/src/tools/policy-context.ts +3 -1
- package/src/tools/shared/filesystem/file-ops-service.ts +1 -31
- package/src/tools/shared/shell-output.test.ts +10 -0
- package/src/tools/shared/shell-output.ts +14 -2
- package/src/tools/skills/load.ts +2 -1
- package/src/tools/skills/sandbox-runner.ts +24 -4
- package/src/tools/skills/scaffold-managed.ts +2 -1
- package/src/tools/skills/skill-script-runner.ts +7 -0
- package/src/tools/skills/skill-tool-factory.ts +15 -1
- package/src/tools/terminal/__tests__/safe-env.test.ts +44 -4
- package/src/tools/terminal/__tests__/sanitized-bash.test.ts +0 -14
- package/src/tools/terminal/safe-env.ts +39 -24
- package/src/tools/terminal/sanitized-bash.ts +15 -2
- package/src/tools/terminal/shell-launch.test.ts +162 -0
- package/src/tools/terminal/shell.test.ts +29 -0
- package/src/tools/terminal/shell.ts +15 -7
- package/src/tools/tool-approval-handler.ts +5 -1
- package/src/tools/tool-types.ts +3 -3
- package/src/tools/types.ts +8 -2
- package/src/util/__tests__/worker-process-command.test.ts +5 -4
- package/src/util/abort-reasons.ts +18 -26
- package/src/util/browser-human-verification.ts +14 -0
- package/src/util/canonicalize-identity.test.ts +21 -0
- package/src/util/canonicalize-identity.ts +16 -0
- package/src/util/errors.ts +13 -3
- package/src/util/host-process.test.ts +17 -1
- package/src/util/host-process.ts +24 -0
- package/src/util/provider-error-patterns.ts +3 -0
- package/src/util/unicode.ts +29 -0
- package/src/util/worker-process.ts +77 -35
- package/src/workspace/migrations/157-add-desktop-files-launcher.ts +94 -0
- package/src/workspace/migrations/__tests__/156-extract-workspace-mcp-json.test.ts +4 -10
- package/src/workspace/migrations/__tests__/157-add-desktop-files-launcher.test.ts +113 -0
- package/src/workspace/migrations/registry.ts +2 -0
- package/src/__tests__/computer-use-window-schema.test.ts +0 -24
- package/src/api/events/heartbeat-alert.ts +0 -20
- package/src/daemon/message-types/web-activity.ts +0 -65
- package/src/desktop/desktop-dependencies.test.ts +0 -131
- package/src/tools/computer-use/definitions.ts +0 -607
|
@@ -1089,6 +1089,7 @@ async function drainSingleMessage(
|
|
|
1089
1089
|
content: resolvedContent,
|
|
1090
1090
|
attachments: next.attachments,
|
|
1091
1091
|
requestId: next.requestId,
|
|
1092
|
+
activeSurfaceId: next.activeSurfaceId,
|
|
1092
1093
|
metadata: { ...next.metadata, sentAt: next.sentAt },
|
|
1093
1094
|
displayContent: next.displayContent,
|
|
1094
1095
|
clientMessageId: next.clientMessageId,
|
|
@@ -1155,6 +1156,7 @@ async function drainSingleMessage(
|
|
|
1155
1156
|
messageId: userMessageId,
|
|
1156
1157
|
requestId: next.requestId,
|
|
1157
1158
|
clientMessageId: next.clientMessageId,
|
|
1159
|
+
modeSession: conversation.modeSessions.getTurnOwner(next.requestId),
|
|
1158
1160
|
});
|
|
1159
1161
|
// The row this echo announces is already durably persisted, so advance
|
|
1160
1162
|
// the snapshot↔stream anchor to the echo's seq (stamped inline by the
|
|
@@ -1464,6 +1466,7 @@ async function drainBatch(
|
|
|
1464
1466
|
content: qmContent,
|
|
1465
1467
|
attachments: qm.attachments,
|
|
1466
1468
|
requestId: qm.requestId,
|
|
1469
|
+
activeSurfaceId: qm.activeSurfaceId,
|
|
1467
1470
|
metadata: { ...qm.metadata, sentAt: qm.sentAt },
|
|
1468
1471
|
displayContent: qm.displayContent,
|
|
1469
1472
|
clientMessageId: qm.clientMessageId,
|
|
@@ -1574,6 +1577,7 @@ async function drainBatch(
|
|
|
1574
1577
|
messageId: lastUserMessageId,
|
|
1575
1578
|
requestId: qm.requestId,
|
|
1576
1579
|
clientMessageId: qm.clientMessageId,
|
|
1580
|
+
modeSession: conversation.modeSessions.getTurnOwner(qm.requestId),
|
|
1577
1581
|
});
|
|
1578
1582
|
// Advance the snapshot↔stream anchor to this echo's seq — the batched
|
|
1579
1583
|
// row persisted just above and the agent loop for the batch has not
|
|
@@ -2298,6 +2302,7 @@ export async function processMessage(
|
|
|
2298
2302
|
content: resolvedContent,
|
|
2299
2303
|
attachments,
|
|
2300
2304
|
requestId,
|
|
2305
|
+
activeSurfaceId,
|
|
2301
2306
|
displayContent,
|
|
2302
2307
|
scripted,
|
|
2303
2308
|
...(callerMetadata ? { metadata: callerMetadata } : {}),
|
|
@@ -8,6 +8,8 @@
|
|
|
8
8
|
import { statSync } from "node:fs";
|
|
9
9
|
import { join } from "node:path";
|
|
10
10
|
|
|
11
|
+
import { TrustClassSchema } from "@vellumai/gateway-client";
|
|
12
|
+
|
|
11
13
|
import {
|
|
12
14
|
getApp,
|
|
13
15
|
getAppDirPath,
|
|
@@ -81,7 +83,6 @@ import type {
|
|
|
81
83
|
import type { ContentBlock, Message } from "../providers/types.js";
|
|
82
84
|
import type { TrustClass } from "../runtime/actor-trust-resolver.js";
|
|
83
85
|
import { resolveCapabilities } from "../runtime/capabilities.js";
|
|
84
|
-
import { trustClassSchema } from "../runtime/trust-class.js";
|
|
85
86
|
import type { SubagentState } from "../subagent/types.js";
|
|
86
87
|
import { TERMINAL_STATUSES } from "../subagent/types.js";
|
|
87
88
|
import { canonicalizeInboundIdentity } from "../util/canonicalize-identity.js";
|
|
@@ -903,6 +904,11 @@ export function buildChannelCapabilityBlock(
|
|
|
903
904
|
"- Do NOT use markdown tables — use bullet lists instead. No markdown headers — use **bold** or CAPS for emphasis.",
|
|
904
905
|
);
|
|
905
906
|
}
|
|
907
|
+
if (caps.channel === "email") {
|
|
908
|
+
lines.push(
|
|
909
|
+
"- Conversation text is not emailed. To reply, run `assistant email send` (see `assistant email send --help`). Use `--reply-to` to keep the thread. Skip a reply only when none is needed.",
|
|
910
|
+
);
|
|
911
|
+
}
|
|
906
912
|
}
|
|
907
913
|
|
|
908
914
|
// Inject group chat etiquette only when the chat type indicates a multi-party
|
|
@@ -1183,7 +1189,7 @@ function placeholderForBlockType(type: ContentBlock["type"]): string | null {
|
|
|
1183
1189
|
function rowToRenderable(row: SlackTranscriptInputRow): RenderableSlackMessage {
|
|
1184
1190
|
const slackMeta = readSlackMetadataFromMessageMetadata(row.metadata);
|
|
1185
1191
|
const provenanceTrustClass = row.metadata
|
|
1186
|
-
?
|
|
1192
|
+
? TrustClassSchema.safeParse(
|
|
1187
1193
|
safeParseRecord(row.metadata).provenanceTrustClass,
|
|
1188
1194
|
).data
|
|
1189
1195
|
: undefined;
|
|
@@ -2035,20 +2041,6 @@ export async function composeInjectorChain(ctx: TurnContext): Promise<string> {
|
|
|
2035
2041
|
*/
|
|
2036
2042
|
const DEFAULT_PLACEMENT: InjectionPlacement = "append-user-tail";
|
|
2037
2043
|
|
|
2038
|
-
/**
|
|
2039
|
-
* Count leading memory-prefix blocks on a user message's `content`.
|
|
2040
|
-
*
|
|
2041
|
-
* Delegates to {@link countMemoryPrefixBlocks} from
|
|
2042
|
-
* `memory/graph/conversation-graph-memory.js` — the canonical state-machine
|
|
2043
|
-
* for locating the memory-prefix boundary. Reusing it here keeps the
|
|
2044
|
-
* PKB-context / PKB-reminder / NOW splice rules aligned on a single source
|
|
2045
|
-
* of truth so their ordering relative to any memory prefix is stable and
|
|
2046
|
-
* testable.
|
|
2047
|
-
*/
|
|
2048
|
-
function countMemoryPrefixBlocksOnContent(content: ContentBlock[]): number {
|
|
2049
|
-
return countMemoryPrefixBlocks(content);
|
|
2050
|
-
}
|
|
2051
|
-
|
|
2052
2044
|
/**
|
|
2053
2045
|
* Apply one injector block to a `runMessages` array according to its
|
|
2054
2046
|
* declared {@link InjectionPlacement}:
|
|
@@ -2101,9 +2093,7 @@ function applyInjectionBlock(
|
|
|
2101
2093
|
{ ...userTail, content: [...userTail.content, textBlock] },
|
|
2102
2094
|
];
|
|
2103
2095
|
case "after-memory-prefix": {
|
|
2104
|
-
const memoryPrefixCount =
|
|
2105
|
-
userTail.content,
|
|
2106
|
-
);
|
|
2096
|
+
const memoryPrefixCount = countMemoryPrefixBlocks(userTail.content);
|
|
2107
2097
|
return [
|
|
2108
2098
|
...runMessages.slice(0, -1),
|
|
2109
2099
|
{
|
|
@@ -2161,7 +2151,7 @@ function stripTailV2DynamicMemoryPrefix(
|
|
|
2161
2151
|
if (!last || last.role !== "user") {
|
|
2162
2152
|
return messages;
|
|
2163
2153
|
}
|
|
2164
|
-
const prefixCount =
|
|
2154
|
+
const prefixCount = countMemoryPrefixBlocks(last.content);
|
|
2165
2155
|
if (prefixCount === 0) {
|
|
2166
2156
|
return messages;
|
|
2167
2157
|
}
|
|
@@ -550,10 +550,10 @@ export function clearAllActiveConversations(): number {
|
|
|
550
550
|
/**
|
|
551
551
|
* Evict in-memory conversations after a config/prompt reload so the next
|
|
552
552
|
* turn rebuilds them against the new config. Conversations with in-flight work
|
|
553
|
-
*
|
|
554
|
-
*
|
|
555
|
-
* conversations are disposed and dropped. Also
|
|
556
|
-
* change.
|
|
553
|
+
* are marked stale and rebuilt by `getOrCreateConversation` once that work
|
|
554
|
+
* finishes. This includes live mode-session sources and process-local
|
|
555
|
+
* structural continuations. Idle conversations are disposed and dropped. Also
|
|
556
|
+
* used when provider credentials change.
|
|
557
557
|
*/
|
|
558
558
|
export function evictConversationsForReload(): void {
|
|
559
559
|
const subagentManager = getSubagentManager();
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Canonical conversation surface snapshots for host readers.
|
|
3
|
+
*
|
|
4
|
+
* Combines an unbounded persisted `ui_surface` scan with the loaded
|
|
5
|
+
* conversation's live `surfaceState` and current-turn snapshots. Hits from
|
|
6
|
+
* the persisted scan are never written into `surfaceState`.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { coerceSurfaceDataRecord } from "../api/surfaces.js";
|
|
10
|
+
import { getMessages } from "../persistence/conversation-crud.js";
|
|
11
|
+
import { resolveCapabilities } from "../runtime/capabilities.js";
|
|
12
|
+
import { isPlainObject } from "../util/object.js";
|
|
13
|
+
import { findConversationOrSubagent } from "./conversation-registry.js";
|
|
14
|
+
import { isRowVisibleToUntrustedActor } from "./message-provenance.js";
|
|
15
|
+
|
|
16
|
+
export interface ConversationSurfaceSnapshot {
|
|
17
|
+
surfaceId: string;
|
|
18
|
+
surfaceType: string;
|
|
19
|
+
data: Record<string, unknown>;
|
|
20
|
+
completed: boolean;
|
|
21
|
+
completionSummary?: string;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function cloneSurfaceData(value: unknown): Record<string, unknown> {
|
|
25
|
+
const record = coerceSurfaceDataRecord(value);
|
|
26
|
+
try {
|
|
27
|
+
return structuredClone(record);
|
|
28
|
+
} catch {
|
|
29
|
+
return JSON.parse(JSON.stringify(record)) as Record<string, unknown>;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function surfaceTypeFromBlock(value: unknown): string {
|
|
34
|
+
return typeof value === "string" && value.trim().length > 0
|
|
35
|
+
? value
|
|
36
|
+
: "dynamic_page";
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function rememberOrder(order: string[], surfaceId: string): void {
|
|
40
|
+
if (!order.includes(surfaceId)) {
|
|
41
|
+
order.push(surfaceId);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Current surface snapshots for a loaded conversation.
|
|
47
|
+
*
|
|
48
|
+
* Fails closed to an empty list when the conversation is not already
|
|
49
|
+
* loaded or the turn-or-resting trust snapshot is missing. Untrusted
|
|
50
|
+
* views reuse {@link isRowVisibleToUntrustedActor} and do not add
|
|
51
|
+
* live-only cards that never appeared in a visible persisted row.
|
|
52
|
+
*/
|
|
53
|
+
export function listConversationSurfaceSnapshots(
|
|
54
|
+
conversationId: string,
|
|
55
|
+
): ConversationSurfaceSnapshot[] {
|
|
56
|
+
const conversation = findConversationOrSubagent(conversationId);
|
|
57
|
+
if (!conversation) {
|
|
58
|
+
return [];
|
|
59
|
+
}
|
|
60
|
+
const trust = conversation.getTurnOrRestingTrust();
|
|
61
|
+
if (!trust) {
|
|
62
|
+
return [];
|
|
63
|
+
}
|
|
64
|
+
const canAccessMemory = resolveCapabilities(trust.trustClass).canAccessMemory;
|
|
65
|
+
const filterByProvenance = !canAccessMemory;
|
|
66
|
+
|
|
67
|
+
const order: string[] = [];
|
|
68
|
+
const records = new Map<string, ConversationSurfaceSnapshot>();
|
|
69
|
+
|
|
70
|
+
const rows = getMessages(conversationId);
|
|
71
|
+
for (const row of rows) {
|
|
72
|
+
if (filterByProvenance && !isRowVisibleToUntrustedActor(row.metadata)) {
|
|
73
|
+
continue;
|
|
74
|
+
}
|
|
75
|
+
if (!Array.isArray(row.content)) {
|
|
76
|
+
continue;
|
|
77
|
+
}
|
|
78
|
+
for (const block of row.content) {
|
|
79
|
+
if (!isPlainObject(block) || block.type !== "ui_surface") {
|
|
80
|
+
continue;
|
|
81
|
+
}
|
|
82
|
+
const surfaceId = block.surfaceId;
|
|
83
|
+
if (typeof surfaceId !== "string" || surfaceId.length === 0) {
|
|
84
|
+
continue;
|
|
85
|
+
}
|
|
86
|
+
const completionSummary =
|
|
87
|
+
typeof block.completionSummary === "string"
|
|
88
|
+
? block.completionSummary
|
|
89
|
+
: undefined;
|
|
90
|
+
rememberOrder(order, surfaceId);
|
|
91
|
+
records.set(surfaceId, {
|
|
92
|
+
surfaceId,
|
|
93
|
+
surfaceType: surfaceTypeFromBlock(block.surfaceType),
|
|
94
|
+
data: cloneSurfaceData(block.data),
|
|
95
|
+
completed: block.completed === true,
|
|
96
|
+
...(completionSummary !== undefined ? { completionSummary } : {}),
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
for (const [surfaceId, entry] of conversation.surfaceState.entries()) {
|
|
102
|
+
const existing = records.get(surfaceId);
|
|
103
|
+
if (!existing && filterByProvenance) {
|
|
104
|
+
continue;
|
|
105
|
+
}
|
|
106
|
+
rememberOrder(order, surfaceId);
|
|
107
|
+
records.set(surfaceId, {
|
|
108
|
+
surfaceId,
|
|
109
|
+
surfaceType: entry.surfaceType,
|
|
110
|
+
data: cloneSurfaceData(entry.data),
|
|
111
|
+
completed: existing?.completed === true,
|
|
112
|
+
...(existing?.completionSummary !== undefined
|
|
113
|
+
? { completionSummary: existing.completionSummary }
|
|
114
|
+
: {}),
|
|
115
|
+
});
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
for (const surface of conversation.currentTurnSurfaces) {
|
|
119
|
+
const existing = records.get(surface.surfaceId);
|
|
120
|
+
const completionSummary =
|
|
121
|
+
typeof surface.completionSummary === "string"
|
|
122
|
+
? surface.completionSummary
|
|
123
|
+
: existing?.completionSummary;
|
|
124
|
+
rememberOrder(order, surface.surfaceId);
|
|
125
|
+
records.set(surface.surfaceId, {
|
|
126
|
+
surfaceId: surface.surfaceId,
|
|
127
|
+
surfaceType: surface.surfaceType,
|
|
128
|
+
data: cloneSurfaceData(surface.data),
|
|
129
|
+
completed: surface.completed === true || existing?.completed === true,
|
|
130
|
+
...(completionSummary !== undefined ? { completionSummary } : {}),
|
|
131
|
+
});
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
return order.map((surfaceId) => records.get(surfaceId)!);
|
|
135
|
+
}
|
|
@@ -28,6 +28,8 @@ import {
|
|
|
28
28
|
resolveEffectiveAppHtml,
|
|
29
29
|
updateApp,
|
|
30
30
|
} from "../apps/app-store.js";
|
|
31
|
+
import { executeDesktopComputerUse } from "../desktop/desktop-computer-use.js";
|
|
32
|
+
import { canUseVirtualDesktop } from "../desktop/virtual-desktop-feature.js";
|
|
31
33
|
import { recordActivationEvent } from "../onboarding/onboarding-events-store.js";
|
|
32
34
|
import {
|
|
33
35
|
getMessages,
|
|
@@ -57,6 +59,7 @@ import {
|
|
|
57
59
|
import { resolveAppId } from "../tools/apps/resolve-app-id.js";
|
|
58
60
|
import { formatDesktopAppRequired } from "../tools/capability-offer.js";
|
|
59
61
|
import { POINT_AT_PROXY_TOOL } from "../tools/computer-use/skill-proxy-bridge.js";
|
|
62
|
+
import { computerUseTarget } from "../tools/computer-use/target.js";
|
|
60
63
|
import type { ToolExecutionResult } from "../tools/types.js";
|
|
61
64
|
import { getLogger } from "../util/logger.js";
|
|
62
65
|
import { isPlainObject } from "../util/object.js";
|
|
@@ -70,7 +73,7 @@ import {
|
|
|
70
73
|
type SurfaceShowPair,
|
|
71
74
|
type SurfaceStateEntry,
|
|
72
75
|
} from "./conversation-surface-state.js";
|
|
73
|
-
import
|
|
76
|
+
import { HostCuProxy } from "./host-cu-proxy.js";
|
|
74
77
|
import { resolveHostCuTarget } from "./host-cu-target.js";
|
|
75
78
|
import type {
|
|
76
79
|
AnySurfaceData,
|
|
@@ -89,9 +92,11 @@ import type {
|
|
|
89
92
|
} from "./message-protocol.js";
|
|
90
93
|
import { INTERACTIVE_SURFACE_TYPES } from "./message-protocol.js";
|
|
91
94
|
import { isRowVisibleToUntrustedActor } from "./message-provenance.js";
|
|
95
|
+
import { bestEffortModeSessionTracking } from "./mode-session-tracking.js";
|
|
92
96
|
import type { TrustContext } from "./trust-context-types.js";
|
|
93
97
|
import { restingTrust } from "./trust-context-types.js";
|
|
94
98
|
import { turnActorPrincipalId } from "./turn-actor.js";
|
|
99
|
+
import { virtualDesktopContext } from "./virtual-desktop-context.js";
|
|
95
100
|
export {
|
|
96
101
|
buildSurfaceShowPair,
|
|
97
102
|
type CurrentTurnSurface,
|
|
@@ -178,12 +183,61 @@ const NON_BLOCKING_PENDING_SURFACE_TYPES = new Set<SurfaceType>([
|
|
|
178
183
|
export function hasBlockingPendingSurface(ctx: {
|
|
179
184
|
pendingSurfaceActions: Map<string, { surfaceType: SurfaceType }>;
|
|
180
185
|
}): boolean {
|
|
181
|
-
|
|
186
|
+
return blockingPendingSurfaceIds(ctx).length > 0;
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
export function blockingPendingSurfaceIds(ctx: {
|
|
190
|
+
pendingSurfaceActions: Map<string, { surfaceType: SurfaceType }>;
|
|
191
|
+
}): string[] {
|
|
192
|
+
const ids: string[] = [];
|
|
193
|
+
for (const [surfaceId, entry] of ctx.pendingSurfaceActions) {
|
|
182
194
|
if (!NON_BLOCKING_PENDING_SURFACE_TYPES.has(entry.surfaceType)) {
|
|
183
|
-
|
|
195
|
+
ids.push(surfaceId);
|
|
184
196
|
}
|
|
185
197
|
}
|
|
186
|
-
return
|
|
198
|
+
return ids;
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
function acceptModeSessionSurfaceResponse(
|
|
202
|
+
ctx: Conversation,
|
|
203
|
+
requestId: string,
|
|
204
|
+
surfaceId: string,
|
|
205
|
+
): void {
|
|
206
|
+
bestEffortModeSessionTracking("surface response admission", () =>
|
|
207
|
+
ctx.modeSessions?.acceptTurn(requestId, {
|
|
208
|
+
kind: "surface",
|
|
209
|
+
responseId: surfaceId,
|
|
210
|
+
}),
|
|
211
|
+
);
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
function invalidateModeSessionSurfaceWait(
|
|
215
|
+
ctx: Partial<Pick<Conversation, "modeSessions">>,
|
|
216
|
+
surfaceId: string,
|
|
217
|
+
): void {
|
|
218
|
+
bestEffortModeSessionTracking("surface wait invalidation", () =>
|
|
219
|
+
ctx.modeSessions?.invalidateStructuralWait({
|
|
220
|
+
kind: "surface",
|
|
221
|
+
responseId: surfaceId,
|
|
222
|
+
}),
|
|
223
|
+
);
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
function settleModeSessionSurfaceWait(
|
|
227
|
+
ctx: Pick<Conversation, "conversationId" | "modeSessions">,
|
|
228
|
+
surfaceId: string,
|
|
229
|
+
): void {
|
|
230
|
+
try {
|
|
231
|
+
ctx.modeSessions?.settleStructuralWait(
|
|
232
|
+
{ kind: "surface", responseId: surfaceId },
|
|
233
|
+
{ status: "completed", endReason: "surface_launch_settled" },
|
|
234
|
+
);
|
|
235
|
+
} catch (err) {
|
|
236
|
+
log.warn(
|
|
237
|
+
{ err, conversationId: ctx.conversationId, surfaceId },
|
|
238
|
+
"Mode-session launcher settlement failed",
|
|
239
|
+
);
|
|
240
|
+
}
|
|
187
241
|
}
|
|
188
242
|
|
|
189
243
|
/**
|
|
@@ -1292,7 +1346,8 @@ export function cleanupStandaloneSurface(
|
|
|
1292
1346
|
| "lastSurfaceAction"
|
|
1293
1347
|
| "accumulatedSurfaceState"
|
|
1294
1348
|
| "surfaceUndoStacks"
|
|
1295
|
-
|
|
1349
|
+
> &
|
|
1350
|
+
Partial<Pick<Conversation, "modeSessions">>,
|
|
1296
1351
|
surfaceId: string,
|
|
1297
1352
|
): void {
|
|
1298
1353
|
const entry = ctx.pendingStandaloneSurfaces?.get(surfaceId);
|
|
@@ -1305,6 +1360,7 @@ export function cleanupStandaloneSurface(
|
|
|
1305
1360
|
ctx.lastSurfaceAction.delete(surfaceId);
|
|
1306
1361
|
ctx.accumulatedSurfaceState.delete(surfaceId);
|
|
1307
1362
|
ctx.surfaceUndoStacks.delete(surfaceId);
|
|
1363
|
+
invalidateModeSessionSurfaceWait(ctx, surfaceId);
|
|
1308
1364
|
|
|
1309
1365
|
// Record a tombstone so late client actions are silently dropped.
|
|
1310
1366
|
if (ctx.recentlyCompletedStandaloneSurfaces) {
|
|
@@ -2110,6 +2166,7 @@ export async function handleSurfaceAction(
|
|
|
2110
2166
|
...(anchorMessageId ? { anchorMessageId } : {}),
|
|
2111
2167
|
...(originTrustContext ? { originTrustContext } : {}),
|
|
2112
2168
|
});
|
|
2169
|
+
settleModeSessionSurfaceWait(ctx, surfaceId);
|
|
2113
2170
|
log.info(
|
|
2114
2171
|
{ originConversationId: ctx.conversationId, conversationId, surfaceId },
|
|
2115
2172
|
"launch_conversation dispatched inline from surface action",
|
|
@@ -2284,6 +2341,8 @@ export async function handleSurfaceAction(
|
|
|
2284
2341
|
return QUEUE_FULL_RESULT;
|
|
2285
2342
|
}
|
|
2286
2343
|
|
|
2344
|
+
acceptModeSessionSurfaceResponse(ctx, requestId, surfaceId);
|
|
2345
|
+
|
|
2287
2346
|
// Terminal user commit accepted — record the activation milestone if this
|
|
2288
2347
|
// surface was tagged (best-effort, no-op otherwise). Deferred until after
|
|
2289
2348
|
// the rejection check so a queue-full click doesn't over-report a moment
|
|
@@ -2310,6 +2369,7 @@ export async function handleSurfaceAction(
|
|
|
2310
2369
|
type: "user_message_echo",
|
|
2311
2370
|
text: prompt,
|
|
2312
2371
|
conversationId: ctx.conversationId,
|
|
2372
|
+
modeSession: ctx.modeSessions.getTurnOwner(requestId),
|
|
2313
2373
|
});
|
|
2314
2374
|
}
|
|
2315
2375
|
|
|
@@ -2537,6 +2597,8 @@ export async function handleSurfaceAction(
|
|
|
2537
2597
|
return QUEUE_FULL_RESULT;
|
|
2538
2598
|
}
|
|
2539
2599
|
|
|
2600
|
+
acceptModeSessionSurfaceResponse(ctx, requestId, surfaceId);
|
|
2601
|
+
|
|
2540
2602
|
// Terminal user commit accepted — record the activation milestone if this
|
|
2541
2603
|
// surface was tagged (best-effort, no-op otherwise). Deferred until after the
|
|
2542
2604
|
// rejection check so a queue-full click doesn't over-report a moment (and the
|
|
@@ -2563,6 +2625,7 @@ export async function handleSurfaceAction(
|
|
|
2563
2625
|
type: "user_message_echo",
|
|
2564
2626
|
text: prompt,
|
|
2565
2627
|
conversationId: ctx.conversationId,
|
|
2628
|
+
modeSession: ctx.modeSessions.getTurnOwner(requestId),
|
|
2566
2629
|
});
|
|
2567
2630
|
}
|
|
2568
2631
|
if (result.queued) {
|
|
@@ -3161,6 +3224,28 @@ export async function surfaceProxyResolver(
|
|
|
3161
3224
|
): Promise<ToolExecutionResult> {
|
|
3162
3225
|
// Route CU proxy tools (all computer_use_* action tools)
|
|
3163
3226
|
if (toolName.startsWith("computer_use_")) {
|
|
3227
|
+
const desktopContext = virtualDesktopContext(ctx, signal);
|
|
3228
|
+
if (
|
|
3229
|
+
toolName !== POINT_AT_PROXY_TOOL &&
|
|
3230
|
+
computerUseTarget(input, desktopContext) === "assistant-desktop"
|
|
3231
|
+
) {
|
|
3232
|
+
if (!canUseVirtualDesktop(desktopContext)) {
|
|
3233
|
+
return {
|
|
3234
|
+
content:
|
|
3235
|
+
"The assistant desktop requires an identified guardian and an enabled platform-hosted assistant.",
|
|
3236
|
+
isError: true,
|
|
3237
|
+
};
|
|
3238
|
+
}
|
|
3239
|
+
if (!ctx.hostCuProxy) {
|
|
3240
|
+
ctx.setHostCuProxy(new HostCuProxy());
|
|
3241
|
+
}
|
|
3242
|
+
return executeDesktopComputerUse(
|
|
3243
|
+
toolName,
|
|
3244
|
+
input,
|
|
3245
|
+
desktopContext,
|
|
3246
|
+
ctx.hostCuProxy!,
|
|
3247
|
+
);
|
|
3248
|
+
}
|
|
3164
3249
|
const hostCuProxy = ensureHostCuProxy(ctx);
|
|
3165
3250
|
if (!hostCuProxy || !hostCuProxy.isAvailable()) {
|
|
3166
3251
|
return {
|
|
@@ -3180,6 +3265,15 @@ export async function surfaceProxyResolver(
|
|
|
3180
3265
|
: typeof input.answer === "string"
|
|
3181
3266
|
? input.answer
|
|
3182
3267
|
: "Task complete";
|
|
3268
|
+
bestEffortModeSessionTracking("computer completion", () =>
|
|
3269
|
+
ctx.computerUseModeSessions.endTask({
|
|
3270
|
+
turnId: ctx.currentRequestId,
|
|
3271
|
+
source: {
|
|
3272
|
+
sourceId: hostCuProxy.sourceId,
|
|
3273
|
+
generation: hostCuProxy.resetGeneration,
|
|
3274
|
+
},
|
|
3275
|
+
}),
|
|
3276
|
+
);
|
|
3183
3277
|
hostCuProxy.endTask(ctx.conversationId);
|
|
3184
3278
|
return { content: summary, isError: false };
|
|
3185
3279
|
}
|
|
@@ -3215,9 +3309,11 @@ export async function surfaceProxyResolver(
|
|
|
3215
3309
|
// `maxStepsPerSession` would let a long walkthrough exhaust a budget
|
|
3216
3310
|
// meant for actions and be told to call `computer_use_done`, which has
|
|
3217
3311
|
// nothing to do with what it was doing.
|
|
3312
|
+
const activityAt = Date.now();
|
|
3218
3313
|
if (toolName !== POINT_AT_PROXY_TOOL) {
|
|
3219
3314
|
hostCuProxy.recordAction(toolName, input, reasoning);
|
|
3220
3315
|
}
|
|
3316
|
+
const turnId = ctx.currentRequestId;
|
|
3221
3317
|
return hostCuProxy.request(
|
|
3222
3318
|
toolName,
|
|
3223
3319
|
input,
|
|
@@ -3227,6 +3323,18 @@ export async function surfaceProxyResolver(
|
|
|
3227
3323
|
signal,
|
|
3228
3324
|
targetClientId,
|
|
3229
3325
|
sourceActorPrincipalId,
|
|
3326
|
+
toolName !== POINT_AT_PROXY_TOOL && turnId
|
|
3327
|
+
? () => {
|
|
3328
|
+
ctx.computerUseModeSessions.recordAction({
|
|
3329
|
+
turnId,
|
|
3330
|
+
source: {
|
|
3331
|
+
sourceId: hostCuProxy.sourceId,
|
|
3332
|
+
generation: hostCuProxy.resetGeneration,
|
|
3333
|
+
},
|
|
3334
|
+
at: activityAt,
|
|
3335
|
+
});
|
|
3336
|
+
}
|
|
3337
|
+
: undefined,
|
|
3230
3338
|
);
|
|
3231
3339
|
}
|
|
3232
3340
|
|
|
@@ -19,9 +19,15 @@ import {
|
|
|
19
19
|
resolveSendUserMessageActive,
|
|
20
20
|
SEND_USER_MESSAGE_TOOL_NAME,
|
|
21
21
|
} from "../config/send-user-message-gate.js";
|
|
22
|
+
import { canUseVirtualDesktop } from "../desktop/virtual-desktop-feature.js";
|
|
22
23
|
import { supportsChannelReaction } from "../messaging/providers/index.js";
|
|
23
24
|
import type { PermissionPrompter } from "../permissions/prompter.js";
|
|
24
25
|
import type { SecretPrompter } from "../permissions/secret-prompter.js";
|
|
26
|
+
import {
|
|
27
|
+
type ConversationToolSurface,
|
|
28
|
+
hashConversationToolSurface,
|
|
29
|
+
recordConversationToolSurface,
|
|
30
|
+
} from "../persistence/conversation-tool-surface.js";
|
|
25
31
|
import { getBindingByConversation } from "../persistence/external-conversation-store.js";
|
|
26
32
|
import { getAllDefaultPluginNames } from "../plugins/defaults/main.js";
|
|
27
33
|
import { isActivationSession } from "../plugins/defaults/memory/activation-session-store.js";
|
|
@@ -82,6 +88,7 @@ import {
|
|
|
82
88
|
} from "./doordash-steps.js";
|
|
83
89
|
import { runPostExecutionSideEffects } from "./tool-side-effects.js";
|
|
84
90
|
import { FALLBACK_TURN_TRUST, resolveTrustClass } from "./trust-context.js";
|
|
91
|
+
import { virtualDesktopContext } from "./virtual-desktop-context.js";
|
|
85
92
|
|
|
86
93
|
const log = getLogger("conversation-tool-setup");
|
|
87
94
|
|
|
@@ -457,8 +464,7 @@ export function createToolExecutor(
|
|
|
457
464
|
sendUserMessageActive: resolveSendUserMessageActive(ctx),
|
|
458
465
|
toolUseId,
|
|
459
466
|
isPlatformHosted: getIsPlatform(),
|
|
460
|
-
|
|
461
|
-
clientOs: resolveTurnClientOs(ctx).clientOs,
|
|
467
|
+
...resolveTurnClientOs(ctx),
|
|
462
468
|
overrideProfile: ctx.currentTurnOverrideProfile,
|
|
463
469
|
cronRunId: ctx.currentTurnCronRunId,
|
|
464
470
|
invokingCallSite: ctx.currentCallSite ?? "mainAgent",
|
|
@@ -738,6 +744,7 @@ function isToolSupportedOnClientOs(name: string, ctx: Conversation): boolean {
|
|
|
738
744
|
clientOs,
|
|
739
745
|
transportInterface,
|
|
740
746
|
sourceActorPrincipalId: ctx.getTurnActorPrincipalId?.(),
|
|
747
|
+
trustClass: virtualDesktopContext(ctx).trustClass,
|
|
741
748
|
});
|
|
742
749
|
}
|
|
743
750
|
|
|
@@ -932,8 +939,12 @@ export function canSpawnSubagentsForTurn(ctx: Conversation): boolean {
|
|
|
932
939
|
// executor instead. That is the right answer to "is this tool on the wire"
|
|
933
940
|
// and the wrong one to "could this turn actually spawn": the memory
|
|
934
941
|
// retrospective wake runs in execution mode with an allowlist that names
|
|
935
|
-
// `skill_load` but neither the dispatcher nor the spawn tool, so
|
|
936
|
-
//
|
|
942
|
+
// `skill_load` but neither the dispatcher nor the spawn tool, so a spawn is
|
|
943
|
+
// denied at execution. A wake replaying its source's recorded surface
|
|
944
|
+
// renders the source's delegation state in place of this answer
|
|
945
|
+
// (`Conversation.delegateIndependentTasksReplay`): a denied spawn attempt
|
|
946
|
+
// costs one tool error, a system prompt that differs from the source's
|
|
947
|
+
// costs the whole cached prefix behind it.
|
|
937
948
|
const allowlist = ctx.subagentAllowedTools;
|
|
938
949
|
return SUBAGENT_SPAWN_PATH_TOOL_NAMES.every(
|
|
939
950
|
(name) =>
|
|
@@ -943,6 +954,57 @@ export function canSpawnSubagentsForTurn(ctx: Conversation): boolean {
|
|
|
943
954
|
);
|
|
944
955
|
}
|
|
945
956
|
|
|
957
|
+
/**
|
|
958
|
+
* Build the agent loop's `onToolsSent` observer for a conversation: record
|
|
959
|
+
* the tool array each provider call sends, with the delegation-section state
|
|
960
|
+
* the prompt build captured for the prompt that call carries
|
|
961
|
+
* (`Conversation.renderedDelegateIndependentTasks`), so a later fork wake can
|
|
962
|
+
* replay both (`recordConversationToolSurface`). Only the loop's send boundary sees the
|
|
963
|
+
* sent array. The resolver is also consulted out of band (the token count
|
|
964
|
+
* behind `/compact` and `/clean`, compaction estimates), where a read outside
|
|
965
|
+
* any turn resolves a clientless surface that would overwrite the one the
|
|
966
|
+
* conversation's turns actually send.
|
|
967
|
+
*
|
|
968
|
+
* Arrays that are not the conversation's own surface are skipped: a replaying
|
|
969
|
+
* wake sends its source's array, an empty array is a tools-disabled call (a
|
|
970
|
+
* fork replaying it could never call `remember`), and disk-pressure cleanup
|
|
971
|
+
* mode narrows the wire to cleanup tools. Best-effort: a failed write is
|
|
972
|
+
* logged and the surface still counts as recorded, so a persistent failure
|
|
973
|
+
* logs once per distinct surface rather than once per provider call.
|
|
974
|
+
*/
|
|
975
|
+
export function createWireToolSurfaceRecorder(
|
|
976
|
+
ctx: Conversation,
|
|
977
|
+
): (tools: ToolDefinition[]) => void {
|
|
978
|
+
return (tools) => {
|
|
979
|
+
if (
|
|
980
|
+
!ctx.conversationId ||
|
|
981
|
+
ctx.wireToolReplay ||
|
|
982
|
+
tools.length === 0 ||
|
|
983
|
+
ctx.diskPressureCleanupModeActive === true
|
|
984
|
+
) {
|
|
985
|
+
return;
|
|
986
|
+
}
|
|
987
|
+
const surface: ConversationToolSurface = {
|
|
988
|
+
tools,
|
|
989
|
+
// Unknown before the first prompt build or for a verbatim override.
|
|
990
|
+
delegateIndependentTasks: ctx.renderedDelegateIndependentTasks ?? null,
|
|
991
|
+
};
|
|
992
|
+
try {
|
|
993
|
+
ctx.recordedToolSurfaceHash = recordConversationToolSurface(
|
|
994
|
+
ctx.conversationId,
|
|
995
|
+
surface,
|
|
996
|
+
ctx.recordedToolSurfaceHash,
|
|
997
|
+
);
|
|
998
|
+
} catch (err) {
|
|
999
|
+
log.warn(
|
|
1000
|
+
{ err, conversationId: ctx.conversationId },
|
|
1001
|
+
"failed to record the conversation's wire tool surface; continuing",
|
|
1002
|
+
);
|
|
1003
|
+
ctx.recordedToolSurfaceHash = hashConversationToolSurface(surface);
|
|
1004
|
+
}
|
|
1005
|
+
};
|
|
1006
|
+
}
|
|
1007
|
+
|
|
946
1008
|
/**
|
|
947
1009
|
* Build a resolveTools callback that merges base tool definitions with
|
|
948
1010
|
* dynamically projected skill tools on each agent turn. Also updates
|
|
@@ -1131,7 +1193,7 @@ export function createResolveToolsCallback(
|
|
|
1131
1193
|
definition.name === "bash"
|
|
1132
1194
|
? {
|
|
1133
1195
|
...definition,
|
|
1134
|
-
description: `${definition.description} For browser tasks, use assistant browser navigate --url <url> directly.
|
|
1196
|
+
description: `${definition.description} For browser tasks, use assistant browser navigate --url <url> directly. Chrome and desktop components are included in the assistant image. The command starts Chrome and completes the action without installing dependencies. If components are missing, report the image problem. Use assistant browser --help for other browser actions. Use this managed path even if saved notes describe manual setup. Do not install packages or launch Chrome, X servers, or screenshot scripts yourself.`,
|
|
1135
1197
|
}
|
|
1136
1198
|
: definition,
|
|
1137
1199
|
);
|
|
@@ -1147,6 +1209,9 @@ export function createResolveToolsCallback(
|
|
|
1147
1209
|
const effectivePreactivated = [
|
|
1148
1210
|
...DEFAULT_PREACTIVATED_SKILL_IDS,
|
|
1149
1211
|
...(ctx.preactivatedSkillIds ?? []),
|
|
1212
|
+
...(canUseVirtualDesktop(virtualDesktopContext(ctx))
|
|
1213
|
+
? ["computer-use"]
|
|
1214
|
+
: []),
|
|
1150
1215
|
];
|
|
1151
1216
|
const projection = projectSkillTools(history, {
|
|
1152
1217
|
preactivatedSkillIds: effectivePreactivated,
|
|
@@ -1237,6 +1302,14 @@ export function createResolveToolsCallback(
|
|
|
1237
1302
|
|
|
1238
1303
|
ctx.allowedToolNames = turnAllowed;
|
|
1239
1304
|
|
|
1240
|
-
|
|
1305
|
+
// A wake replaying its source's recorded surface sends that array
|
|
1306
|
+
// verbatim: the wire tool block is the first tier of the provider cache
|
|
1307
|
+
// prefix (tools → system → messages), so only the same bytes read the
|
|
1308
|
+
// source's cached prefix instead of rewriting it. Execution is unaffected:
|
|
1309
|
+
// `allowedToolNames` above and the executor's allowlist gate still decide
|
|
1310
|
+
// what may run.
|
|
1311
|
+
return ctx.wireToolReplay
|
|
1312
|
+
? [...ctx.wireToolReplay]
|
|
1313
|
+
: applyActivityField(allBaseDefs);
|
|
1241
1314
|
};
|
|
1242
1315
|
}
|