@vellumai/assistant 0.12.2 → 0.12.3-staging.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +2 -0
- package/AGENTS.md +1 -1
- package/ARCHITECTURE.md +43 -29
- package/Dockerfile +63 -0
- package/docs/architecture/integrations.md +7 -0
- package/docs/architecture/memory.md +17 -2
- package/docs/architecture/security.md +10 -5
- package/docs/architecture/turn-actor.md +2 -0
- package/docs/credential-execution-service.md +1 -1
- package/docs/desktop-browser-cli.md +50 -5
- package/docs/live-voice-task-outcomes.md +100 -0
- package/docs/mode-sessions.md +183 -0
- package/docs/voice-input-diagnostics.md +58 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/package.json +2 -1
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +1 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/secret-detection.ts +4 -1
- package/node_modules/@vellumai/environments/src/shell.test.ts +21 -0
- package/node_modules/@vellumai/environments/src/shell.ts +24 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/package.json +2 -1
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +1 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/secret-detection.ts +4 -1
- package/node_modules/@vellumai/gateway-client/src/gateway-ipc-contracts.ts +20 -0
- package/node_modules/@vellumai/gateway-client/src/inbound-contract.ts +8 -2
- package/node_modules/@vellumai/gateway-client/src/index.ts +4 -1
- package/node_modules/@vellumai/gateway-client/src/outbound-contract.ts +38 -7
- package/node_modules/@vellumai/gateway-client/src/trust-verdict-contract.ts +5 -6
- package/node_modules/@vellumai/service-contracts/package.json +2 -1
- package/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +1 -0
- package/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
- package/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
- package/node_modules/@vellumai/service-contracts/src/secret-detection.ts +4 -1
- package/node_modules/@vellumai/slack-text/src/index.ts +4 -8
- package/openapi.yaml +716 -16
- package/package.json +1 -1
- package/scripts/smoke-desktop-browser-cli.ts +11 -26
- package/scripts/smoke-desktop-window-theme.ts +2 -3
- package/scripts/sync-llm-catalog.ts +3 -0
- package/scripts/voice-ttft-spike.ts +2 -2
- package/src/__tests__/agent-loop-output-hooks.test.ts +150 -1
- package/src/__tests__/agent-loop-provider-error-recording.test.ts +87 -19
- package/src/__tests__/agent-loop.test.ts +259 -0
- package/src/__tests__/agent-wake-delegation-prompt.test.ts +64 -2
- package/src/__tests__/annotate-activity-metadata.test.ts +3 -1
- package/src/__tests__/annotate-risk-options.test.ts +2 -0
- package/src/__tests__/answered-question-persistence.test.ts +2 -0
- package/src/__tests__/anthropic-gateway-shared.test.ts +22 -0
- package/src/__tests__/anthropic-provider.test.ts +55 -0
- package/src/__tests__/approval-interception-trust-gates.test.ts +40 -0
- package/src/__tests__/approval-message-composer.test.ts +1 -118
- package/src/__tests__/approval-routes-http.test.ts +3 -0
- package/src/__tests__/assistant-event-hub-connection-history.test.ts +111 -0
- package/src/__tests__/assistant-event-hub-targeted.test.ts +50 -0
- package/src/__tests__/assistant-event-targeting.test.ts +186 -0
- package/src/__tests__/assistant-stream-state.test.ts +29 -9
- package/src/__tests__/attachments-store.test.ts +22 -3
- package/src/__tests__/browser-fill-credential.test.ts +11 -22
- package/src/__tests__/btw-routes.test.ts +4 -1
- package/src/__tests__/build-persisted-content.test.ts +1 -1
- package/src/__tests__/call-conversation-messages.test.ts +5 -0
- package/src/__tests__/call-recovery.test.ts +7 -2
- package/src/__tests__/call-routes-http.test.ts +12 -0
- package/src/__tests__/call-store.test.ts +7 -2
- package/src/__tests__/channel-approval.test.ts +9 -14
- package/src/__tests__/channel-reply-delivery.test.ts +56 -0
- package/src/__tests__/channel-retry-sweep.test.ts +9 -0
- package/src/__tests__/chat-credential-redaction.test.ts +23 -0
- package/src/__tests__/chat-reveal-guard-priming.test.ts +2 -0
- package/src/__tests__/checker.test.ts +11 -3
- package/src/__tests__/client-os-metadata-persistence.test.ts +35 -1
- package/src/__tests__/computer-use-screenshot-attachments.test.ts +411 -0
- package/src/__tests__/computer-use-screenshot-selection.test.ts +50 -0
- package/src/__tests__/computer-use-skill-manifest-regression.test.ts +0 -25
- package/src/__tests__/computer-use-tools.test.ts +1 -233
- package/src/__tests__/config-model-image-provider.test.ts +39 -0
- package/src/__tests__/config-schema.test.ts +16 -0
- package/src/__tests__/config-watcher.test.ts +24 -0
- package/src/__tests__/context-search-agent-runner.test.ts +260 -1
- package/src/__tests__/context-search-fanout.test.ts +1 -1
- package/src/__tests__/context-search-types.test.ts +7 -7
- package/src/__tests__/conversation-agent-loop-disk-pressure.test.ts +8 -2
- package/src/__tests__/conversation-agent-loop-fatal-cleanup.test.ts +2 -0
- package/src/__tests__/conversation-agent-loop-inference-profile.test.ts +2 -0
- package/src/__tests__/conversation-agent-loop-overflow.test.ts +2 -0
- package/src/__tests__/conversation-agent-loop.test.ts +797 -16
- package/src/__tests__/conversation-app-control-lifecycle.test.ts +53 -1
- package/src/__tests__/conversation-attachments.test.ts +106 -0
- package/src/__tests__/conversation-evictor.test.ts +62 -37
- package/src/__tests__/conversation-fork-crud.test.ts +191 -0
- package/src/__tests__/conversation-history-boundary-repair-failure.test.ts +174 -0
- package/src/__tests__/conversation-loop-db-lock-resilience.test.ts +2 -0
- package/src/__tests__/conversation-process-app-control-preactivation.test.ts +2 -0
- package/src/__tests__/conversation-rendered-delegation-state.test.ts +276 -0
- package/src/__tests__/conversation-retry-route.test.ts +22 -0
- package/src/__tests__/conversation-routes-disk-view.test.ts +2 -0
- package/src/__tests__/conversation-routes-enabled-plugins.test.ts +2 -0
- package/src/__tests__/conversation-routes-guardian-reply.test.ts +10 -0
- package/src/__tests__/conversation-routes-hidden-queue.test.ts +2 -0
- package/src/__tests__/conversation-routes-slash-commands.test.ts +2 -0
- package/src/__tests__/conversation-runtime-assembly.test.ts +29 -0
- package/src/__tests__/conversation-surface-snapshots.test.ts +543 -0
- package/src/__tests__/conversation-surfaces-action-delivery.test.ts +151 -0
- package/src/__tests__/conversation-surfaces-point-at-budget.test.ts +63 -1
- package/src/__tests__/conversation-tool-setup-attribution.test.ts +23 -0
- package/src/__tests__/conversation-undo.test.ts +61 -0
- package/src/__tests__/credential-execution-client.test.ts +72 -55
- package/src/__tests__/credential-prompt-route.test.ts +35 -3
- package/src/__tests__/cu-unified-flow.test.ts +82 -1
- package/src/__tests__/db-conversation-tool-surface.test.ts +221 -0
- package/src/__tests__/db-mode-sessions-migration.test.ts +175 -0
- package/src/__tests__/deterministic-verification-control-plane.test.ts +2 -0
- package/src/__tests__/dm-backfill.test.ts +91 -1
- package/src/__tests__/document-create-route.test.ts +88 -0
- package/src/__tests__/document-sync-tags.test.ts +27 -0
- package/src/__tests__/emit-signal-routing-intent.test.ts +142 -2
- package/src/__tests__/events-tail-route.test.ts +58 -0
- package/src/__tests__/evict-conversations-for-reload.test.ts +57 -8
- package/src/__tests__/guardian-action-sweep.test.ts +16 -0
- package/src/__tests__/guardian-dispatch.test.ts +13 -0
- package/src/__tests__/headless-browser-interactions.test.ts +64 -20
- package/src/__tests__/heartbeat-service.test.ts +2 -59
- package/src/__tests__/helpers/mock-conversation.ts +21 -0
- package/src/__tests__/helpers/native-web-search-harness.ts +2 -0
- package/src/__tests__/helpers/verification-sessions-ipc-sim.ts +5 -0
- package/src/__tests__/history-repair.test.ts +53 -0
- package/src/__tests__/host-cu-proxy.test.ts +46 -2
- package/src/__tests__/http-user-message-parity.test.ts +2 -0
- package/src/__tests__/image-credentials.test.ts +49 -0
- package/src/__tests__/image-service-dispatcher.test.ts +45 -0
- package/src/__tests__/inference-profile-session-handler.test.ts +26 -0
- package/src/__tests__/interrupt-on-send.test.ts +57 -43
- package/src/__tests__/interrupt-turn-note.test.ts +5 -6
- package/src/__tests__/list-messages-tool-merge.test.ts +85 -1
- package/src/__tests__/llm-catalog-parity.test.ts +27 -4
- package/src/__tests__/media-generate-image.test.ts +49 -1
- package/src/__tests__/message-provenance-contact-id.test.ts +69 -0
- package/src/__tests__/migration-export-to-gcs.test.ts +142 -0
- package/src/__tests__/mode-session-history.test.ts +245 -0
- package/src/__tests__/mode-session-recovery.test.ts +222 -0
- package/src/__tests__/mode-session-wire.test.ts +141 -0
- package/src/__tests__/notification-guardian-path.test.ts +7 -0
- package/src/__tests__/notification-telegram-adapter.test.ts +2 -2
- package/src/__tests__/notification-vellum-adapter.test.ts +34 -0
- package/src/__tests__/oauth-apps-routes.test.ts +1 -0
- package/src/__tests__/oauth-commands-routes.test.ts +114 -101
- package/src/__tests__/oauth-connect-orchestrator.test.ts +2 -0
- package/src/__tests__/oauth-provider-serializer.test.ts +13 -0
- package/src/__tests__/oauth-providers-routes.test.ts +3 -0
- package/src/__tests__/outbound-slack-persistence.test.ts +2 -0
- package/src/__tests__/persist-media-references.test.ts +50 -0
- package/src/__tests__/persistence-secret-redaction.test.ts +2 -0
- package/src/__tests__/plugin-api-provider.test.ts +5 -0
- package/src/__tests__/plugin-api-resolve-credential.test.ts +97 -0
- package/src/__tests__/plugin-import-boundary-guard.test.ts +9 -1
- package/src/__tests__/process-message-display-content.test.ts +56 -0
- package/src/__tests__/reaction-persistence.test.ts +5 -0
- package/src/__tests__/request-secret-standalone-channel-gate.test.ts +25 -9
- package/src/__tests__/run-conversation-turn-persistence.test.ts +138 -1
- package/src/__tests__/runtime-events-sse-reconnect.test.ts +1 -1
- package/src/__tests__/schedule-routes.test.ts +20 -0
- package/src/__tests__/scheduler-result-notification.test.ts +23 -4
- package/src/__tests__/script-proxy-certs.test.ts +1 -1
- package/src/__tests__/scripted-turn-metadata-persistence.test.ts +73 -0
- package/src/__tests__/secret-ingress-http.test.ts +2 -0
- package/src/__tests__/secret-routes-platform-proxy.test.ts +86 -31
- package/src/__tests__/secret-routes-scrub.test.ts +22 -27
- package/src/__tests__/secret-scanner.test.ts +20 -0
- package/src/__tests__/secure-keys-managed-failover.test.ts +8 -0
- package/src/__tests__/secure-keys.test.ts +50 -4
- package/src/__tests__/send-endpoint-busy.test.ts +163 -1
- package/src/__tests__/server-history-render.test.ts +91 -1
- package/src/__tests__/server-tool-pairing.test.ts +107 -0
- package/src/__tests__/settings-routes.test.ts +50 -1
- package/src/__tests__/skill-script-runner-sandbox.test.ts +32 -0
- package/src/__tests__/skills.test.ts +5 -4
- package/src/__tests__/subagent-fork-notifications.test.ts +1 -2
- package/src/__tests__/subagent-manager-notify.test.ts +1 -2
- package/src/__tests__/subagent-notify-parent.test.ts +146 -39
- package/src/__tests__/subagent-notify-stale-rebuild.test.ts +2 -4
- package/src/__tests__/subagent-tool-gate-mode.test.ts +226 -0
- package/src/__tests__/surface-completion-in-flight-snapshot.test.ts +2 -0
- package/src/__tests__/terminal-tools.test.ts +8 -0
- package/src/__tests__/thread-backfill.test.ts +53 -0
- package/src/__tests__/tool-preview-lifecycle.test.ts +2 -0
- package/src/__tests__/tool-result-follow-up.test.ts +99 -0
- package/src/__tests__/tool-result-metadata-plumbing.test.ts +66 -1
- package/src/__tests__/tool-start-timestamp.test.ts +2 -0
- package/src/__tests__/twilio-routes.test.ts +1 -0
- package/src/__tests__/unicode.test.ts +36 -0
- package/src/__tests__/web-search-backend-failure.test.ts +1 -1
- package/src/activation/progress-store.test.ts +12 -7
- package/src/agent/loop.ts +136 -53
- package/src/agent/tool-result-follow-up.ts +70 -0
- package/src/api/attachment-provenance.test.ts +68 -0
- package/src/api/computer-use-tool.test.ts +55 -0
- package/src/api/computer-use-tool.ts +27 -0
- package/src/api/events/assistant-outbound-attachment.ts +3 -0
- package/src/api/events/assistant-turn-start.ts +3 -0
- package/src/api/events/desktop-activity-changed.ts +10 -0
- package/src/api/events/generation-handoff.ts +2 -0
- package/src/api/events/message-complete.ts +2 -0
- package/src/api/events/notification-conversation-created.ts +14 -16
- package/src/api/events/notification-intent.ts +4 -2
- package/src/api/events/question-request.ts +1 -0
- package/src/api/events/tool-result.ts +85 -0
- package/src/api/events/tool-use-start.ts +3 -0
- package/src/api/events/user-message-echo.ts +3 -0
- package/src/api/index.ts +42 -6
- package/src/api/mode-session.test.ts +131 -0
- package/src/api/mode-session.ts +148 -0
- package/src/api/remember-tool.ts +15 -0
- package/src/api/responses/conversation-message.ts +9 -0
- package/src/approvals/AGENTS.md +4 -2
- package/src/approvals/approval-primitive.ts +5 -2
- package/src/approvals/scoped-approval-grants.ts +6 -2
- package/src/browser/__tests__/operations.test.ts +29 -0
- package/src/browser/operation-meta.ts +2 -1
- package/src/browser/operations.ts +29 -0
- package/src/browser/virtual-desktop-target.ts +2 -9
- package/src/bundler/bundle-signer.ts +1 -3
- package/src/calls/__tests__/barge-in-guard.test.ts +3 -0
- package/src/calls/__tests__/call-controller.test.ts +94 -0
- package/src/calls/__tests__/call-funnel-telemetry.test.ts +204 -0
- package/src/calls/__tests__/voice-control-protocol.test.ts +15 -0
- package/src/calls/__tests__/voice-escalation-judge.test.ts +209 -0
- package/src/calls/__tests__/voice-leg-coordinator.test.ts +168 -0
- package/src/{live-voice/__tests__/live-voice-metrics.test.ts → calls/__tests__/voice-metrics.test.ts} +32 -38
- package/src/calls/__tests__/voice-session-bridge.test.ts +576 -122
- package/src/calls/barge-in-guard.ts +5 -0
- package/src/calls/call-controller.ts +196 -1
- package/src/calls/call-domain.ts +4 -0
- package/src/calls/call-store.ts +87 -2
- package/src/calls/guardian-action-sweep.ts +22 -8
- package/src/calls/media-stream-audio-transcode.ts +1 -3
- package/src/calls/typesafe-noul.ts +156 -0
- package/src/calls/voice-control-protocol.ts +6 -0
- package/src/calls/voice-escalation-judge.ts +145 -0
- package/src/calls/voice-escalation-target.ts +10 -0
- package/src/calls/voice-leg-coordinator.ts +121 -3
- package/src/{live-voice/live-voice-metrics.ts → calls/voice-metrics.ts} +94 -87
- package/src/calls/voice-session-bridge.ts +256 -112
- package/src/calls/voice-triage-escalate.ts +7 -0
- package/src/cli/__tests__/roadmap-cli.test.ts +4 -41
- package/src/cli/commands/__tests__/cli-test-harness.ts +21 -3
- package/src/cli/commands/__tests__/notifications.test.ts +35 -10
- package/src/cli/commands/__tests__/plugins.test.ts +20 -1
- package/src/cli/commands/__tests__/schedules.test.ts +14 -0
- package/src/cli/commands/bash.help.ts +4 -3
- package/src/cli/commands/browser.help.ts +9 -6
- package/src/cli/commands/browser.ts +0 -1
- package/src/cli/commands/clients.help.ts +46 -0
- package/src/cli/commands/clients.ts +116 -0
- package/src/cli/commands/credentials.help.ts +9 -4
- package/src/cli/commands/notifications.help.ts +16 -2
- package/src/cli/commands/notifications.ts +30 -8
- package/src/cli/commands/oauth/index.help.ts +10 -0
- package/src/cli/commands/oauth/providers.ts +12 -0
- package/src/cli/commands/oauth/request.test.ts +179 -2
- package/src/cli/commands/oauth/request.ts +41 -10
- package/src/cli/commands/plugins.ts +12 -5
- package/src/cli/commands/roadmap.help.ts +18 -21
- package/src/cli/commands/roadmap.ts +4 -21
- package/src/cli/commands/schedules.ts +2 -0
- package/src/cli/lib/__tests__/inspect-plugin.test.ts +54 -0
- package/src/cli/lib/__tests__/install-from-github.test.ts +41 -0
- package/src/cli/lib/__tests__/list-installed-plugins.test.ts +5 -2
- package/src/cli/lib/__tests__/local-plugin-upgrade.test.ts +133 -51
- package/src/cli/lib/__tests__/plugin-catalog-cache.test.ts +8 -2
- package/src/cli/lib/bundled-marketplace.json +906 -1
- package/src/cli/lib/bundled-plugin-packages.json +783 -1
- package/src/cli/lib/inspect-plugin.ts +11 -4
- package/src/cli/lib/plugin-marketplace.ts +8 -0
- package/src/cli/lib/publish-plugin.ts +1 -2
- package/src/cli/lib/upgrade-plugin.ts +19 -11
- package/src/cli/output.ts +18 -4
- package/src/config/__tests__/default-provider.test.ts +6 -0
- package/src/config/__tests__/memory-retrospective-schema.test.ts +14 -1
- package/src/config/bundled-skills/computer-use/SKILL.md +1 -1
- package/src/config/bundled-skills/computer-use/TOOLS.json +89 -1
- package/src/config/bundled-skills/image-studio/SKILL.md +17 -2
- package/src/config/bundled-skills/image-studio/tools/media-generate-image.ts +7 -11
- package/src/config/bundled-skills/schedule/SKILL.md +6 -6
- package/src/config/bundled-skills/sequences/TOOLS.json +1 -5
- package/src/config/call-site-defaults.ts +9 -0
- package/src/config/feature-flag-registry.json +16 -0
- package/src/config/loader.ts +9 -17
- package/src/config/platform-identity.ts +8 -8
- package/src/config/profile-text-generation.test.ts +51 -0
- package/src/config/profile-text-generation.ts +51 -0
- package/src/config/schemas/__tests__/memory-v3.test.ts +12 -0
- package/src/config/schemas/__tests__/tools.test.ts +31 -0
- package/src/config/schemas/call-site-catalog.ts +14 -0
- package/src/config/schemas/channels.ts +0 -18
- package/src/config/schemas/llm.ts +9 -2
- package/src/config/schemas/mcp.ts +6 -1
- package/src/config/schemas/memory-retrospective.ts +11 -2
- package/src/config/schemas/memory-v3.ts +12 -0
- package/src/config/schemas/services.ts +15 -6
- package/src/config/schemas/tools.ts +10 -0
- package/src/config/session-groups-gate.ts +21 -0
- package/src/config/skills.ts +2 -6
- package/src/conversations/__tests__/message-consolidation.test.ts +149 -0
- package/src/conversations/message-consolidation.ts +102 -5
- package/src/credential-execution/ces-connect.test.ts +69 -0
- package/src/credential-execution/ces-connect.ts +115 -0
- package/src/credential-execution/ces-runtime.ts +55 -93
- package/src/credential-execution/executable-discovery.ts +34 -142
- package/src/credential-execution/process-manager.test.ts +20 -24
- package/src/credential-execution/process-manager.ts +24 -19
- package/src/daemon/__tests__/conversation-surfaces-launch.test.ts +10 -1
- package/src/daemon/__tests__/conversation-tool-setup.test.ts +43 -0
- package/src/daemon/__tests__/inference-profile-notification.test.ts +2 -0
- package/src/daemon/__tests__/turn-tail-deleted-conversation.test.ts +36 -0
- package/src/daemon/assistant-attachments.ts +20 -12
- package/src/daemon/browser-mode-session.test.ts +402 -0
- package/src/daemon/browser-mode-session.ts +186 -0
- package/src/daemon/chat-credential-redaction.ts +16 -1
- package/src/daemon/computer-use-mode-session.test.ts +210 -0
- package/src/daemon/computer-use-mode-session.ts +105 -0
- package/src/daemon/config-watcher.ts +10 -0
- package/src/daemon/conversation-agent-loop-handlers.ts +132 -13
- package/src/daemon/conversation-agent-loop.ts +231 -25
- package/src/daemon/conversation-attachments.ts +93 -12
- package/src/daemon/conversation-evictor.ts +5 -24
- package/src/daemon/conversation-history.ts +10 -0
- package/src/daemon/conversation-interrupt-repair.ts +4 -43
- package/src/daemon/conversation-interrupt.ts +7 -11
- package/src/daemon/conversation-messaging.ts +58 -5
- package/src/daemon/conversation-mode-session.test.ts +1667 -0
- package/src/daemon/conversation-mode-session.ts +1008 -0
- package/src/daemon/conversation-process.ts +5 -0
- package/src/daemon/conversation-runtime-assembly.ts +10 -20
- package/src/daemon/conversation-skill-tools.ts +1 -0
- package/src/daemon/conversation-store.ts +4 -4
- package/src/daemon/conversation-surface-snapshots.ts +135 -0
- package/src/daemon/conversation-surfaces.ts +113 -5
- package/src/daemon/conversation-tool-setup.ts +79 -6
- package/src/daemon/conversation-turn-finalize.ts +26 -13
- package/src/daemon/conversation.ts +215 -58
- package/src/daemon/daemon-control.ts +3 -7
- package/src/daemon/handlers/config-model.ts +7 -5
- package/src/daemon/handlers/shared.ts +55 -16
- package/src/daemon/host-cu-proxy.ts +51 -22
- package/src/daemon/lifecycle.ts +30 -13
- package/src/daemon/mcp-reload-service.ts +1 -1
- package/src/daemon/message-protocol.ts +0 -1
- package/src/daemon/message-provenance.ts +28 -4
- package/src/daemon/message-types/shared.ts +1 -0
- package/src/daemon/mode-session-startup-recovery.ts +23 -0
- package/src/daemon/mode-session-tracking.ts +54 -0
- package/src/daemon/orphan-reaper.ts +4 -3
- package/src/daemon/persist-media-references.ts +20 -3
- package/src/daemon/process-message.ts +13 -4
- package/src/daemon/tool-setup-types.ts +6 -0
- package/src/daemon/virtual-desktop-context.ts +21 -0
- package/src/daemon/wake-conversation-ops.ts +50 -15
- package/src/desktop/__tests__/fake-desktop.ts +1 -0
- package/src/desktop/desktop-accessibility-script.ts +88 -0
- package/src/desktop/desktop-accessibility.test.ts +160 -0
- package/src/desktop/desktop-accessibility.ts +169 -0
- package/src/desktop/desktop-automation-lease.test.ts +302 -64
- package/src/desktop/desktop-automation-lease.ts +129 -54
- package/src/desktop/desktop-browser-endpoint.ts +4 -0
- package/src/desktop/desktop-computer-use-routing.test.ts +330 -0
- package/src/desktop/desktop-computer-use.test.ts +449 -0
- package/src/desktop/desktop-computer-use.ts +502 -0
- package/src/desktop/desktop-dependencies.ts +47 -293
- package/src/desktop/desktop-display.ts +2 -2
- package/src/desktop/desktop-help.ts +15 -0
- package/src/desktop/desktop-panel-config.test.ts +13 -3
- package/src/desktop/desktop-panel-config.ts +109 -15
- package/src/desktop/desktop-session-manager.test.ts +69 -21
- package/src/desktop/desktop-session-manager.ts +76 -22
- package/src/desktop/desktop-stream-bridge.test.ts +76 -132
- package/src/desktop/desktop-stream-bridge.ts +9 -11
- package/src/desktop/desktop-wallpaper-renderer.ts +132 -0
- package/src/desktop/desktop-wallpaper-worker.ts +17 -0
- package/src/desktop/desktop-wallpaper.test.ts +48 -7
- package/src/desktop/desktop-wallpaper.ts +34 -102
- package/src/desktop/desktop-window-theme.ts +5 -25
- package/src/desktop/virtual-desktop-feature.ts +53 -3
- package/src/desktop/virtual-desktop-platform.test.ts +5 -9
- package/src/documents/document-store.ts +23 -0
- package/src/heartbeat/__tests__/heartbeat-service.test.ts +22 -26
- package/src/heartbeat/heartbeat-service.ts +5 -33
- package/src/ipc/__tests__/browser-ipc.test.ts +215 -6
- package/src/ipc/assistant-server.ts +2 -0
- package/src/ipc/routes/__tests__/channel-reply-ipc-routes.test.ts +111 -0
- package/src/ipc/routes/channel-reply-ipc-routes.ts +73 -0
- package/src/live-voice/__tests__/continuation-judge.test.ts +151 -0
- package/src/live-voice/__tests__/live-activity-reporter.test.ts +154 -15
- package/src/live-voice/__tests__/live-voice-flux-turn-end.test.ts +743 -31
- package/src/live-voice/__tests__/live-voice-look-follow-up.test.ts +191 -10
- package/src/live-voice/__tests__/live-voice-session-groups-gate.test.ts +87 -0
- package/src/live-voice/__tests__/live-voice-session-manager.test.ts +5 -1
- package/src/live-voice/__tests__/live-voice-sight-frame-echo.test.ts +41 -0
- package/src/live-voice/__tests__/live-voice-sight-frame.test.ts +1017 -9
- package/src/live-voice/__tests__/live-voice-triage-escalate.test.ts +224 -1
- package/src/live-voice/__tests__/live-voice-tts-session.test.ts +101 -0
- package/src/live-voice/__tests__/live-voice-tts.test.ts +44 -0
- package/src/live-voice/__tests__/live-voice-vad.test.ts +939 -75
- package/src/live-voice/__tests__/microphone-idle-gate.test.ts +87 -0
- package/src/live-voice/__tests__/session-controls.test.ts +23 -0
- package/src/live-voice/__tests__/task-outcomes.test.ts +117 -0
- package/src/live-voice/__tests__/voice-input-diagnostics.test.ts +117 -0
- package/src/live-voice/camera-mode-session.test.ts +265 -0
- package/src/live-voice/camera-mode-session.ts +277 -0
- package/src/live-voice/continuation-judge.ts +123 -0
- package/src/live-voice/live-activity-reporter.ts +115 -6
- package/src/live-voice/live-voice-manager.ts +36 -1
- package/src/live-voice/live-voice-photo.ts +88 -7
- package/src/live-voice/live-voice-session-manager.ts +15 -0
- package/src/live-voice/live-voice-session.ts +1443 -747
- package/src/live-voice/live-voice-tts.ts +22 -4
- package/src/live-voice/microphone-idle-gate.ts +56 -0
- package/src/live-voice/protocol.ts +171 -4
- package/src/live-voice/session-controls.ts +11 -3
- package/src/live-voice/task-outcomes.ts +131 -0
- package/src/live-voice/voice-input-diagnostics.ts +117 -0
- package/src/mcp/__tests__/manager-tool-caps.test.ts +156 -0
- package/src/mcp/__tests__/startup.test.ts +30 -7
- package/src/mcp/__tests__/tool-caps.test.ts +134 -0
- package/src/mcp/manager.ts +171 -101
- package/src/mcp/startup.ts +33 -13
- package/src/mcp/tool-caps.ts +184 -0
- package/src/media/__tests__/image-models.test.ts +58 -0
- package/src/media/image-credentials.ts +5 -0
- package/src/media/image-models.ts +52 -0
- package/src/media/image-service.ts +8 -0
- package/src/media/openrouter-image-service.test.ts +133 -0
- package/src/media/openrouter-image-service.ts +153 -0
- package/src/media/types.ts +1 -1
- package/src/messaging/provider-message-metadata.ts +3 -3
- package/src/messaging/providers/__tests__/transport-dispatch.test.ts +65 -11
- package/src/messaging/providers/discord/send.ts +4 -17
- package/src/messaging/providers/discord/transport.ts +2 -2
- package/src/messaging/providers/outlook/client.ts +16 -22
- package/src/messaging/providers/telegram-bot/send.test.ts +236 -1
- package/src/messaging/providers/telegram-bot/send.ts +10 -7
- package/src/messaging/providers/telegram-bot/transport.ts +3 -10
- package/src/messaging/providers/whatsapp/send.test.ts +169 -4
- package/src/monitoring/__tests__/control.test.ts +17 -5
- package/src/monitoring/control.ts +15 -5
- package/src/notifications/AGENTS.md +1 -1
- package/src/notifications/README.md +83 -99
- package/src/notifications/__tests__/broadcaster.test.ts +1 -1
- package/src/notifications/__tests__/channel-allowlist.test.ts +39 -0
- package/src/notifications/__tests__/copy-composer.test.ts +71 -1
- package/src/notifications/__tests__/decision-engine.test.ts +79 -2
- package/src/notifications/__tests__/deterministic-checks.test.ts +10 -10
- package/src/notifications/__tests__/home-feed-side-effect.test.ts +93 -4
- package/src/notifications/__tests__/notification-utils.test.ts +18 -0
- package/src/notifications/adapters/macos.ts +37 -46
- package/src/notifications/broadcaster.ts +5 -6
- package/src/notifications/channel-allowlist.ts +44 -0
- package/src/notifications/copy-composer.ts +11 -26
- package/src/notifications/decision-engine.ts +24 -14
- package/src/notifications/deterministic-checks.ts +1 -1
- package/src/notifications/emit-signal.ts +123 -37
- package/src/notifications/home-feed-side-effect.ts +9 -7
- package/src/notifications/notification-utils.ts +19 -0
- package/src/notifications/schedule-result-producer.ts +4 -6
- package/src/notifications/signal.ts +4 -19
- package/src/notifications/types.ts +15 -0
- package/src/oauth/AGENTS.md +2 -0
- package/src/oauth/__tests__/identity-verifier.test.ts +36 -1
- package/src/oauth/connection-resolver.ts +1 -2
- package/src/oauth/identity-verifier.ts +24 -0
- package/src/oauth/oauth-store.ts +12 -4
- package/src/oauth/provider-serializer.ts +8 -0
- package/src/oauth/seed-providers.ts +42 -0
- package/src/onboarding/onboarding-events-store.ts +99 -12
- package/src/permissions/confirmation-guardian-request.test.ts +16 -0
- package/src/permissions/confirmation-guardian-request.ts +2 -3
- package/src/permissions/question-prompter.test.ts +31 -0
- package/src/permissions/question-prompter.ts +2 -0
- package/src/persistence/__tests__/client-connection-events-store.test.ts +322 -0
- package/src/persistence/attachments-store.ts +84 -44
- package/src/persistence/client-connection-events-store.ts +377 -0
- package/src/persistence/conversation-crud.ts +166 -26
- package/src/persistence/conversation-mode-sessions.test.ts +424 -0
- package/src/persistence/conversation-mode-sessions.ts +460 -0
- package/src/persistence/conversation-plugin-facade.ts +30 -0
- package/src/persistence/conversation-queries.ts +6 -19
- package/src/persistence/conversation-tool-surface.ts +135 -0
- package/src/persistence/conversation-types.test.ts +32 -0
- package/src/persistence/conversation-types.ts +26 -4
- package/src/persistence/llm-usage-store.ts +3 -9
- package/src/persistence/message-metadata.ts +25 -0
- package/src/persistence/migrations/378-create-conversation-tool-surfaces.test.ts +78 -0
- package/src/persistence/migrations/378-create-conversation-tool-surfaces.ts +29 -0
- package/src/persistence/migrations/379-oauth-providers-response-ok-field.test.ts +92 -0
- package/src/persistence/migrations/379-oauth-providers-response-ok-field.ts +32 -0
- package/src/persistence/migrations/380-conversation-tool-surfaces-delegate-independent-tasks.test.ts +90 -0
- package/src/persistence/migrations/380-conversation-tool-surfaces-delegate-independent-tasks.ts +34 -0
- package/src/persistence/migrations/381-create-conversation-mode-sessions.ts +41 -0
- package/src/persistence/migrations/382-create-client-connection-events.ts +40 -0
- package/src/persistence/migrations/__tests__/382-create-client-connection-events.test.ts +56 -0
- package/src/persistence/schema/conversation-tool-surfaces.ts +32 -0
- package/src/persistence/schema/conversations.ts +67 -0
- package/src/persistence/schema/index.ts +1 -0
- package/src/persistence/schema/infrastructure.ts +29 -0
- package/src/persistence/schema/oauth.ts +1 -0
- package/src/persistence/steps.ts +41 -3
- package/src/plugin-api/__tests__/plugin-name-env.test.ts +249 -0
- package/src/plugin-api/conversation-turn.ts +31 -7
- package/src/plugin-api/effective-context-window.ts +33 -0
- package/src/plugin-api/index.ts +35 -6
- package/src/plugin-api/plugin-channel-turn-trust.test.ts +133 -0
- package/src/plugin-api/plugin-channel-turn-trust.ts +71 -0
- package/src/plugin-api/plugin-name-env.ts +125 -0
- package/src/plugin-api/resolve-credential.ts +79 -20
- package/src/plugin-api/vision-support.test.ts +22 -0
- package/src/plugin-api/vision-support.ts +17 -5
- package/src/plugins/__tests__/active-plugin-env.test.ts +136 -0
- package/src/plugins/active-plugin-env.ts +51 -0
- package/src/plugins/defaults/index.ts +24 -0
- package/src/plugins/defaults/memory/AGENTS.md +39 -5
- package/src/plugins/defaults/memory/__tests__/archive-removal-framing.test.ts +64 -0
- package/src/plugins/defaults/memory/__tests__/buffer-file.test.ts +365 -0
- package/src/plugins/defaults/memory/__tests__/buffer-format.test.ts +43 -0
- package/src/plugins/defaults/memory/__tests__/conversation-memory-purge.test.ts +1 -0
- package/src/plugins/defaults/memory/__tests__/db-memory-attach.test.ts +2 -0
- package/src/plugins/defaults/memory/__tests__/fixtures/buffer-appender.ts +17 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-job.test.ts +74 -0
- package/src/plugins/defaults/memory/__tests__/memory-run-evidence.test.ts +161 -0
- package/src/plugins/defaults/memory/__tests__/relocated-memory-test-rows.ts +10 -0
- package/src/plugins/defaults/memory/__tests__/worker-control.test.ts +19 -6
- package/src/plugins/defaults/memory/buffer-file.ts +354 -0
- package/src/plugins/defaults/memory/buffer-format.ts +40 -0
- package/src/plugins/defaults/memory/context-search/agent-protocol.ts +9 -12
- package/src/plugins/defaults/memory/context-search/agent-runner.ts +77 -22
- package/src/plugins/defaults/memory/context-search/format.ts +53 -8
- package/src/plugins/defaults/memory/context-search/limits.ts +10 -12
- package/src/plugins/defaults/memory/context-search/recall-input.ts +38 -0
- package/src/plugins/defaults/memory/context-search/search.ts +8 -14
- package/src/plugins/defaults/memory/context-search/sources/conversations.ts +1 -0
- package/src/plugins/defaults/memory/context-search/sources/memory-v2.ts +2 -1
- package/src/plugins/defaults/memory/context-search/sources/workspace.ts +2 -1
- package/src/plugins/defaults/memory/context-search/types.ts +1 -11
- package/src/plugins/defaults/memory/conversation-memory-purge.ts +4 -0
- package/src/plugins/defaults/memory/graph/__tests__/handle-remember-v2.test.ts +11 -4
- package/src/plugins/defaults/memory/graph/capability-seed.ts +1 -2
- package/src/plugins/defaults/memory/graph/tool-handlers.ts +13 -63
- package/src/plugins/defaults/memory/graph/tools.ts +35 -64
- package/src/plugins/defaults/memory/host-utils.ts +0 -10
- package/src/plugins/defaults/memory/injectors.ts +4 -3
- package/src/plugins/defaults/memory/memory-retrospective-job.ts +72 -183
- package/src/plugins/defaults/memory/memory-run-evidence.ts +213 -0
- package/src/plugins/defaults/memory/src/memory-eval-routes.ts +2 -3
- package/src/plugins/defaults/memory/src/memory-item-routes.test.ts +6 -1
- package/src/plugins/defaults/memory/src/memory-item-routes.ts +2 -1
- package/src/plugins/defaults/memory/substrate/__tests__/consolidation-job.test.ts +407 -99
- package/src/plugins/defaults/memory/substrate/__tests__/consolidation-prompt-flag-gating-guard.test.ts +10 -0
- package/src/plugins/defaults/memory/substrate/__tests__/prompts-consolidation.test.ts +107 -7
- package/src/plugins/defaults/memory/substrate/consolidation-job.ts +307 -86
- package/src/plugins/defaults/memory/substrate/consolidation-tool-surface.ts +34 -0
- package/src/plugins/defaults/memory/substrate/page-index.ts +2 -1
- package/src/plugins/defaults/memory/substrate/prompts/consolidation.ts +91 -51
- package/src/plugins/defaults/memory/substrate/sweep-job.ts +1 -1
- package/src/plugins/defaults/memory/tools.test.ts +124 -9
- package/src/plugins/defaults/memory/tools.ts +38 -11
- package/src/plugins/defaults/memory/v1/graph/consolidation.ts +2 -2
- package/src/plugins/defaults/memory/v1/graph/extraction.ts +2 -1
- package/src/plugins/defaults/memory/v1/graph/retriever.ts +1 -1
- package/src/plugins/defaults/memory/v2/__tests__/migration.test.ts +5 -0
- package/src/plugins/defaults/memory/v2/__tests__/reranker.test.ts +5 -2
- package/src/plugins/defaults/memory/v2/reranker.ts +2 -1
- package/src/plugins/defaults/memory/v3/__tests__/injection.test.ts +81 -1
- package/src/plugins/defaults/memory/v3/__tests__/orchestrate.test.ts +87 -0
- package/src/plugins/defaults/memory/v3/__tests__/plugin-schema.test.ts +12 -0
- package/src/plugins/defaults/memory/v3/__tests__/pool-log-store.test.ts +332 -15
- package/src/plugins/defaults/memory/v3/__tests__/pool-select.test.ts +626 -27
- package/src/plugins/defaults/memory/v3/__tests__/shadow-plugin.test.ts +148 -28
- package/src/plugins/defaults/memory/v3/card.ts +2 -1
- package/src/plugins/defaults/memory/v3/injector.ts +212 -178
- package/src/plugins/defaults/memory/v3/orchestrate.ts +159 -32
- package/src/plugins/defaults/memory/v3/plugin-schema.ts +54 -4
- package/src/plugins/defaults/memory/v3/pool-log-store.ts +275 -37
- package/src/plugins/defaults/memory/v3/pool-select.test.ts +47 -5
- package/src/plugins/defaults/memory/v3/pool-select.ts +710 -59
- package/src/plugins/defaults/memory/v3/sections.ts +2 -1
- package/src/plugins/defaults/memory/v3/shadow-plugin.ts +41 -14
- package/src/plugins/defaults/memory/worker-control.ts +15 -5
- package/src/plugins/defaults/task-progress/__tests__/registration.test.ts +70 -0
- package/src/plugins/defaults/task-progress/__tests__/task-progress-context.test.ts +380 -0
- package/src/plugins/defaults/task-progress/hooks/post-compact.ts +19 -0
- package/src/plugins/defaults/task-progress/hooks/user-prompt-submit.ts +24 -0
- package/src/plugins/defaults/task-progress/package.json +14 -0
- package/src/plugins/defaults/task-progress/src/apply-task-progress-context.ts +93 -0
- package/src/plugins/defaults/task-progress/src/format-task-progress-context.ts +67 -0
- package/src/plugins/defaults/tool-error/hooks/post-tool-use.ts +4 -1
- package/src/plugins/defaults/tool-result-truncate/terminal.ts +1 -46
- package/src/prompts/__tests__/parallel-tasks-section.test.ts +25 -0
- package/src/prompts/delegation-gate.ts +57 -0
- package/src/prompts/system-prompt.ts +14 -32
- package/src/providers/__tests__/context-overflow-error.test.ts +10 -0
- package/src/providers/__tests__/search-provider-catalog.test.ts +18 -0
- package/src/providers/anthropic/client.ts +10 -1
- package/src/providers/anthropic-gateway-shared.ts +6 -0
- package/src/providers/fetch-provider-catalog.ts +11 -0
- package/src/providers/gemini/client.ts +25 -8
- package/src/providers/inference/adapter-factory.ts +6 -0
- package/src/providers/jev/client.test.ts +270 -0
- package/src/providers/jev/client.ts +534 -0
- package/src/providers/model-catalog.ts +59 -3
- package/src/providers/openai/__tests__/chat-completions-extra-body.test.ts +94 -1
- package/src/providers/openai/__tests__/chat-completions-provider-reasoning.test.ts +106 -1
- package/src/providers/openai/__tests__/chat-template-flatten.test.ts +24 -0
- package/src/providers/openai/__tests__/google-thought-signature.test.ts +1 -2
- package/src/providers/openai/chat-completions-provider.ts +100 -29
- package/src/providers/openai/google-thought-signature.ts +0 -6
- package/src/providers/openai/responses-provider.ts +23 -1
- package/src/providers/ratelimit.ts +1 -3
- package/src/providers/search-provider-catalog.ts +24 -0
- package/src/providers/server-tool-pairing.ts +16 -7
- package/src/providers/speech-to-text/__tests__/deepgram-flux-frames.test.ts +24 -0
- package/src/providers/speech-to-text/__tests__/deepgram-flux-realtime.test.ts +7 -1
- package/src/providers/speech-to-text/__tests__/vellum-managed-realtime.test.ts +50 -1
- package/src/providers/speech-to-text/deepgram-flux-frames.ts +11 -0
- package/src/providers/speech-to-text/vellum-managed-flux-realtime.ts +13 -0
- package/src/providers/types.ts +5 -0
- package/src/routes/control.ts +7 -5
- package/src/runtime/AGENTS.md +6 -4
- package/src/runtime/__tests__/agent-wake.test.ts +94 -1
- package/src/runtime/actor-trust-resolver.ts +3 -3
- package/src/runtime/agent-wake.ts +32 -5
- package/src/runtime/approval-message-composer.ts +3 -69
- package/src/runtime/assistant-event-hub.ts +73 -75
- package/src/runtime/assistant-event-publish-options.ts +7 -0
- package/src/runtime/assistant-event-targeting.ts +79 -0
- package/src/runtime/assistant-stream-state.ts +13 -100
- package/src/runtime/background-job-runner.ts +5 -5
- package/src/runtime/channel-retry-sweep.ts +1 -0
- package/src/runtime/guardian-action-message-composer.ts +8 -161
- package/src/runtime/guardian-action-service.ts +2 -17
- package/src/runtime/guardian-reply-router.ts +29 -40
- package/src/runtime/http-server.ts +2 -2
- package/src/runtime/http-types.ts +6 -0
- package/src/runtime/message-composer-types.ts +3 -61
- package/src/runtime/migrations/__tests__/vbundle-import-parity.test.ts +67 -1
- package/src/runtime/migrations/__tests__/vbundle-import-policy.test.ts +89 -0
- package/src/runtime/migrations/vbundle-builder.ts +24 -1
- package/src/runtime/migrations/vbundle-import-analyzer.ts +13 -4
- package/src/runtime/migrations/vbundle-import-policy.ts +43 -11
- package/src/runtime/migrations/vbundle-importer.ts +4 -1
- package/src/runtime/migrations/vbundle-streaming-importer.ts +4 -1
- package/src/runtime/migrations/vbundle-validator.ts +1 -0
- package/src/runtime/pending-interactions-mode-session.test.ts +62 -0
- package/src/runtime/pending-interactions.ts +29 -0
- package/src/runtime/routes/__tests__/client-history-routes.test.ts +104 -0
- package/src/runtime/routes/__tests__/image-generation-routes.test.ts +97 -0
- package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +24 -2
- package/src/runtime/routes/__tests__/roadmap-routes.test.ts +29 -6
- package/src/runtime/routes/browser-context.ts +31 -11
- package/src/runtime/routes/browser-routes.ts +43 -4
- package/src/runtime/routes/btw-routes.ts +9 -4
- package/src/runtime/routes/channel-route-shared.ts +1 -9
- package/src/runtime/routes/client-routes.ts +108 -0
- package/src/runtime/routes/conversation-management-routes.ts +4 -1
- package/src/runtime/routes/conversation-routes.ts +125 -14
- package/src/runtime/routes/credential-prompt-routes.ts +4 -3
- package/src/runtime/routes/credential-routes.ts +1 -2
- package/src/runtime/routes/desktop-setup-routes.test.ts +9 -18
- package/src/runtime/routes/desktop-setup-routes.ts +8 -7
- package/src/runtime/routes/documents-routes.ts +51 -0
- package/src/runtime/routes/email-routes.ts +3 -7
- package/src/runtime/routes/events-routes.ts +75 -54
- package/src/runtime/routes/guardian-approval-interception.ts +24 -0
- package/src/runtime/routes/identity-routes.ts +1 -5
- package/src/runtime/routes/image-generation-routes.ts +8 -14
- package/src/runtime/routes/inbound-message-handler.ts +37 -41
- package/src/runtime/routes/inbound-stages/backfill-sender-provenance.test.ts +168 -0
- package/src/runtime/routes/inbound-stages/backfill-sender-provenance.ts +96 -0
- package/src/runtime/routes/inbound-stages/background-dispatch.test.ts +39 -10
- package/src/runtime/routes/inbound-stages/background-dispatch.ts +9 -9
- package/src/runtime/routes/inbound-stages/reaction-intercept.ts +2 -0
- package/src/runtime/routes/inference-profile-session-handler.ts +11 -1
- package/src/runtime/routes/inference-profiles-routes.ts +9 -0
- package/src/runtime/routes/llm-context-normalization.ts +1 -7
- package/src/runtime/routes/migration-routes.ts +73 -2
- package/src/runtime/routes/notification-routes.ts +10 -0
- package/src/runtime/routes/oauth-commands-routes.ts +57 -54
- package/src/runtime/routes/oauth-providers.ts +11 -0
- package/src/runtime/routes/oauth-request-hints.test.ts +246 -0
- package/src/runtime/routes/oauth-request-hints.ts +149 -0
- package/src/runtime/routes/plugins-routes.ts +6 -0
- package/src/runtime/routes/roadmap-routes.ts +47 -8
- package/src/runtime/routes/schedule-routes.ts +2 -0
- package/src/runtime/routes/secret-routes.ts +99 -119
- package/src/runtime/routes/settings-routes.ts +10 -4
- package/src/runtime/routes/watcher-routes.ts +1 -3
- package/src/runtime/trust-class.test.ts +7 -6
- package/src/runtime/trust-class.ts +9 -17
- package/src/schedule/__tests__/worker-control.test.ts +36 -5
- package/src/schedule/__tests__/worker-mcp-bootstrap.test.ts +107 -0
- package/src/schedule/__tests__/worker-mcp-readiness.test.ts +173 -0
- package/src/schedule/__tests__/worker-mcp-tools.test.ts +13 -47
- package/src/schedule/run-script.ts +2 -2
- package/src/schedule/scheduler.ts +47 -11
- package/src/schedule/tool-surface-readiness.ts +65 -0
- package/src/schedule/worker-control.ts +15 -5
- package/src/schedule/worker-mcp.ts +85 -0
- package/src/schedule/worker.ts +9 -31
- package/src/security/secure-keys.ts +74 -112
- package/src/skills/clawhub.ts +3 -8
- package/src/skills/managed-store.ts +1 -5
- package/src/stt/types.ts +4 -0
- package/src/subagent/notify.ts +54 -34
- package/src/subagent/parent-notification.ts +6 -0
- package/src/telemetry/__tests__/phone-call-funnel.test.ts +72 -0
- package/src/telemetry/phone-call-funnel.ts +144 -0
- package/src/tools/__tests__/tool-schema-root-combinator-guard.test.ts +81 -0
- package/src/tools/ask-question/ask-question-tool.test.ts +167 -5
- package/src/tools/ask-question/ask-question-tool.ts +120 -22
- package/src/tools/browser/browser-execution.ts +29 -60
- package/src/tools/browser/cdp-client/__tests__/cdp-dom-helpers.test.ts +561 -39
- package/src/tools/browser/cdp-client/cdp-dom-helpers.ts +435 -39
- package/src/tools/browser/cdp-client/cdp-inspect-client.ts +1 -8
- package/src/tools/client-os.ts +15 -0
- package/src/tools/computer-use/skill-proxy-bridge.ts +1 -1
- package/src/tools/computer-use/target.ts +42 -0
- package/src/tools/document/document-tool.ts +13 -18
- package/src/tools/execution-target.ts +14 -16
- package/src/tools/host-terminal/host-shell.ts +16 -7
- package/src/tools/network/__tests__/web-fetch-tinyfish.test.ts +250 -0
- package/src/tools/network/__tests__/web-search.test.ts +203 -0
- package/src/tools/network/web-fetch.ts +274 -7
- package/src/tools/network/web-search.ts +468 -6
- package/src/tools/permission-checker.ts +3 -3
- package/src/tools/policy-context.ts +3 -1
- package/src/tools/shared/filesystem/file-ops-service.ts +1 -31
- package/src/tools/shared/shell-output.test.ts +10 -0
- package/src/tools/shared/shell-output.ts +14 -2
- package/src/tools/skills/load.ts +2 -1
- package/src/tools/skills/sandbox-runner.ts +24 -4
- package/src/tools/skills/scaffold-managed.ts +2 -1
- package/src/tools/skills/skill-script-runner.ts +7 -0
- package/src/tools/skills/skill-tool-factory.ts +15 -1
- package/src/tools/terminal/__tests__/safe-env.test.ts +44 -4
- package/src/tools/terminal/__tests__/sanitized-bash.test.ts +0 -14
- package/src/tools/terminal/safe-env.ts +39 -24
- package/src/tools/terminal/sanitized-bash.ts +15 -2
- package/src/tools/terminal/shell-launch.test.ts +162 -0
- package/src/tools/terminal/shell.test.ts +29 -0
- package/src/tools/terminal/shell.ts +15 -7
- package/src/tools/tool-approval-handler.ts +5 -1
- package/src/tools/tool-types.ts +3 -3
- package/src/tools/types.ts +8 -2
- package/src/util/__tests__/worker-process-command.test.ts +5 -4
- package/src/util/abort-reasons.ts +18 -26
- package/src/util/browser-human-verification.ts +14 -0
- package/src/util/canonicalize-identity.test.ts +21 -0
- package/src/util/canonicalize-identity.ts +16 -0
- package/src/util/errors.ts +13 -3
- package/src/util/host-process.test.ts +17 -1
- package/src/util/host-process.ts +24 -0
- package/src/util/provider-error-patterns.ts +3 -0
- package/src/util/unicode.ts +29 -0
- package/src/util/worker-process.ts +77 -35
- package/src/workspace/migrations/157-add-desktop-files-launcher.ts +94 -0
- package/src/workspace/migrations/__tests__/156-extract-workspace-mcp-json.test.ts +4 -10
- package/src/workspace/migrations/__tests__/157-add-desktop-files-launcher.test.ts +113 -0
- package/src/workspace/migrations/registry.ts +2 -0
- package/src/__tests__/computer-use-window-schema.test.ts +0 -24
- package/src/api/events/heartbeat-alert.ts +0 -20
- package/src/daemon/message-types/web-activity.ts +0 -65
- package/src/desktop/desktop-dependencies.test.ts +0 -131
- package/src/tools/computer-use/definitions.ts +0 -607
|
@@ -27,7 +27,10 @@ import {
|
|
|
27
27
|
type InstalledPluginInfo,
|
|
28
28
|
readInstalledPlugin,
|
|
29
29
|
} from "./list-installed-plugins.js";
|
|
30
|
-
import {
|
|
30
|
+
import {
|
|
31
|
+
arePlatformFeaturesEnabled,
|
|
32
|
+
readBundledLocalPluginCatalog,
|
|
33
|
+
} from "./plugin-catalog-local.js";
|
|
31
34
|
import { DEFAULT_PLUGIN_REF } from "./plugin-constants.js";
|
|
32
35
|
import {
|
|
33
36
|
compareFingerprint,
|
|
@@ -338,17 +341,21 @@ export async function inspectPlugin(
|
|
|
338
341
|
let remote: PluginRemoteInfo | null = null;
|
|
339
342
|
let remoteError: string | null = null;
|
|
340
343
|
const installedLocalSource = local?.source?.kind === "local";
|
|
344
|
+
const useBundledLocalCatalog =
|
|
345
|
+
installedLocalSource || (!installed && !arePlatformFeaturesEnabled());
|
|
341
346
|
try {
|
|
342
|
-
if (
|
|
347
|
+
if (useBundledLocalCatalog) {
|
|
343
348
|
const catalog = deps.localCatalog ?? readBundledLocalPluginCatalog();
|
|
344
349
|
const match = catalog.matches.find(
|
|
345
350
|
(candidate) =>
|
|
346
351
|
candidate.name === name &&
|
|
347
352
|
candidate.source.kind === "local" &&
|
|
348
|
-
|
|
353
|
+
(!installedLocalSource ||
|
|
354
|
+
candidate.source.path === local.source?.path),
|
|
349
355
|
);
|
|
350
356
|
remote = match ? readLocalPackageRemote(match, catalog.ref) : null;
|
|
351
|
-
}
|
|
357
|
+
}
|
|
358
|
+
if (!remote && !installedLocalSource) {
|
|
352
359
|
const entries = await fetchMarketplaceEntries(
|
|
353
360
|
{ fetch: deps.fetch },
|
|
354
361
|
{ ref: marketplaceRef },
|
|
@@ -127,6 +127,14 @@ export const mcpIntegrationSchema = z.object({
|
|
|
127
127
|
}),
|
|
128
128
|
/** Filename under the web client's bundled integration-image directory. */
|
|
129
129
|
logo: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9_.-]*$/),
|
|
130
|
+
/**
|
|
131
|
+
* Where the integrations catalog files the entry. Any string at this
|
|
132
|
+
* boundary: a catalog newer than this build can name a category it does not
|
|
133
|
+
* know, and one unfamiliar slug must not reject the whole catalog. Clients
|
|
134
|
+
* normalize against the shared vocabulary; the bundled inventory test holds
|
|
135
|
+
* every entry in this repo to it.
|
|
136
|
+
*/
|
|
137
|
+
category: z.string().optional(),
|
|
130
138
|
/** Existing main OAuth provider used to group a related MCP connection. */
|
|
131
139
|
oauthProvider: z.string().regex(PLUGIN_NAME_RE).optional(),
|
|
132
140
|
});
|
|
@@ -412,6 +412,24 @@ export async function upgradePlugin(
|
|
|
412
412
|
};
|
|
413
413
|
}
|
|
414
414
|
|
|
415
|
+
const usesMergeStrategy =
|
|
416
|
+
strategy === "ours" || strategy === "theirs" || strategy === "assistant";
|
|
417
|
+
const canOverwriteCleanBundledInstall =
|
|
418
|
+
remote.kind === "local" &&
|
|
419
|
+
strategy === "theirs" &&
|
|
420
|
+
local.localChanges?.clean === true;
|
|
421
|
+
|
|
422
|
+
if (
|
|
423
|
+
remote.kind === "local" &&
|
|
424
|
+
usesMergeStrategy &&
|
|
425
|
+
!canOverwriteCleanBundledInstall
|
|
426
|
+
) {
|
|
427
|
+
throw new PluginMergeBaselineError(
|
|
428
|
+
name,
|
|
429
|
+
"bundled packages do not retain the previous package version needed for a three-way merge",
|
|
430
|
+
);
|
|
431
|
+
}
|
|
432
|
+
|
|
415
433
|
if (dryRun) {
|
|
416
434
|
return {
|
|
417
435
|
name,
|
|
@@ -433,17 +451,7 @@ export async function upgradePlugin(
|
|
|
433
451
|
// `ours`/`theirs`/`assistant` carry local edits forward via a three-way
|
|
434
452
|
// merge; the default `overwrite` discards them and re-installs the pin
|
|
435
453
|
// wholesale.
|
|
436
|
-
if (
|
|
437
|
-
strategy === "ours" ||
|
|
438
|
-
strategy === "theirs" ||
|
|
439
|
-
strategy === "assistant"
|
|
440
|
-
) {
|
|
441
|
-
if (remote.kind === "local") {
|
|
442
|
-
throw new PluginMergeBaselineError(
|
|
443
|
-
name,
|
|
444
|
-
"bundled packages do not retain the previous package version needed for a three-way merge",
|
|
445
|
-
);
|
|
446
|
-
}
|
|
454
|
+
if (usesMergeStrategy && remote.kind !== "local") {
|
|
447
455
|
const [remoteOwner, remoteRepo] = remote.repo.split("/");
|
|
448
456
|
return mergeUpgrade(
|
|
449
457
|
{
|
package/src/cli/output.ts
CHANGED
|
@@ -11,20 +11,34 @@ export function shouldOutputJson(cmd: Command): boolean {
|
|
|
11
11
|
return false;
|
|
12
12
|
}
|
|
13
13
|
|
|
14
|
-
|
|
14
|
+
/**
|
|
15
|
+
* Write a JSON payload to stdout. `onFlushed` runs once the bytes have left
|
|
16
|
+
* the process, for a command that must exit itself and would otherwise cut
|
|
17
|
+
* a piped payload short.
|
|
18
|
+
*/
|
|
19
|
+
export function writeOutput(
|
|
20
|
+
cmd: Command,
|
|
21
|
+
payload: unknown,
|
|
22
|
+
onFlushed?: () => void,
|
|
23
|
+
): void {
|
|
15
24
|
const compact = shouldOutputJson(cmd);
|
|
16
25
|
process.stdout.write(
|
|
17
26
|
compact
|
|
18
27
|
? JSON.stringify(payload) + "\n"
|
|
19
28
|
: JSON.stringify(payload, null, 2) + "\n",
|
|
29
|
+
onFlushed,
|
|
20
30
|
);
|
|
21
31
|
}
|
|
22
32
|
|
|
23
33
|
/** Format-aware error output: JSON envelope with --json, stderr otherwise. */
|
|
24
|
-
export function writeError(
|
|
34
|
+
export function writeError(
|
|
35
|
+
cmd: Command,
|
|
36
|
+
message: string,
|
|
37
|
+
onFlushed?: () => void,
|
|
38
|
+
): void {
|
|
25
39
|
if (shouldOutputJson(cmd)) {
|
|
26
|
-
writeOutput(cmd, { ok: false, error: message });
|
|
40
|
+
writeOutput(cmd, { ok: false, error: message }, onFlushed);
|
|
27
41
|
} else {
|
|
28
|
-
process.stderr.write(`Error: ${message}\n
|
|
42
|
+
process.stderr.write(`Error: ${message}\n`, onFlushed);
|
|
29
43
|
}
|
|
30
44
|
}
|
|
@@ -72,6 +72,12 @@ describe("LLMSchema.defaultProvider", () => {
|
|
|
72
72
|
).toThrow();
|
|
73
73
|
});
|
|
74
74
|
|
|
75
|
+
test("rejects a structured-decision catalog provider", () => {
|
|
76
|
+
expect(() =>
|
|
77
|
+
DefaultProviderSchema.parse({ provider: "typesafe" }),
|
|
78
|
+
).toThrow();
|
|
79
|
+
});
|
|
80
|
+
|
|
75
81
|
test("rejects an empty connectionName", () => {
|
|
76
82
|
expect(() =>
|
|
77
83
|
DefaultProviderSchema.parse({
|
|
@@ -12,10 +12,11 @@ import { describe, expect, test } from "bun:test";
|
|
|
12
12
|
import { MemoryRetrospectiveConfigSchema } from "../schemas/memory-retrospective.js";
|
|
13
13
|
|
|
14
14
|
describe("memory.retrospective config schema", () => {
|
|
15
|
-
test("an empty block leaves
|
|
15
|
+
test("an empty block leaves skill improvement on and monitoring off", () => {
|
|
16
16
|
const parsed = MemoryRetrospectiveConfigSchema.parse({});
|
|
17
17
|
expect(parsed.enabled).toBe(true);
|
|
18
18
|
expect(parsed.skillImprovement).toBe(true);
|
|
19
|
+
expect(parsed.skillImprovementMonitoring).toBe(false);
|
|
19
20
|
expect(parsed).not.toHaveProperty("forkStrategy");
|
|
20
21
|
});
|
|
21
22
|
|
|
@@ -41,6 +42,18 @@ describe("memory.retrospective config schema", () => {
|
|
|
41
42
|
).toBe(false);
|
|
42
43
|
});
|
|
43
44
|
|
|
45
|
+
test("skillImprovementMonitoring is a boolean-only opt-in", () => {
|
|
46
|
+
const parsed = MemoryRetrospectiveConfigSchema.parse({
|
|
47
|
+
skillImprovementMonitoring: true,
|
|
48
|
+
});
|
|
49
|
+
expect(parsed.skillImprovementMonitoring).toBe(true);
|
|
50
|
+
expect(
|
|
51
|
+
MemoryRetrospectiveConfigSchema.safeParse({
|
|
52
|
+
skillImprovementMonitoring: "true",
|
|
53
|
+
}).success,
|
|
54
|
+
).toBe(false);
|
|
55
|
+
});
|
|
56
|
+
|
|
44
57
|
test("a leftover forkStrategy key is ignored", () => {
|
|
45
58
|
const parsed = MemoryRetrospectiveConfigSchema.parse({
|
|
46
59
|
forkStrategy: "cloning",
|
|
@@ -21,7 +21,7 @@ connected desktop. CU tools run through the main agent loop via HostCuProxy.
|
|
|
21
21
|
The skill is internally preactivated for conversations with a connected desktop client.
|
|
22
22
|
|
|
23
23
|
Tools in this skill are proxy tools. Execution is forwarded to a connected
|
|
24
|
-
desktop client
|
|
24
|
+
desktop client or handled by the assistant's virtual desktop.
|
|
25
25
|
|
|
26
26
|
## Observations
|
|
27
27
|
|
|
@@ -21,11 +21,16 @@
|
|
|
21
21
|
},
|
|
22
22
|
"full_tree": {
|
|
23
23
|
"type": "boolean",
|
|
24
|
-
"description": "
|
|
24
|
+
"description": "On connected personal computers, walk the accessibility tree to full depth. The virtual desktop uses a bounded tree; use screenshot coordinates when an element is not listed."
|
|
25
25
|
},
|
|
26
26
|
"target_client_id": {
|
|
27
27
|
"type": "string",
|
|
28
28
|
"description": "ID of the specific client to target. Required when multiple clients support host_cu; omit when only one is connected. Obtain IDs from `assistant clients list --capability host_cu`."
|
|
29
|
+
},
|
|
30
|
+
"target": {
|
|
31
|
+
"type": "string",
|
|
32
|
+
"enum": ["connected-computer", "assistant-desktop"],
|
|
33
|
+
"description": "Desktop to control. Platform-hosted web conversations default to assistant-desktop; native desktop apps and non-platform assistants default to connected-computer. Explicit target or target_client_id overrides the default. Use assistant-desktop from a native app only when explicitly requested; do not combine it with target_client_id."
|
|
29
34
|
}
|
|
30
35
|
},
|
|
31
36
|
"required": []
|
|
@@ -65,6 +70,15 @@
|
|
|
65
70
|
"target_client_id": {
|
|
66
71
|
"type": "string",
|
|
67
72
|
"description": "ID of the specific client to target. Required when multiple clients support host_cu; omit when only one is connected. Obtain IDs from `assistant clients list --capability host_cu`."
|
|
73
|
+
},
|
|
74
|
+
"target": {
|
|
75
|
+
"type": "string",
|
|
76
|
+
"enum": ["connected-computer", "assistant-desktop"],
|
|
77
|
+
"description": "Desktop to control. Platform-hosted web conversations default to assistant-desktop; native desktop apps and non-platform assistants default to connected-computer. Explicit target or target_client_id overrides the default. Use assistant-desktop from a native app only when explicitly requested; do not combine it with target_client_id."
|
|
78
|
+
},
|
|
79
|
+
"observation_id": {
|
|
80
|
+
"type": "string",
|
|
81
|
+
"description": "Required on assistant-desktop: the ID from its latest screenshot. Observe again after browser actions, user handoff, or interruption. Omit for connected-computer. For a sequence, applies to the whole batch."
|
|
68
82
|
}
|
|
69
83
|
},
|
|
70
84
|
"required": ["reasoning"]
|
|
@@ -91,6 +105,15 @@
|
|
|
91
105
|
"target_client_id": {
|
|
92
106
|
"type": "string",
|
|
93
107
|
"description": "ID of the specific client to target. Required when multiple clients support host_cu; omit when only one is connected. Obtain IDs from `assistant clients list --capability host_cu`."
|
|
108
|
+
},
|
|
109
|
+
"target": {
|
|
110
|
+
"type": "string",
|
|
111
|
+
"enum": ["connected-computer", "assistant-desktop"],
|
|
112
|
+
"description": "Desktop to control. Platform-hosted web conversations default to assistant-desktop; native desktop apps and non-platform assistants default to connected-computer. Explicit target or target_client_id overrides the default. Use assistant-desktop from a native app only when explicitly requested; do not combine it with target_client_id."
|
|
113
|
+
},
|
|
114
|
+
"observation_id": {
|
|
115
|
+
"type": "string",
|
|
116
|
+
"description": "Required on assistant-desktop: the ID from its latest screenshot. Observe again after browser actions, user handoff, or interruption. Omit for connected-computer. For a sequence, applies to the whole batch."
|
|
94
117
|
}
|
|
95
118
|
},
|
|
96
119
|
"required": ["text", "reasoning"]
|
|
@@ -117,6 +140,15 @@
|
|
|
117
140
|
"target_client_id": {
|
|
118
141
|
"type": "string",
|
|
119
142
|
"description": "ID of the specific client to target. Required when multiple clients support host_cu; omit when only one is connected. Obtain IDs from `assistant clients list --capability host_cu`."
|
|
143
|
+
},
|
|
144
|
+
"target": {
|
|
145
|
+
"type": "string",
|
|
146
|
+
"enum": ["connected-computer", "assistant-desktop"],
|
|
147
|
+
"description": "Desktop to control. Platform-hosted web conversations default to assistant-desktop; native desktop apps and non-platform assistants default to connected-computer. Explicit target or target_client_id overrides the default. Use assistant-desktop from a native app only when explicitly requested; do not combine it with target_client_id."
|
|
148
|
+
},
|
|
149
|
+
"observation_id": {
|
|
150
|
+
"type": "string",
|
|
151
|
+
"description": "Required on assistant-desktop: the ID from its latest screenshot. Observe again after browser actions, user handoff, or interruption. Omit for connected-computer. For a sequence, applies to the whole batch."
|
|
120
152
|
}
|
|
121
153
|
},
|
|
122
154
|
"required": ["key", "reasoning"]
|
|
@@ -160,6 +192,15 @@
|
|
|
160
192
|
"target_client_id": {
|
|
161
193
|
"type": "string",
|
|
162
194
|
"description": "ID of the specific client to target. Required when multiple clients support host_cu; omit when only one is connected. Obtain IDs from `assistant clients list --capability host_cu`."
|
|
195
|
+
},
|
|
196
|
+
"target": {
|
|
197
|
+
"type": "string",
|
|
198
|
+
"enum": ["connected-computer", "assistant-desktop"],
|
|
199
|
+
"description": "Desktop to control. Platform-hosted web conversations default to assistant-desktop; native desktop apps and non-platform assistants default to connected-computer. Explicit target or target_client_id overrides the default. Use assistant-desktop from a native app only when explicitly requested; do not combine it with target_client_id."
|
|
200
|
+
},
|
|
201
|
+
"observation_id": {
|
|
202
|
+
"type": "string",
|
|
203
|
+
"description": "Required on assistant-desktop: the ID from its latest screenshot. Observe again after browser actions, user handoff, or interruption. Omit for connected-computer. For a sequence, applies to the whole batch."
|
|
163
204
|
}
|
|
164
205
|
},
|
|
165
206
|
"required": ["direction", "amount", "reasoning"]
|
|
@@ -206,6 +247,15 @@
|
|
|
206
247
|
"target_client_id": {
|
|
207
248
|
"type": "string",
|
|
208
249
|
"description": "ID of the specific client to target. Required when multiple clients support host_cu; omit when only one is connected. Obtain IDs from `assistant clients list --capability host_cu`."
|
|
250
|
+
},
|
|
251
|
+
"target": {
|
|
252
|
+
"type": "string",
|
|
253
|
+
"enum": ["connected-computer", "assistant-desktop"],
|
|
254
|
+
"description": "Desktop to control. Platform-hosted web conversations default to assistant-desktop; native desktop apps and non-platform assistants default to connected-computer. Explicit target or target_client_id overrides the default. Use assistant-desktop from a native app only when explicitly requested; do not combine it with target_client_id."
|
|
255
|
+
},
|
|
256
|
+
"observation_id": {
|
|
257
|
+
"type": "string",
|
|
258
|
+
"description": "Required on assistant-desktop: the ID from its latest screenshot. Observe again after browser actions, user handoff, or interruption. Omit for connected-computer. For a sequence, applies to the whole batch."
|
|
209
259
|
}
|
|
210
260
|
},
|
|
211
261
|
"required": ["reasoning"]
|
|
@@ -233,6 +283,15 @@
|
|
|
233
283
|
"target_client_id": {
|
|
234
284
|
"type": "string",
|
|
235
285
|
"description": "ID of the specific client to target. Required when multiple clients support host_cu; omit when only one is connected. Obtain IDs from `assistant clients list --capability host_cu`."
|
|
286
|
+
},
|
|
287
|
+
"target": {
|
|
288
|
+
"type": "string",
|
|
289
|
+
"enum": ["connected-computer", "assistant-desktop"],
|
|
290
|
+
"description": "Desktop to control. Platform-hosted web conversations default to assistant-desktop; native desktop apps and non-platform assistants default to connected-computer. Explicit target or target_client_id overrides the default. Use assistant-desktop from a native app only when explicitly requested; do not combine it with target_client_id."
|
|
291
|
+
},
|
|
292
|
+
"observation_id": {
|
|
293
|
+
"type": "string",
|
|
294
|
+
"description": "Required on assistant-desktop: the ID from its latest screenshot. Observe again after browser actions, user handoff, or interruption. Omit for connected-computer. For a sequence, applies to the whole batch."
|
|
236
295
|
}
|
|
237
296
|
},
|
|
238
297
|
"required": ["duration_ms", "reasoning"]
|
|
@@ -259,6 +318,11 @@
|
|
|
259
318
|
"target_client_id": {
|
|
260
319
|
"type": "string",
|
|
261
320
|
"description": "ID of the specific client to target. Required when multiple clients support host_cu; omit when only one is connected. Obtain IDs from `assistant clients list --capability host_cu`."
|
|
321
|
+
},
|
|
322
|
+
"target": {
|
|
323
|
+
"type": "string",
|
|
324
|
+
"enum": ["connected-computer", "assistant-desktop"],
|
|
325
|
+
"description": "Desktop to control. Platform-hosted web conversations default to assistant-desktop; native desktop apps and non-platform assistants default to connected-computer. Explicit target or target_client_id overrides the default. Use assistant-desktop from a native app only when explicitly requested; do not combine it with target_client_id."
|
|
262
326
|
}
|
|
263
327
|
},
|
|
264
328
|
"required": ["app_name", "reasoning"]
|
|
@@ -286,6 +350,11 @@
|
|
|
286
350
|
"target_client_id": {
|
|
287
351
|
"type": "string",
|
|
288
352
|
"description": "ID of the specific client to target. Required when multiple clients support host_cu; omit when only one is connected. Obtain IDs from `assistant clients list --capability host_cu`."
|
|
353
|
+
},
|
|
354
|
+
"target": {
|
|
355
|
+
"type": "string",
|
|
356
|
+
"enum": ["connected-computer", "assistant-desktop"],
|
|
357
|
+
"description": "Desktop to control. Platform-hosted web conversations default to assistant-desktop; native desktop apps and non-platform assistants default to connected-computer. Explicit target or target_client_id overrides the default. Use assistant-desktop from a native app only when explicitly requested; do not combine it with target_client_id."
|
|
289
358
|
}
|
|
290
359
|
},
|
|
291
360
|
"required": ["script", "reasoning"]
|
|
@@ -377,6 +446,15 @@
|
|
|
377
446
|
"target_client_id": {
|
|
378
447
|
"type": "string",
|
|
379
448
|
"description": "ID of the specific client to target. Required when multiple clients support host_cu; omit when only one is connected. Obtain IDs from `assistant clients list --capability host_cu`."
|
|
449
|
+
},
|
|
450
|
+
"target": {
|
|
451
|
+
"type": "string",
|
|
452
|
+
"enum": ["connected-computer", "assistant-desktop"],
|
|
453
|
+
"description": "Desktop to control. Platform-hosted web conversations default to assistant-desktop; native desktop apps and non-platform assistants default to connected-computer. Explicit target or target_client_id overrides the default. Use assistant-desktop from a native app only when explicitly requested; do not combine it with target_client_id."
|
|
454
|
+
},
|
|
455
|
+
"observation_id": {
|
|
456
|
+
"type": "string",
|
|
457
|
+
"description": "Required on assistant-desktop: the ID from its latest screenshot. Observe again after browser actions, user handoff, or interruption. Omit for connected-computer. For a sequence, applies to the whole batch."
|
|
380
458
|
}
|
|
381
459
|
},
|
|
382
460
|
"required": ["actions"]
|
|
@@ -396,6 +474,11 @@
|
|
|
396
474
|
"summary": {
|
|
397
475
|
"type": "string",
|
|
398
476
|
"description": "Human-readable summary of what was accomplished"
|
|
477
|
+
},
|
|
478
|
+
"target": {
|
|
479
|
+
"type": "string",
|
|
480
|
+
"enum": ["connected-computer", "assistant-desktop"],
|
|
481
|
+
"description": "Desktop to control. Platform-hosted web conversations default to assistant-desktop; native desktop apps and non-platform assistants default to connected-computer. Explicit target or target_client_id overrides the default. Use assistant-desktop from a native app only when explicitly requested; do not combine it with target_client_id."
|
|
399
482
|
}
|
|
400
483
|
},
|
|
401
484
|
"required": ["summary"]
|
|
@@ -418,6 +501,11 @@
|
|
|
418
501
|
"reasoning": {
|
|
419
502
|
"type": "string",
|
|
420
503
|
"description": "Explanation of how you determined the answer"
|
|
504
|
+
},
|
|
505
|
+
"target": {
|
|
506
|
+
"type": "string",
|
|
507
|
+
"enum": ["connected-computer", "assistant-desktop"],
|
|
508
|
+
"description": "Desktop to control. Platform-hosted web conversations default to assistant-desktop; native desktop apps and non-platform assistants default to connected-computer. Explicit target or target_client_id overrides the default. Use assistant-desktop from a native app only when explicitly requested; do not combine it with target_client_id."
|
|
421
509
|
}
|
|
422
510
|
},
|
|
423
511
|
"required": ["answer", "reasoning"]
|
|
@@ -37,13 +37,27 @@ Pass a concrete model ID only if the user names one explicitly. If the tool reje
|
|
|
37
37
|
Generate (no model parameter, default is correct):
|
|
38
38
|
|
|
39
39
|
```json
|
|
40
|
-
{
|
|
40
|
+
{
|
|
41
|
+
"tool": "media_generate_image",
|
|
42
|
+
"input": {
|
|
43
|
+
"prompt": "A sunset over the ocean, golden hour, soft haze, 35mm photo style",
|
|
44
|
+
"variants": 2
|
|
45
|
+
}
|
|
46
|
+
}
|
|
41
47
|
```
|
|
42
48
|
|
|
43
49
|
Edit:
|
|
44
50
|
|
|
45
51
|
```json
|
|
46
|
-
{
|
|
52
|
+
{
|
|
53
|
+
"tool": "media_generate_image",
|
|
54
|
+
"input": {
|
|
55
|
+
"prompt": "Remove the watermark text from the background. Keep the subject, framing, lighting, and colors exactly identical. Change nothing else.",
|
|
56
|
+
"mode": "edit",
|
|
57
|
+
"source_paths": ["conversations/<conv-id>/attachments/photo.jpeg"],
|
|
58
|
+
"model": "openai"
|
|
59
|
+
}
|
|
60
|
+
}
|
|
47
61
|
```
|
|
48
62
|
|
|
49
63
|
`source_paths` is a flat array of file path strings. Do NOT pass objects:
|
|
@@ -72,6 +86,7 @@ Edits on large photos are slow (1 to 2 minutes). If the tool reports a timeout (
|
|
|
72
86
|
Each generated image is saved into the workspace under `media/generated/` and the tool result lists the saved paths. The images also come back as inline content blocks so you can judge the result before presenting it.
|
|
73
87
|
|
|
74
88
|
- Present an image to the user by embedding its saved path in your reply: ``. The app renders it inline where your text refers to it, and chat channels (Slack, Telegram, WhatsApp) deliver it as a native image upload.
|
|
89
|
+
- Label a plain image link "View image", for example `[View image](vellum://workspace/media/generated/<file>.png)`. The link opens a preview; do not call it "Download" or promise a download in the surrounding text. To save the image, the user can choose Download in the preview.
|
|
75
90
|
- If you do not embed it, the image is still auto-attached to your reply as a file, so it is never lost. Prefer embedding: an attachment chip at the end of the message is a worse presentation than the image inline.
|
|
76
91
|
- To iterate on a result, pass its saved path via `source_paths` with `mode: "edit"`.
|
|
77
92
|
|
|
@@ -5,10 +5,7 @@ import {
|
|
|
5
5
|
resolveImageGenCredentials,
|
|
6
6
|
resolveImageGenRouting,
|
|
7
7
|
} from "../../../../media/image-credentials.js";
|
|
8
|
-
import {
|
|
9
|
-
describeImageModels,
|
|
10
|
-
resolveImageModel,
|
|
11
|
-
} from "../../../../media/image-models.js";
|
|
8
|
+
import { resolveRequestedImageModel } from "../../../../media/image-models.js";
|
|
12
9
|
import {
|
|
13
10
|
generateImage,
|
|
14
11
|
mapImageGenError,
|
|
@@ -115,18 +112,17 @@ export async function run(
|
|
|
115
112
|
const config = getConfig();
|
|
116
113
|
const svc = config.services["image-generation"];
|
|
117
114
|
let modelOverride = input.model;
|
|
118
|
-
// Resolve
|
|
119
|
-
//
|
|
120
|
-
// callers can self-correct without a stale schema enum.
|
|
115
|
+
// Resolve aliases and OpenRouter slugs against the configured provider.
|
|
116
|
+
// Built-in providers still list the current catalog on unknown values.
|
|
121
117
|
if (typeof modelOverride === "string" && modelOverride) {
|
|
122
|
-
const
|
|
123
|
-
if (
|
|
118
|
+
const resolved = resolveRequestedImageModel(modelOverride, svc.provider);
|
|
119
|
+
if (resolved.error) {
|
|
124
120
|
return {
|
|
125
|
-
content:
|
|
121
|
+
content: `${resolved.error}\n\nRetry with one of the aliases above, or omit the model parameter to use the configured default.`,
|
|
126
122
|
isError: true,
|
|
127
123
|
};
|
|
128
124
|
}
|
|
129
|
-
modelOverride =
|
|
125
|
+
modelOverride = resolved.model;
|
|
130
126
|
}
|
|
131
127
|
// Backend and managed-ness resolve together: an explicit model re-routes
|
|
132
128
|
// to the model's backend (e.g. `gpt-image-2` under a gemini config routes
|
|
@@ -249,18 +249,18 @@ If any required capability is missing:
|
|
|
249
249
|
|
|
250
250
|
Scheduled messages run without user interaction, in a conversation nobody has open. If the task produces output the user should see (a digest, summary, report, or a check whose answer is "nothing changed"), the scheduled message **must** end with an explicit instruction to deliver it. Without one, the output lives in a conversation log the user never opens.
|
|
251
251
|
|
|
252
|
-
Write the delivery step into the `message` when you create the schedule
|
|
252
|
+
Write the delivery step into the `message` when you create the schedule, not as a vague "let me know", but as the actual call, with a real title:
|
|
253
253
|
|
|
254
|
-
> "…then send the summary with `assistant notifications send --title \"Inbox digest\" --message \"<the summary>\"`."
|
|
254
|
+
> "…then send the summary with `assistant notifications send --source-channel scheduler --title \"Inbox digest\" --message \"<the summary>\"`."
|
|
255
255
|
|
|
256
256
|
A schedule whose message has no delivery step is not finished. Before calling `schedule_create` in `execute` mode, read your own message back and check that it says where the output goes.
|
|
257
257
|
|
|
258
|
-
There is a safety net, and it is not a substitute for the above. When an execute-mode run finishes with user-facing output and nothing delivered it, the assistant sends a notification carrying the run's final reply, so a schedule can no longer run and leave no trace. `messaging_send` counts as a delivery only when the call reported success, and `assistant notifications send` once the notification pipeline has taken it: a send the channel refused, or one that never reported at all, leaves the run eligible for the net, so the failure reaches you instead of vanishing into a conversation nobody has open. A Slack `chat.postMessage` never counts, because
|
|
258
|
+
There is a safety net, and it is not a substitute for the above. When an execute-mode run finishes with user-facing output and nothing delivered it, the assistant sends a notification carrying the run's final reply, so a schedule can no longer run and leave no trace. `messaging_send` counts as a delivery only when the call reported success, and `assistant notifications send` once the notification pipeline has taken it: a send the channel refused, or one that never reported at all, leaves the run eligible for the net, so the failure reaches you instead of vanishing into a conversation nobody has open. A Slack `chat.postMessage` never counts, because the net recognizes only the recorded routes; a run that posts Block Kit that way also gets the net's notification. It fires on the raw reply, whatever the run happened to end on, at whatever length. An authored delivery step gets a title and body you chose, sent at the moment you chose. Rely on the net and you get the machine's guess instead.
|
|
259
259
|
|
|
260
|
-
Choose the right delivery tool based on the
|
|
260
|
+
Choose the right delivery tool based on the destination:
|
|
261
261
|
|
|
262
|
-
- **
|
|
263
|
-
- **
|
|
262
|
+
- **A specific chat or mailbox** (a named Slack, Telegram, or Discord conversation, or an email draft): Use `messaging_send` with the target platform and conversation ID. It reaches Gmail or Outlook as a draft, and posts to a Slack, Telegram, or Discord chat through that channel's own transport, where the post is recorded and can be found again afterwards. Reach for `chat.postMessage` through `assistant oauth request` only for a Block Kit shape `messaging_send` cannot carry, with the provider the **slack** skill says to pass for posting on this workspace (`slack_channel` when the bot is set up; where only the `slack` integration exists, the skill says to tell the user before posting as them), never through `curl` or a revealed token. Either way the full content posts directly.
|
|
263
|
+
- **The user's notification surfaces** (Vellum inbox, Telegram, platform push): Use `assistant notifications send` via `bash`. Pass `--source-channel scheduler` and the complete authored body as `--message`. The pipeline keeps that body. Do not summarize the report before sending.
|
|
264
264
|
|
|
265
265
|
Example schedule message for a Slack digest:
|
|
266
266
|
|
|
@@ -165,11 +165,7 @@
|
|
|
165
165
|
},
|
|
166
166
|
"description": "Replacement steps (replaces all existing steps)"
|
|
167
167
|
}
|
|
168
|
-
}
|
|
169
|
-
"oneOf": [
|
|
170
|
-
{ "required": ["id"] },
|
|
171
|
-
{ "required": ["enrollment_id", "enrollment_action"] }
|
|
172
|
-
]
|
|
168
|
+
}
|
|
173
169
|
},
|
|
174
170
|
"executor": "tools/sequence-update.ts",
|
|
175
171
|
"execution_target": "host"
|
|
@@ -153,6 +153,15 @@ export const CALL_SITE_DEFAULTS: Record<LLMCallSite, CallSiteDefaultConfig> = {
|
|
|
153
153
|
effort: "low",
|
|
154
154
|
thinking: { enabled: false },
|
|
155
155
|
},
|
|
156
|
+
// The escalation judge only runs when this site resolves to the TypeSafe
|
|
157
|
+
// provider (see calls/voice-escalation-judge.ts), so the shipped default is
|
|
158
|
+
// any ordinary profile: the judge stays off until a user pins a TypeSafe
|
|
159
|
+
// profile here.
|
|
160
|
+
voiceEscalationJudge: { profile: "cost-optimized" },
|
|
161
|
+
// Same arrangement for the barge-in continuation judge
|
|
162
|
+
// (live-voice/continuation-judge.ts): off until a TypeSafe profile is
|
|
163
|
+
// pinned here.
|
|
164
|
+
voiceContinuationJudge: { profile: "cost-optimized" },
|
|
156
165
|
// Names the background continuation a barge-in spawns, from the interrupted
|
|
157
166
|
// transcript. The label is fixed at spawn, so the call runs under a short
|
|
158
167
|
// timeout and the deterministic label stands in when it misses; that makes
|
|
@@ -536,6 +536,14 @@
|
|
|
536
536
|
"description": "Route all user-facing assistant text through the send_user_message tool instead of streamed assistant text",
|
|
537
537
|
"defaultEnabled": false
|
|
538
538
|
},
|
|
539
|
+
{
|
|
540
|
+
"id": "session-groups",
|
|
541
|
+
"scope": "both",
|
|
542
|
+
"key": "session-groups",
|
|
543
|
+
"label": "Session Groups",
|
|
544
|
+
"description": "Records eligible computer use, browser, live vision, and ambient runs and presents them as collapsible transcript groups. Off preserves ordinary actions and the flat transcript without new session tracking.",
|
|
545
|
+
"defaultEnabled": false
|
|
546
|
+
},
|
|
539
547
|
{
|
|
540
548
|
"id": "assistant-desktop",
|
|
541
549
|
"scope": "assistant",
|
|
@@ -543,6 +551,14 @@
|
|
|
543
551
|
"label": "Virtual desktop",
|
|
544
552
|
"description": "Streams an interactive virtual desktop from a platform-hosted, containerized assistant over /v1/desktop/stream. Requires platform hosting; self-hosted assistants are excluded. Off: the virtual desktop is hidden. On: the chat header offers its viewer and controls.",
|
|
545
553
|
"defaultEnabled": false
|
|
554
|
+
},
|
|
555
|
+
{
|
|
556
|
+
"id": "assistant-inbox",
|
|
557
|
+
"scope": "client",
|
|
558
|
+
"key": "assistant-inbox",
|
|
559
|
+
"label": "Assistant Inbox",
|
|
560
|
+
"description": "Shows an Assistant Inbox entry above Preferences in the web sidebar and the /assistant/inbox page it opens: a mailbox for the assistant's managed vellum.me address (received and sent, with a reading pane), the inline email setup card for entitled orgs with no address yet, and an upgrade card for orgs without the managed_email entitlement. Off hides the entry and redirects the route to chat.",
|
|
561
|
+
"defaultEnabled": false
|
|
546
562
|
}
|
|
547
563
|
]
|
|
548
564
|
}
|
package/src/config/loader.ts
CHANGED
|
@@ -58,14 +58,6 @@ type ConfigFileSignature =
|
|
|
58
58
|
exists: false;
|
|
59
59
|
};
|
|
60
60
|
|
|
61
|
-
function getConfigPath(): string {
|
|
62
|
-
return getWorkspaceConfigPath();
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
function ensureMigratedDataDir(): void {
|
|
66
|
-
ensureDataDir();
|
|
67
|
-
}
|
|
68
|
-
|
|
69
61
|
function readConfigFileSignature(configPath: string): ConfigFileSignature {
|
|
70
62
|
const stats = safeStatSync(configPath);
|
|
71
63
|
if (!stats) {
|
|
@@ -104,7 +96,7 @@ function getCachedConfigIfFresh(): AssistantConfig | null {
|
|
|
104
96
|
return null;
|
|
105
97
|
}
|
|
106
98
|
|
|
107
|
-
const currentSignature = readConfigFileSignature(
|
|
99
|
+
const currentSignature = readConfigFileSignature(getWorkspaceConfigPath());
|
|
108
100
|
if (configFileSignaturesEqual(cachedFileSignature, currentSignature)) {
|
|
109
101
|
return cached;
|
|
110
102
|
}
|
|
@@ -1002,7 +994,7 @@ export function mergeDefaultWorkspaceConfig(): DefaultWorkspaceConfigMergeResult
|
|
|
1002
994
|
Object.prototype.hasOwnProperty.call(llmDefaults, "activeProfile"),
|
|
1003
995
|
};
|
|
1004
996
|
|
|
1005
|
-
const configPath =
|
|
997
|
+
const configPath = getWorkspaceConfigPath();
|
|
1006
998
|
let existing: Record<string, unknown> = {};
|
|
1007
999
|
if (existsSync(configPath)) {
|
|
1008
1000
|
try {
|
|
@@ -1070,8 +1062,8 @@ export function loadConfig(): AssistantConfig {
|
|
|
1070
1062
|
loading = true;
|
|
1071
1063
|
|
|
1072
1064
|
try {
|
|
1073
|
-
|
|
1074
|
-
const configPath =
|
|
1065
|
+
ensureDataDir();
|
|
1066
|
+
const configPath = getWorkspaceConfigPath();
|
|
1075
1067
|
|
|
1076
1068
|
let fileConfig: Record<string, unknown> = {};
|
|
1077
1069
|
let configFileExisted = true;
|
|
@@ -1259,7 +1251,7 @@ export function getConfigReadOnly(): AssistantConfig {
|
|
|
1259
1251
|
return freshCached;
|
|
1260
1252
|
}
|
|
1261
1253
|
|
|
1262
|
-
const configPath =
|
|
1254
|
+
const configPath = getWorkspaceConfigPath();
|
|
1263
1255
|
let fileConfig: Record<string, unknown> = {};
|
|
1264
1256
|
if (existsSync(configPath)) {
|
|
1265
1257
|
try {
|
|
@@ -1312,8 +1304,8 @@ export function withSuppressedConfigDiskWritesSync<T>(fn: () => T): T {
|
|
|
1312
1304
|
* type without runtime shape-checking — the boundary check happens here.
|
|
1313
1305
|
*/
|
|
1314
1306
|
export function loadRawConfig(): Record<string, unknown> {
|
|
1315
|
-
|
|
1316
|
-
const configPath =
|
|
1307
|
+
ensureDataDir();
|
|
1308
|
+
const configPath = getWorkspaceConfigPath();
|
|
1317
1309
|
if (!existsSync(configPath)) {
|
|
1318
1310
|
return {};
|
|
1319
1311
|
}
|
|
@@ -1372,8 +1364,8 @@ function describeJsonShape(value: unknown): string {
|
|
|
1372
1364
|
}
|
|
1373
1365
|
|
|
1374
1366
|
export function saveRawConfig(config: Record<string, unknown>): void {
|
|
1375
|
-
|
|
1376
|
-
const configPath =
|
|
1367
|
+
ensureDataDir();
|
|
1368
|
+
const configPath = getWorkspaceConfigPath();
|
|
1377
1369
|
|
|
1378
1370
|
// Strip legacy apiKeys — provider keys belong in secure storage, not plaintext config
|
|
1379
1371
|
delete config.apiKeys;
|