@vellumai/assistant 0.12.1 → 0.12.2-dev.202609162214.308bda0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +2 -0
- package/AGENTS.md +5 -1
- package/ARCHITECTURE.md +20 -21
- package/Dockerfile +7 -7
- package/docs/architecture/integrations.md +7 -0
- package/docs/architecture/memory.md +11 -2
- package/docs/architecture/turn-actor.md +15 -8
- package/docs/credential-execution-service.md +1 -1
- package/docs/desktop-browser-cli.md +35 -0
- package/docs/guardian-request-flow.md +16 -0
- package/node_modules/@vellumai/environments/src/shell.test.ts +21 -0
- package/node_modules/@vellumai/environments/src/shell.ts +24 -0
- package/node_modules/@vellumai/gateway-client/src/inbound-contract.ts +8 -2
- package/node_modules/@vellumai/slack-text/src/index.ts +13 -8
- package/node_modules/@vellumai/slack-text/src/label-resolution-entities.test.ts +95 -0
- package/openapi.yaml +564 -111
- package/package.json +7 -4
- package/scripts/bundled-plugin-packages.ts +154 -0
- package/scripts/generate-bundled-plugin-packages.ts +29 -0
- package/scripts/postinstall.ts +33 -0
- package/scripts/smoke-desktop-browser-cli.ts +280 -0
- package/scripts/smoke-desktop-window-theme.ts +186 -0
- package/scripts/sync-llm-catalog.ts +3 -0
- package/scripts/test.ts +15 -13
- package/scripts/voice-ttft-spike.ts +2 -2
- package/src/__tests__/agent-loop.test.ts +124 -0
- package/src/__tests__/always-loaded-tools-guard.test.ts +18 -10
- package/src/__tests__/anthropic-provider.test.ts +0 -63
- package/src/__tests__/app-compiler.test.ts +6 -0
- package/src/__tests__/app-control-tool-schemas.test.ts +5 -2
- package/src/__tests__/app-dir-path-guard.test.ts +1 -3
- package/src/__tests__/approval-interception-trust-gates.test.ts +40 -0
- package/src/__tests__/assistant-event-hub-machine-name.test.ts +6 -7
- package/src/__tests__/assistant-event-hub.test.ts +44 -0
- package/src/__tests__/attachments-store.test.ts +22 -3
- package/src/__tests__/channel-approval.test.ts +9 -14
- package/src/__tests__/channel-reply-delivery.test.ts +56 -0
- package/src/__tests__/computer-use-screenshot-attachments.test.ts +411 -0
- package/src/__tests__/computer-use-screenshot-selection.test.ts +50 -0
- package/src/__tests__/computer-use-skill-manifest-regression.test.ts +1 -1
- package/src/__tests__/computer-use-skill-proxy-bridge.test.ts +5 -2
- package/src/__tests__/computer-use-tools.test.ts +19 -7
- package/src/__tests__/config-schema-cmd.test.ts +2 -2
- package/src/__tests__/config-schema.test.ts +29 -30
- package/src/__tests__/config-sounds-sync.test.ts +23 -0
- package/src/__tests__/config-watcher.test.ts +23 -8
- package/src/__tests__/connection-model-compat.test.ts +2 -2
- package/src/__tests__/conversation-agent-loop-inference-profile.test.ts +2 -0
- package/src/__tests__/conversation-agent-loop-overflow.test.ts +2 -0
- package/src/__tests__/conversation-agent-loop.test.ts +257 -6
- package/src/__tests__/conversation-attachments.test.ts +106 -0
- package/src/__tests__/conversation-error.test.ts +4 -1
- package/src/__tests__/conversation-fork-crud.test.ts +191 -0
- package/src/__tests__/conversation-key-store-disk-view.test.ts +1 -1
- package/src/__tests__/conversation-pairing.test.ts +26 -0
- package/src/__tests__/conversation-process-app-control-preactivation.test.ts +4 -10
- package/src/__tests__/conversation-queue.test.ts +48 -207
- package/src/__tests__/conversation-retry-discard.test.ts +54 -1
- package/src/__tests__/conversation-runtime-assembly.test.ts +33 -3
- package/src/__tests__/conversation-skill-tools.test.ts +27 -0
- package/src/__tests__/conversation-surfaces-app-control.test.ts +8 -3
- package/src/__tests__/conversation-surfaces-point-at-budget.test.ts +1 -0
- package/src/__tests__/conversation-surfaces-point-at-capability.test.ts +1 -0
- package/src/__tests__/conversation-surfaces-task-progress.test.ts +67 -0
- package/src/__tests__/conversation-title-service.test.ts +14 -10
- package/src/__tests__/conversation-tool-setup-app-refresh.test.ts +3 -5
- package/src/__tests__/conversation-tool-setup-attribution.test.ts +51 -0
- package/src/__tests__/credential-execution-client.test.ts +70 -53
- package/src/__tests__/credential-record-retire.test.ts +171 -1
- package/src/__tests__/credential-routes.test.ts +250 -3
- package/src/__tests__/credential-security-invariants.test.ts +5 -3
- package/src/__tests__/cu-unified-flow.test.ts +14 -4
- package/src/__tests__/db-conversation-tool-surface.test.ts +144 -0
- package/src/__tests__/document-sync-tags.test.ts +25 -0
- package/src/__tests__/drain-kick-guard.test.ts +0 -2
- package/src/__tests__/drain-requeue-on-contention.test.ts +0 -6
- package/src/__tests__/events-tail-route.test.ts +33 -0
- package/src/__tests__/evict-conversations-for-reload.test.ts +1 -1
- package/src/__tests__/external-plugin-loader.test.ts +52 -0
- package/src/__tests__/git-porcelain.test.ts +47 -0
- package/src/__tests__/headless-browser-mode.test.ts +48 -0
- package/src/__tests__/heartbeat-service.test.ts +56 -40
- package/src/__tests__/helpers/set-workspace-mcp.ts +11 -0
- package/src/__tests__/host-bash-proxy.test.ts +26 -45
- package/src/__tests__/host-browser-proxy.test.ts +6 -17
- package/src/__tests__/host-cu-proxy.test.ts +712 -18
- package/src/__tests__/host-file-proxy-targeted.test.ts +23 -0
- package/src/__tests__/host-file-proxy.test.ts +11 -23
- package/src/__tests__/host-transfer-proxy-targeted.test.ts +30 -9
- package/src/__tests__/inference-profile-session-handler.test.ts +26 -0
- package/src/__tests__/list-all-apps.test.ts +40 -8
- package/src/__tests__/list-messages-camera-frame.test.ts +121 -0
- package/src/__tests__/list-messages-tool-merge.test.ts +85 -1
- package/src/__tests__/llm-catalog-parity.test.ts +51 -3
- package/src/__tests__/managed-store.test.ts +121 -0
- package/src/__tests__/mcp-auth-routes.test.ts +104 -8
- package/src/__tests__/mcp-client-auth.test.ts +54 -0
- package/src/__tests__/mcp-config-secret-boundary.test.ts +78 -136
- package/src/__tests__/mcp-health-check.test.ts +40 -43
- package/src/__tests__/mcp-list-plugin-servers.test.ts +85 -42
- package/src/__tests__/mcp-tool-annotations-risk.test.ts +4 -2
- package/src/__tests__/message-deleted-hook-dispatch.test.ts +119 -0
- package/src/__tests__/messages-after-tiebreaker.test.ts +91 -0
- package/src/__tests__/messages-read-boundary-guard.test.ts +1 -1
- package/src/__tests__/messaging-send-tool.test.ts +355 -548
- package/src/__tests__/model-intents.test.ts +3 -0
- package/src/__tests__/notification-vellum-adapter.test.ts +85 -2
- package/src/__tests__/oauth-apps-routes.test.ts +1 -0
- package/src/__tests__/oauth-commands-routes.test.ts +282 -101
- package/src/__tests__/oauth-connect-orchestrator.test.ts +2 -0
- package/src/__tests__/oauth-provider-profiles.test.ts +18 -0
- package/src/__tests__/oauth-provider-serializer.test.ts +23 -0
- package/src/__tests__/oauth-providers-routes.test.ts +3 -0
- package/src/__tests__/openai-provider.test.ts +1 -0
- package/src/__tests__/outlook-messaging-provider.test.ts +6 -6
- package/src/__tests__/persist-media-references.test.ts +50 -0
- package/src/__tests__/platform-callback-registration.test.ts +8 -16
- package/src/__tests__/platform-client-verify-credential.test.ts +2 -0
- package/src/__tests__/plugin-import-boundary-guard.test.ts +13 -0
- package/src/__tests__/qdrant-manager.test.ts +132 -0
- package/src/__tests__/resolve-app.test.ts +135 -0
- package/src/__tests__/run-conversation-turn-persistence.test.ts +138 -1
- package/src/__tests__/scaffold-managed-skill-tool.test.ts +317 -6
- package/src/__tests__/schedule-routes.test.ts +20 -0
- package/src/__tests__/scheduler-result-notification.test.ts +23 -4
- package/src/__tests__/script-proxy-certs.test.ts +35 -0
- package/src/__tests__/secret-routes-acp-guard.test.ts +59 -1
- package/src/__tests__/secret-routes-platform-proxy.test.ts +86 -31
- package/src/__tests__/secret-routes-scrub.test.ts +22 -27
- package/src/__tests__/secure-keys-managed-failover.test.ts +8 -0
- package/src/__tests__/secure-keys.test.ts +7 -3
- package/src/__tests__/service-account-token.test.ts +247 -0
- package/src/__tests__/skill-feature-flags-integration.test.ts +1 -1
- package/src/__tests__/skill-feature-flags.test.ts +2 -2
- package/src/__tests__/skills.test.ts +20 -4
- package/src/__tests__/source-fingerprint.test.ts +184 -0
- package/src/__tests__/subagent-tool-gate-mode.test.ts +239 -3
- package/src/__tests__/subagent-tools.test.ts +3 -2
- package/src/__tests__/telephony-tts-guard.test.ts +38 -3
- package/src/__tests__/terminal-tools.test.ts +8 -0
- package/src/__tests__/test-support/computer-use-skill-harness.ts +3 -2
- package/src/__tests__/tool-result-metadata-plumbing.test.ts +63 -0
- package/src/__tests__/ui-channel-variants.test.ts +1 -56
- package/src/__tests__/unicode.test.ts +49 -0
- package/src/__tests__/user-plugin-loader.test.ts +21 -0
- package/src/__tests__/vellum-self-knowledge-inline-command.test.ts +4 -0
- package/src/__tests__/watch-retro-tool-availability.test.ts +8 -17
- package/src/__tests__/workspace-git-service.test.ts +26 -0
- package/src/__tests__/workspace-migration-154-repair-retired-codex-gpt-5-4-model-ids.test.ts +336 -0
- package/src/acp/__tests__/acp-claude-oauth.test.ts +257 -6
- package/src/acp/__tests__/acp-credentials.test.ts +13 -0
- package/src/acp/__tests__/agent-process.test.ts +258 -4
- package/src/acp/__tests__/auth-required.test.ts +96 -0
- package/src/acp/__tests__/claude-token-refresh.test.ts +257 -0
- package/src/acp/__tests__/client-handler.test.ts +90 -2
- package/src/acp/__tests__/helpers/acp-model-option.ts +51 -0
- package/src/acp/__tests__/prepare-agent-env.test.ts +60 -1
- package/src/acp/__tests__/session-manager-persistence.test.ts +5 -1
- package/src/acp/__tests__/session-manager-resume.test.ts +270 -6
- package/src/acp/__tests__/session-manager-usage.test.ts +4 -2
- package/src/acp/__tests__/session-manager.test.ts +895 -4
- package/src/acp/acp-claude-oauth.ts +328 -14
- package/src/acp/acp-credentials.ts +19 -0
- package/src/acp/agent-process.test.ts +4 -1
- package/src/acp/agent-process.ts +96 -10
- package/src/acp/auth-required.ts +50 -1
- package/src/acp/auto-install.test.ts +9 -7
- package/src/acp/claude-token-refresh.ts +150 -0
- package/src/acp/client-handler.ts +16 -3
- package/src/acp/model-config.test.ts +166 -0
- package/src/acp/model-config.ts +138 -0
- package/src/acp/prepare-agent-env.ts +21 -6
- package/src/acp/resolve-agent.test.ts +220 -6
- package/src/acp/resolve-agent.ts +88 -23
- package/src/acp/session-manager.test.ts +196 -3
- package/src/acp/session-manager.ts +441 -18
- package/src/acp/session-snapshot.ts +260 -0
- package/src/acp/types.ts +50 -5
- package/src/agent/loop.ts +20 -0
- package/src/api/attachment-provenance.test.ts +68 -0
- package/src/api/computer-use-tool.test.ts +55 -0
- package/src/api/computer-use-tool.ts +27 -0
- package/src/api/events/acp-session-model-update.test.ts +88 -0
- package/src/api/events/acp-session-model-update.ts +46 -0
- package/src/api/events/app-files-changed.ts +4 -3
- package/src/api/events/assistant-outbound-attachment.ts +3 -0
- package/src/api/events/desktop-activity-changed.ts +10 -0
- package/src/api/events/notification-intent.ts +2 -0
- package/src/api/events/user-message-echo.ts +7 -1
- package/src/api/index.ts +17 -0
- package/src/api/responses/conversation-message.ts +8 -0
- package/src/approvals/approval-primitive.ts +5 -2
- package/src/approvals/scoped-approval-grants.ts +6 -2
- package/src/apps/app-store.ts +7 -6
- package/src/apps/resolve-app.ts +81 -0
- package/src/apps/source-fingerprint.ts +266 -0
- package/src/avatar/identity-avatar.ts +7 -2
- package/src/browser/operations.ts +2 -1
- package/src/browser/types.ts +7 -0
- package/src/browser/virtual-desktop-target.ts +34 -0
- package/src/bundler/app-compiler.ts +15 -0
- package/src/calls/__tests__/barge-in-guard.test.ts +81 -0
- package/src/{__tests__ → calls/__tests__}/call-controller.test.ts +503 -82
- package/src/{__tests__ → calls/__tests__}/media-stream-audio-transcode.test.ts +1 -1
- package/src/{__tests__ → calls/__tests__}/media-stream-output.test.ts +37 -8
- package/src/{__tests__ → calls/__tests__}/media-stream-parser.test.ts +1 -1
- package/src/{__tests__ → calls/__tests__}/media-stream-server-integration.test.ts +150 -57
- package/src/{__tests__ → calls/__tests__}/media-stream-stt-session.test.ts +5 -5
- package/src/{__tests__ → calls/__tests__}/media-turn-detector.test.ts +1 -1
- package/src/{live-voice → calls}/__tests__/progress-narration.test.ts +27 -4
- package/src/{live-voice → calls}/__tests__/progress-phrases.test.ts +1 -1
- package/src/calls/__tests__/voice-control-protocol.test.ts +151 -27
- package/src/calls/__tests__/voice-leg-coordinator.test.ts +229 -0
- package/src/calls/__tests__/voice-progress-cadence.test.ts +274 -0
- package/src/{__tests__/voice-session-bridge.test.ts → calls/__tests__/voice-session-bridge-turn-flow.test.ts} +20 -12
- package/src/calls/__tests__/voice-session-bridge.test.ts +140 -3
- package/src/calls/__tests__/voice-triage-escalate.test.ts +105 -0
- package/src/calls/barge-in-guard.ts +118 -0
- package/src/calls/call-constants.ts +24 -4
- package/src/calls/call-controller.ts +499 -199
- package/src/calls/call-transport.ts +16 -0
- package/src/calls/media-stream-output.ts +32 -0
- package/src/calls/media-stream-server.ts +82 -30
- package/src/calls/media-stream-stt-session.ts +15 -0
- package/src/{live-voice → calls}/progress-narration.ts +17 -4
- package/src/calls/resolve-call-tts-provider.ts +33 -2
- package/src/calls/tts-call-strategy.ts +9 -5
- package/src/calls/types.ts +2 -0
- package/src/calls/voice-control-protocol.ts +194 -21
- package/src/calls/voice-leg-coordinator.ts +189 -0
- package/src/calls/voice-progress-cadence.ts +423 -0
- package/src/calls/voice-session-bridge.ts +140 -76
- package/src/calls/voice-triage-escalate.ts +178 -59
- package/src/cli/commands/__tests__/apps.test.ts +245 -0
- package/src/cli/commands/__tests__/bash.test.ts +53 -0
- package/src/cli/commands/__tests__/browser.test.ts +84 -3
- package/src/cli/commands/__tests__/cli-test-harness.ts +21 -3
- package/src/cli/commands/__tests__/plugins.test.ts +85 -0
- package/src/cli/commands/__tests__/roadmap.help.test.ts +19 -0
- package/src/cli/commands/__tests__/schedules.test.ts +14 -0
- package/src/cli/commands/apps.help.ts +67 -13
- package/src/cli/commands/apps.ts +183 -0
- package/src/cli/commands/bash.help.ts +9 -13
- package/src/cli/commands/bash.ts +7 -25
- package/src/cli/commands/browser.help.ts +43 -0
- package/src/cli/commands/browser.ts +52 -8
- package/src/cli/commands/channels/__tests__/send.test.ts +218 -0
- package/src/cli/commands/channels/index.help.ts +70 -2
- package/src/cli/commands/channels/index.ts +16 -3
- package/src/cli/commands/channels/send.ts +128 -0
- package/src/cli/commands/credentials.help.ts +9 -5
- package/src/cli/commands/credentials.ts +2 -2
- package/src/cli/commands/mcp.help.ts +8 -8
- package/src/cli/commands/mcp.ts +1 -1
- package/src/cli/commands/oauth/index.help.ts +10 -0
- package/src/cli/commands/oauth/providers.ts +12 -0
- package/src/cli/commands/oauth/request.test.ts +179 -2
- package/src/cli/commands/oauth/request.ts +41 -10
- package/src/cli/commands/oauth/status.ts +5 -0
- package/src/cli/commands/plugins.ts +73 -23
- package/src/cli/commands/roadmap.help.ts +5 -1
- package/src/cli/commands/roadmap.ts +1 -1
- package/src/cli/commands/schedules.ts +2 -0
- package/src/cli/lib/__tests__/inspect-plugin.test.ts +54 -0
- package/src/cli/lib/__tests__/install-from-github.test.ts +219 -0
- package/src/cli/lib/__tests__/install-from-platform.test.ts +77 -2
- package/src/cli/lib/__tests__/list-installed-plugins.test.ts +45 -3
- package/src/cli/lib/__tests__/local-plugin-upgrade.test.ts +251 -0
- package/src/cli/lib/__tests__/plugin-catalog-cache.test.ts +65 -2
- package/src/cli/lib/__tests__/plugin-catalog-platform.test.ts +14 -0
- package/src/cli/lib/__tests__/plugin-catalog-resolve.test.ts +27 -1
- package/src/cli/lib/__tests__/plugin-details.test.ts +35 -2
- package/src/cli/lib/__tests__/plugin-marketplace.test.ts +20 -0
- package/src/cli/lib/__tests__/plugins-install-offline.test.ts +43 -0
- package/src/cli/lib/__tests__/search-plugins.test.ts +31 -0
- package/src/cli/lib/__tests__/uninstall-plugin.test.ts +122 -5
- package/src/cli/lib/bundled-marketplace.json +962 -81
- package/src/cli/lib/bundled-plugin-packages.json +786 -0
- package/src/cli/lib/bundled-plugin-packages.ts +87 -0
- package/src/cli/lib/diff-plugin.ts +1 -1
- package/src/cli/lib/inspect-plugin.ts +87 -12
- package/src/cli/lib/install-from-github.ts +150 -94
- package/src/cli/lib/install-from-platform.ts +14 -9
- package/src/cli/lib/list-installed-plugins.ts +21 -35
- package/src/cli/lib/plugin-catalog-cache.ts +22 -3
- package/src/cli/lib/plugin-catalog-local.ts +13 -3
- package/src/cli/lib/plugin-catalog-platform.ts +6 -1
- package/src/cli/lib/plugin-catalog-resolve.ts +20 -0
- package/src/cli/lib/plugin-details.ts +33 -8
- package/src/cli/lib/plugin-marketplace.ts +121 -21
- package/src/cli/lib/plugin-pin-history.ts +5 -2
- package/src/cli/lib/search-plugins.ts +58 -16
- package/src/cli/lib/uninstall-plugin.ts +59 -1
- package/src/cli/lib/upgrade-plugin.ts +61 -8
- package/src/cli/output.ts +18 -4
- package/src/config/__tests__/default-profile-catalog.test.ts +22 -0
- package/src/config/__tests__/default-provider.test.ts +4 -0
- package/src/config/__tests__/memory-retrospective-schema.test.ts +15 -1
- package/src/config/__tests__/platform-identity.test.ts +255 -0
- package/src/config/__tests__/platform-rehydration.test.ts +99 -9
- package/src/config/__tests__/plugin-resident-skill-discovery.test.ts +21 -0
- package/src/config/acp-agent-write.ts +43 -0
- package/src/config/acp-defaults.test.ts +13 -3
- package/src/config/acp-defaults.ts +23 -6
- package/src/config/acp-schema.test.ts +81 -0
- package/src/config/acp-schema.ts +30 -2
- package/src/config/bundled-skills/acp/SKILL.md +23 -15
- package/src/config/bundled-skills/acp/TOOLS.json +7 -3
- package/src/config/bundled-skills/app-builder/SKILL.md +12 -9
- package/src/config/bundled-skills/app-builder/references/CUSTOM_ROUTES.md +1 -1
- package/src/config/bundled-skills/computer-use/SKILL.md +59 -2
- package/src/config/bundled-skills/computer-use/TOOLS.json +101 -2
- package/src/config/bundled-skills/computer-use/tools/computer-use-sequence.ts +12 -0
- package/src/config/bundled-skills/media-processing/services/preprocess.ts +2 -1
- package/src/config/bundled-skills/media-processing/services/processing-pipeline.ts +2 -1
- package/src/config/bundled-skills/media-processing/tools/generate-clip.ts +2 -1
- package/src/config/bundled-skills/messaging/SKILL.md +3 -3
- package/src/config/bundled-skills/messaging/TOOLS.json +3 -3
- package/src/config/bundled-skills/messaging/tools/messaging-send.ts +67 -139
- package/src/config/bundled-skills/phone-calls/references/TRANSCRIPTS.md +1 -1
- package/src/config/bundled-skills/playbooks/tools/playbook-create.ts +2 -1
- package/src/config/bundled-skills/playbooks/tools/playbook-update.ts +2 -1
- package/src/config/bundled-skills/schedule/SKILL.md +7 -8
- package/src/config/bundled-skills/screen-annotation/SKILL.md +10 -3
- package/src/config/bundled-skills/screen-annotation/tools/screen-annotation.test.ts +4 -1
- package/src/config/bundled-skills/skill-management/SKILL.md +1 -1
- package/src/config/bundled-skills/skill-management/TOOLS.json +13 -9
- package/src/config/bundled-skills/transcribe/tools/transcribe-media.ts +5 -2
- package/src/config/default-profile-catalog.ts +1 -1
- package/src/config/env.ts +2 -2
- package/src/config/feature-flag-registry.json +33 -13
- package/src/config/loader.ts +11 -17
- package/src/config/memory-v3-gate.test.ts +31 -0
- package/src/config/memory-v3-gate.ts +20 -16
- package/src/config/platform-identity.ts +201 -0
- package/src/config/platform-rehydration.ts +11 -27
- package/src/config/profile-text-generation.test.ts +51 -0
- package/src/config/profile-text-generation.ts +51 -0
- package/src/config/schema.ts +2 -2
- package/src/config/schemas/__tests__/live-voice.test.ts +8 -158
- package/src/config/schemas/__tests__/voice.test.ts +148 -0
- package/src/config/schemas/calls.ts +1 -28
- package/src/config/schemas/live-voice.ts +1 -173
- package/src/config/schemas/llm.ts +7 -2
- package/src/config/schemas/mcp.ts +15 -9
- package/src/config/schemas/memory-retrospective.ts +10 -0
- package/src/config/schemas/monitoring.ts +9 -0
- package/src/config/schemas/services.ts +18 -0
- package/src/config/schemas/skills.ts +0 -14
- package/src/config/schemas/voice.ts +150 -0
- package/src/config/skills.ts +13 -21
- package/src/credential-execution/ces-runtime.ts +2 -2
- package/src/credential-execution/executable-discovery.ts +36 -134
- package/src/credential-execution/process-manager.test.ts +37 -42
- package/src/credential-execution/process-manager.ts +24 -19
- package/src/credential-health/credential-health-service.ts +11 -17
- package/src/daemon/__tests__/conversation-tool-setup-exclude.test.ts +47 -0
- package/src/daemon/__tests__/conversation-tool-setup.test.ts +109 -121
- package/src/daemon/__tests__/plugin-mcp-reconcile.test.ts +5 -1
- package/src/daemon/__tests__/skill-memory-refresh-mtime.test.ts +133 -0
- package/src/daemon/__tests__/turn-tail-deleted-conversation.test.ts +36 -0
- package/src/daemon/app-change-notify.ts +45 -0
- package/src/daemon/assistant-attachments.ts +20 -12
- package/src/daemon/config-watcher.ts +13 -194
- package/src/daemon/conversation-agent-loop-handlers.ts +74 -3
- package/src/daemon/conversation-agent-loop.ts +12 -0
- package/src/daemon/conversation-attachments.ts +93 -12
- package/src/daemon/conversation-client-surface.ts +26 -0
- package/src/daemon/conversation-error.ts +3 -2
- package/src/daemon/conversation-media-retry.ts +2 -1
- package/src/daemon/conversation-process.ts +35 -64
- package/src/daemon/conversation-runtime-assembly.ts +21 -8
- package/src/daemon/conversation-skill-tools.ts +13 -31
- package/src/daemon/conversation-store.ts +1 -1
- package/src/daemon/conversation-surfaces.ts +21 -10
- package/src/daemon/conversation-tool-setup.ts +135 -109
- package/src/daemon/conversation-turn-finalize.ts +26 -13
- package/src/daemon/conversation.ts +17 -3
- package/src/daemon/daemon-control.ts +3 -7
- package/src/daemon/dictation-profile-store.ts +2 -1
- package/src/daemon/handlers/shared.ts +2 -1
- package/src/daemon/handlers/skills.ts +6 -3
- package/src/daemon/host-bash-proxy.ts +8 -12
- package/src/daemon/host-browser-proxy.ts +15 -1
- package/src/daemon/host-cu-proxy.ts +275 -45
- package/src/daemon/host-file-proxy.ts +8 -10
- package/src/daemon/host-proxy-preactivation.ts +4 -1
- package/src/daemon/host-transfer-proxy.ts +14 -18
- package/src/daemon/lifecycle.ts +4 -9
- package/src/daemon/mcp-reload-service.ts +13 -6
- package/src/daemon/message-types/shared.ts +1 -0
- package/src/daemon/message-types/sync.ts +2 -0
- package/src/daemon/orphan-reaper.ts +4 -3
- package/src/daemon/persist-media-references.ts +20 -3
- package/src/daemon/providers-setup.ts +5 -1
- package/src/daemon/reaction-history-render.ts +2 -1
- package/src/daemon/shutdown-handlers.ts +1 -9
- package/src/daemon/skill-memory-refresh.ts +64 -0
- package/src/daemon/tool-setup-types.ts +6 -0
- package/src/daemon/tool-side-effects.ts +7 -18
- package/src/daemon/wake-conversation-ops.ts +38 -15
- package/src/desktop/__tests__/fake-desktop.ts +123 -0
- package/src/desktop/desktop-automation-lease.test.ts +351 -0
- package/src/desktop/desktop-automation-lease.ts +306 -0
- package/src/desktop/desktop-browser-client.test.ts +413 -0
- package/src/desktop/desktop-browser-client.ts +430 -0
- package/src/desktop/desktop-browser-cursor.ts +40 -0
- package/src/desktop/desktop-browser-endpoint.test.ts +78 -0
- package/src/desktop/desktop-browser-endpoint.ts +145 -0
- package/src/desktop/desktop-browser-operations.ts +130 -0
- package/src/desktop/desktop-chrome-policy.test.ts +79 -0
- package/src/desktop/desktop-chrome-policy.ts +47 -0
- package/src/desktop/desktop-chrome-session.test.ts +48 -0
- package/src/desktop/desktop-chrome-session.ts +87 -0
- package/src/desktop/desktop-dependencies.test.ts +131 -0
- package/src/desktop/desktop-dependencies.ts +329 -0
- package/src/desktop/desktop-display.ts +9 -0
- package/src/desktop/desktop-panel-config.test.ts +135 -0
- package/src/desktop/desktop-panel-config.ts +162 -0
- package/src/desktop/desktop-session-manager.test.ts +813 -0
- package/src/desktop/desktop-session-manager.ts +970 -0
- package/src/desktop/desktop-stream-bridge.test.ts +395 -0
- package/src/desktop/desktop-stream-bridge.ts +214 -0
- package/src/desktop/desktop-wallpaper.test.ts +191 -0
- package/src/desktop/desktop-wallpaper.ts +113 -0
- package/src/desktop/desktop-window-drag.ts +177 -0
- package/src/desktop/desktop-window-manager-config.test.ts +230 -0
- package/src/desktop/desktop-window-manager-config.ts +238 -0
- package/src/desktop/desktop-window-theme.ts +170 -0
- package/src/desktop/virtual-desktop-feature.ts +18 -0
- package/src/desktop/virtual-desktop-platform.test.ts +61 -0
- package/src/heartbeat/heartbeat-service.ts +21 -3
- package/src/home/__tests__/feed-writer.test.ts +22 -0
- package/src/home/feed-writer.ts +11 -2
- package/src/hooks/types.ts +33 -0
- package/src/i18n/AGENTS.md +22 -0
- package/src/i18n/__tests__/i18n.test.ts +112 -0
- package/src/i18n/index.ts +16 -0
- package/src/i18n/locales.ts +93 -0
- package/src/i18n/messages.ts +70 -0
- package/src/i18n/title.ts +25 -0
- package/src/inbound/platform-callback-registration.ts +4 -5
- package/src/ipc/__tests__/browser-ipc.test.ts +230 -2
- package/src/ipc/__tests__/cancel-on-disconnect.test.ts +54 -0
- package/src/ipc/assistant-server.ts +15 -1
- package/src/ipc/cli-client.ts +13 -5
- package/src/live-voice/__tests__/live-voice-agent-turn.test.ts +143 -0
- package/src/live-voice/__tests__/live-voice-events.test.ts +9 -7
- package/src/live-voice/__tests__/live-voice-look-follow-up.test.ts +374 -0
- package/src/live-voice/__tests__/live-voice-metrics.test.ts +0 -3
- package/src/live-voice/__tests__/live-voice-progress.test.ts +94 -15
- package/src/live-voice/__tests__/live-voice-session-telemetry.test.ts +19 -0
- package/src/live-voice/__tests__/live-voice-sight-frame-echo.test.ts +238 -0
- package/src/live-voice/__tests__/live-voice-vad.test.ts +660 -5
- package/src/live-voice/__tests__/protocol.test.ts +95 -0
- package/src/live-voice/__tests__/runtime-websocket-shell.test.ts +13 -69
- package/src/live-voice/__tests__/session-controls.test.ts +156 -0
- package/src/live-voice/live-voice-metrics.ts +9 -0
- package/src/live-voice/live-voice-photo.ts +25 -6
- package/src/live-voice/live-voice-session.ts +1081 -760
- package/src/live-voice/protocol.ts +89 -2
- package/src/live-voice/session-controls.ts +197 -0
- package/src/mcp/__tests__/credential-target.test.ts +59 -0
- package/src/mcp/__tests__/effective-config.test.ts +13 -2
- package/src/mcp/__tests__/manager-state.test.ts +128 -0
- package/src/mcp/__tests__/manager-tool-caps.test.ts +111 -0
- package/src/mcp/__tests__/mcp-auth-orchestrator.test.ts +20 -6
- package/src/mcp/__tests__/mcp-auth-state-target.test.ts +49 -0
- package/src/mcp/__tests__/plugin-mcp-oauth-cleanup.test.ts +107 -0
- package/src/mcp/__tests__/plugin-mcp-oauth-provider-isolation.test.ts +89 -0
- package/src/mcp/__tests__/plugin-server-credential-isolation.test.ts +63 -11
- package/src/mcp/__tests__/reload-signal-emission.test.ts +39 -2
- package/src/mcp/__tests__/startup.test.ts +30 -7
- package/src/mcp/__tests__/tool-caps.test.ts +107 -0
- package/src/mcp/__tests__/workspace-mcp-config.test.ts +135 -0
- package/src/mcp/client.ts +64 -16
- package/src/mcp/credential-target.ts +105 -0
- package/src/mcp/effective-config.ts +45 -18
- package/src/mcp/manager.ts +201 -81
- package/src/mcp/mcp-auth-orchestrator.ts +17 -4
- package/src/mcp/mcp-auth-state.ts +38 -6
- package/src/mcp/mcp-header-store.ts +29 -45
- package/src/mcp/mcp-oauth-provider.ts +101 -33
- package/src/mcp/spec-schema.ts +111 -0
- package/src/mcp/startup.ts +41 -17
- package/src/mcp/tool-caps.ts +176 -0
- package/src/mcp/workspace-mcp-config.ts +188 -0
- package/src/messaging/provider-message-metadata.ts +3 -3
- package/src/messaging/provider.ts +10 -3
- package/src/messaging/providers/__tests__/proactive-address.test.ts +95 -0
- package/src/messaging/providers/callback-routing.ts +18 -0
- package/src/messaging/providers/channel-transport.ts +56 -0
- package/src/messaging/providers/discord/transport.ts +17 -0
- package/src/messaging/providers/slack/__tests__/adapter-token-routing.test.ts +6 -17
- package/src/messaging/providers/slack/adapter.ts +7 -44
- package/src/messaging/providers/slack/transport.ts +16 -0
- package/src/messaging/providers/telegram-bot/adapter.ts +1 -48
- package/src/messaging/providers/telegram-bot/client.ts +0 -26
- package/src/messaging/providers/telegram-bot/transport.ts +20 -0
- package/src/messaging/providers/whatsapp/adapter.ts +1 -48
- package/src/messaging/providers/whatsapp/transport.ts +18 -0
- package/src/monitoring/__tests__/mount-watch.test.ts +236 -0
- package/src/monitoring/__tests__/workspace-git-heartbeat.test.ts +8 -0
- package/src/monitoring/mount-watch.ts +348 -0
- package/src/monitoring/plugin-auto-update.ts +6 -0
- package/src/monitoring/worker.ts +9 -0
- package/src/notifications/__tests__/assistant-reply-producer.test.ts +55 -0
- package/src/notifications/__tests__/copy-composer.test.ts +70 -0
- package/src/notifications/__tests__/decision-engine.test.ts +175 -0
- package/src/notifications/__tests__/guardian-feed-projection.test.ts +158 -2
- package/src/notifications/__tests__/home-feed-side-effect.test.ts +153 -3
- package/src/notifications/__tests__/schedule-result-producer.test.ts +99 -3
- package/src/notifications/adapters/macos.ts +3 -0
- package/src/notifications/assistant-reply-producer.ts +8 -3
- package/src/notifications/conversation-pairing.ts +36 -16
- package/src/notifications/copy-composer.ts +11 -3
- package/src/notifications/decision-engine.ts +43 -8
- package/src/notifications/delivered-post-record.ts +11 -1
- package/src/notifications/guardian-feed-projection.ts +125 -5
- package/src/notifications/home-feed-side-effect.ts +55 -7
- package/src/notifications/schedule-result-producer.ts +64 -22
- package/src/oauth/AGENTS.md +4 -0
- package/src/oauth/__tests__/identity-verifier.test.ts +36 -1
- package/src/oauth/__tests__/seed-providers-managed.test.ts +149 -0
- package/src/oauth/byo-connection.test.ts +105 -0
- package/src/oauth/byo-connection.ts +36 -6
- package/src/oauth/connection-resolver.test.ts +100 -0
- package/src/oauth/connection-resolver.ts +120 -3
- package/src/oauth/identity-verifier.ts +24 -0
- package/src/oauth/oauth-store.ts +11 -1
- package/src/oauth/provider-serializer.ts +10 -0
- package/src/oauth/scope-utils.ts +24 -0
- package/src/oauth/seed-providers.ts +367 -27
- package/src/onboarding/checkin-event.ts +2 -1
- package/src/outbound-proxy/certs.ts +3 -5
- package/src/permissions/checker.test.ts +11 -6
- package/src/permissions/confirmation-guardian-request.test.ts +16 -0
- package/src/permissions/confirmation-guardian-request.ts +2 -3
- package/src/persistence/attachments-store.ts +84 -44
- package/src/persistence/compaction-log-store-clickhouse.ts +3 -3
- package/src/persistence/conversation-crud.ts +182 -83
- package/src/persistence/conversation-lineage.ts +12 -4
- package/src/persistence/conversation-plugin-facade.ts +13 -0
- package/src/persistence/conversation-queries.ts +2 -2
- package/src/persistence/conversation-title-placeholders.ts +23 -0
- package/src/persistence/conversation-title-service.ts +9 -26
- package/src/persistence/conversation-tool-surface.ts +86 -0
- package/src/persistence/conversation-types.test.ts +32 -0
- package/src/persistence/conversation-types.ts +26 -4
- package/src/persistence/embeddings/qdrant-manager.ts +163 -18
- package/src/persistence/llm-request-log-sink-clickhouse.ts +3 -3
- package/src/persistence/llm-request-log-source-clickhouse.ts +3 -3
- package/src/persistence/message-cursor.ts +78 -0
- package/src/persistence/message-reads.ts +30 -0
- package/src/persistence/migrations/157-invite-contact-id.ts +18 -6
- package/src/persistence/migrations/378-create-conversation-tool-surfaces.test.ts +78 -0
- package/src/persistence/migrations/378-create-conversation-tool-surfaces.ts +29 -0
- package/src/persistence/migrations/379-oauth-providers-response-ok-field.test.ts +92 -0
- package/src/persistence/migrations/379-oauth-providers-response-ok-field.ts +32 -0
- package/src/persistence/migrations/__tests__/157-invite-contact-id.test.ts +149 -0
- package/src/persistence/schema/conversation-tool-surfaces.ts +22 -0
- package/src/persistence/schema/index.ts +1 -0
- package/src/persistence/schema/memory-core.ts +5 -0
- package/src/persistence/schema/oauth.ts +1 -0
- package/src/persistence/steps.ts +15 -3
- package/src/platform/client.test.ts +4 -2
- package/src/platform/client.ts +2 -10
- package/src/plugin-api/constants.ts +2 -0
- package/src/plugin-api/conversation-turn.ts +31 -7
- package/src/plugin-api/index.ts +10 -1
- package/src/plugin-api/plugin-channel-turn-trust.test.ts +133 -0
- package/src/plugin-api/plugin-channel-turn-trust.ts +71 -0
- package/src/plugin-api/types.ts +2 -0
- package/src/plugins/__tests__/installed-plugin-dirs.test.ts +14 -1
- package/src/plugins/__tests__/mcp-servers.test.ts +38 -8
- package/src/plugins/defaults/index.ts +2 -0
- package/src/plugins/defaults/memory/AGENTS.md +14 -2
- package/src/plugins/defaults/memory/__tests__/buffer-file.test.ts +365 -0
- package/src/plugins/defaults/memory/__tests__/buffer-format.test.ts +43 -0
- package/src/plugins/defaults/memory/__tests__/fixtures/buffer-appender.ts +17 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-accounting.test.ts +46 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-cursor-backfill.test.ts +118 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-cursor-preserve.test.ts +83 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-enqueue.test.ts +9 -5
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-job.test.ts +66 -10
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-prompt.test.ts +20 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-state.test.ts +219 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-sweep.test.ts +31 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-trigger-check.test.ts +63 -0
- package/src/plugins/defaults/memory/__tests__/memory-run-evidence.test.ts +161 -0
- package/src/plugins/defaults/memory/__tests__/shutdown-hook.test.ts +4 -0
- package/src/plugins/defaults/memory/buffer-file.ts +354 -0
- package/src/plugins/defaults/memory/buffer-format.ts +40 -0
- package/src/plugins/defaults/memory/context-search/agent-runner.ts +3 -3
- package/src/plugins/defaults/memory/context-search/format.ts +4 -2
- package/src/plugins/defaults/memory/context-search/sources/memory-v2.ts +7 -3
- package/src/plugins/defaults/memory/context-search/sources/workspace.ts +4 -2
- package/src/plugins/defaults/memory/graph/capability-seed.ts +6 -8
- package/src/plugins/defaults/memory/graph/tool-handlers.ts +1 -42
- package/src/plugins/defaults/memory/hooks/init.ts +10 -0
- package/src/plugins/defaults/memory/hooks/message-deleted.ts +21 -0
- package/src/plugins/defaults/memory/hooks/shutdown.ts +2 -0
- package/src/plugins/defaults/memory/injectors.ts +4 -3
- package/src/plugins/defaults/memory/jobs-worker.ts +13 -0
- package/src/plugins/defaults/memory/memory-retrospective-accounting.ts +34 -60
- package/src/plugins/defaults/memory/memory-retrospective-cursor-backfill.ts +126 -0
- package/src/plugins/defaults/memory/memory-retrospective-cursor-preserve.ts +79 -0
- package/src/plugins/defaults/memory/memory-retrospective-cursor.ts +28 -0
- package/src/plugins/defaults/memory/memory-retrospective-enqueue.ts +3 -5
- package/src/plugins/defaults/memory/memory-retrospective-job.ts +67 -184
- package/src/plugins/defaults/memory/memory-retrospective-prompt.ts +1 -1
- package/src/plugins/defaults/memory/memory-retrospective-state.ts +312 -71
- package/src/plugins/defaults/memory/memory-retrospective-sweep.ts +2 -1
- package/src/plugins/defaults/memory/memory-retrospective-trigger-check.ts +5 -3
- package/src/plugins/defaults/memory/memory-run-evidence.ts +213 -0
- package/src/plugins/defaults/memory/skill-md-mtime-poll.ts +9 -0
- package/src/plugins/defaults/memory/src/memory-item-routes.test.ts +1 -1
- package/src/plugins/defaults/memory/startup.ts +28 -2
- package/src/plugins/defaults/memory/substrate/__tests__/consolidation-job.test.ts +407 -99
- package/src/plugins/defaults/memory/substrate/__tests__/consolidation-prompt-flag-gating-guard.test.ts +10 -0
- package/src/plugins/defaults/memory/substrate/__tests__/prompts-consolidation.test.ts +107 -7
- package/src/plugins/defaults/memory/substrate/__tests__/skill-content.test.ts +7 -9
- package/src/plugins/defaults/memory/substrate/consolidation-job.ts +307 -86
- package/src/plugins/defaults/memory/substrate/consolidation-tool-surface.ts +34 -0
- package/src/plugins/defaults/memory/substrate/page-index.ts +7 -1
- package/src/plugins/defaults/memory/substrate/prompts/consolidation.ts +89 -49
- package/src/plugins/defaults/memory/substrate/skill-content.ts +2 -2
- package/src/plugins/defaults/memory/substrate/sweep-job.ts +1 -1
- package/src/plugins/defaults/memory/tools.ts +1 -1
- package/src/plugins/defaults/memory/v1/graph/consolidation.ts +4 -2
- package/src/plugins/defaults/memory/v1/graph/extraction.ts +4 -1
- package/src/plugins/defaults/memory/v1/graph/retriever.ts +4 -1
- package/src/plugins/defaults/memory/v2/__tests__/migration.test.ts +5 -0
- package/src/plugins/defaults/memory/v2/__tests__/reranker.test.ts +5 -2
- package/src/plugins/defaults/memory/v2/reranker.ts +7 -1
- package/src/plugins/defaults/memory/v3/__tests__/card.test.ts +24 -0
- package/src/plugins/defaults/memory/v3/__tests__/injection.test.ts +81 -1
- package/src/plugins/defaults/memory/v3/__tests__/orchestrate.test.ts +87 -0
- package/src/plugins/defaults/memory/v3/__tests__/pool-select.test.ts +125 -0
- package/src/plugins/defaults/memory/v3/__tests__/sections.test.ts +26 -0
- package/src/plugins/defaults/memory/v3/__tests__/shadow-plugin.test.ts +51 -0
- package/src/plugins/defaults/memory/v3/card.ts +4 -2
- package/src/plugins/defaults/memory/v3/injector.ts +212 -178
- package/src/plugins/defaults/memory/v3/orchestrate.ts +86 -22
- package/src/plugins/defaults/memory/v3/pool-select.ts +13 -8
- package/src/plugins/defaults/memory/v3/sections.ts +10 -3
- package/src/plugins/defaults/memory/v3/shadow-plugin.ts +13 -1
- package/src/plugins/defaults/memory/worker.ts +6 -0
- package/src/plugins/defaults/tool-result-truncate/terminal.ts +1 -46
- package/src/plugins/external-plugin-loader.ts +49 -86
- package/src/plugins/installed-plugin-dirs.ts +6 -4
- package/src/plugins/mcp-servers.ts +24 -90
- package/src/plugins/mtime-cache.ts +2 -1
- package/src/plugins/user-loader.ts +2 -1
- package/src/providers/__tests__/connection-model-compat.test.ts +3 -3
- package/src/providers/__tests__/retry-callsite.test.ts +41 -0
- package/src/providers/__tests__/retry-outbound-sanitize.test.ts +176 -0
- package/src/providers/anthropic/client.ts +0 -47
- package/src/providers/gemini/client.ts +9 -23
- package/src/providers/gemini-thought-signature.test.ts +63 -0
- package/src/providers/gemini-thought-signature.ts +44 -0
- package/src/providers/inference/__tests__/endpoint-probe.test.ts +87 -3
- package/src/providers/inference/adapter-factory.ts +6 -0
- package/src/providers/inference/auth.ts +2 -3
- package/src/providers/inference/connection-availability.ts +27 -8
- package/src/providers/inference/endpoint-probe.ts +36 -12
- package/src/providers/inference/resolve-auth.ts +22 -4
- package/src/providers/inference/service-account-token.ts +248 -0
- package/src/providers/jev/client.test.ts +260 -0
- package/src/providers/jev/client.ts +518 -0
- package/src/providers/model-catalog.ts +92 -3
- package/src/providers/model-intents.ts +10 -5
- package/src/providers/openai/__tests__/chat-completions-output-token-limit.test.ts +141 -0
- package/src/providers/openai/__tests__/chat-completions-provider-reasoning.test.ts +146 -18
- package/src/providers/openai/__tests__/chat-completions-provider-thought-signature.test.ts +613 -0
- package/src/providers/openai/__tests__/google-thought-signature.test.ts +235 -0
- package/src/providers/openai/__tests__/tool-choice-mapping.test.ts +83 -1
- package/src/providers/openai/chat-completions-provider.ts +212 -79
- package/src/providers/openai/codex-models.ts +0 -4
- package/src/providers/openai/google-thought-signature.ts +320 -0
- package/src/providers/openai/responses-provider.ts +9 -2
- package/src/providers/opencode/client.test.ts +240 -4
- package/src/providers/opencode/client.ts +129 -3
- package/src/providers/openrouter/client.ts +20 -0
- package/src/providers/outbound-request-sanitize.ts +96 -0
- package/src/providers/retry.ts +30 -19
- package/src/providers/unparseable-tool-args.ts +5 -1
- package/src/runtime/AGENTS.md +9 -3
- package/src/runtime/__tests__/agent-wake.test.ts +86 -1
- package/src/runtime/__tests__/channel-send.test.ts +443 -0
- package/src/runtime/__tests__/desktop-stream-upgrade.test.ts +132 -0
- package/src/runtime/__tests__/host-observe.test.ts +40 -1
- package/src/runtime/__tests__/live-voice-upgrade.test.ts +58 -0
- package/src/runtime/__tests__/runtime-ws-test-utils.ts +72 -0
- package/src/runtime/__tests__/watch-stream-upgrade.test.ts +59 -0
- package/src/runtime/agent-wake.ts +22 -5
- package/src/runtime/assistant-event-hub.ts +21 -3
- package/src/runtime/auth/same-actor.ts +38 -3
- package/src/runtime/channel-send.ts +316 -0
- package/src/runtime/guardian-action-service.ts +2 -17
- package/src/runtime/guardian-reply-router.ts +1 -8
- package/src/runtime/host-observe.ts +11 -1
- package/src/runtime/http-server.ts +160 -151
- package/src/runtime/migrations/__tests__/vbundle-import-policy.test.ts +89 -0
- package/src/runtime/migrations/vbundle-import-policy.ts +33 -11
- package/src/runtime/routes/__tests__/acp-claude-auth-routes.test.ts +16 -5
- package/src/runtime/routes/__tests__/acp-routes.test.ts +178 -4
- package/src/runtime/routes/__tests__/apps-refresh-route.test.ts +275 -0
- package/src/runtime/routes/__tests__/channel-send-routes.test.ts +127 -0
- package/src/runtime/routes/__tests__/conversation-query-routes.test.ts +108 -1
- package/src/runtime/routes/__tests__/default-provider-routes.test.ts +2 -2
- package/src/runtime/routes/__tests__/dictation-cleanup-guard.test.ts +107 -0
- package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +24 -2
- package/src/runtime/routes/__tests__/mcp-add-default-risk.test.ts +16 -12
- package/src/runtime/routes/__tests__/plugins-routes.test.ts +127 -3
- package/src/runtime/routes/acp-claude-auth-routes.ts +11 -3
- package/src/runtime/routes/acp-routes.test.ts +15 -0
- package/src/runtime/routes/acp-routes.ts +84 -165
- package/src/runtime/routes/app-management-routes.ts +90 -2
- package/src/runtime/routes/app-routes.ts +1 -1
- package/src/runtime/routes/browser-context.ts +39 -0
- package/src/runtime/routes/browser-routes.ts +16 -47
- package/src/runtime/routes/browser-tabs-routes.ts +72 -29
- package/src/runtime/routes/channel-route-shared.ts +1 -9
- package/src/runtime/routes/channel-send-routes.ts +85 -0
- package/src/runtime/routes/conversation-list-routes.ts +14 -3
- package/src/runtime/routes/conversation-query-routes.ts +40 -0
- package/src/runtime/routes/conversation-routes.ts +45 -7
- package/src/runtime/routes/credential-routes.ts +20 -14
- package/src/runtime/routes/desktop-setup-routes.test.ts +89 -0
- package/src/runtime/routes/desktop-setup-routes.ts +41 -0
- package/src/runtime/routes/diagnostics-routes.ts +61 -4
- package/src/runtime/routes/events-routes.ts +4 -0
- package/src/runtime/routes/guardian-approval-interception.ts +24 -0
- package/src/runtime/routes/host-cu-routes.ts +6 -0
- package/src/runtime/routes/inbound-message-handler.ts +2 -3
- package/src/runtime/routes/inbound-stages/background-dispatch.test.ts +1 -1
- package/src/runtime/routes/inbound-stages/background-dispatch.ts +7 -4
- package/src/runtime/routes/index.ts +4 -2
- package/src/runtime/routes/inference-profile-session-handler.ts +11 -1
- package/src/runtime/routes/inference-profiles-routes.ts +9 -0
- package/src/runtime/routes/log-export-routes.ts +2 -0
- package/src/runtime/routes/mcp-auth-routes.ts +141 -137
- package/src/runtime/routes/migration-routes.ts +6 -20
- package/src/runtime/routes/oauth-commands-routes.ts +98 -45
- package/src/runtime/routes/oauth-providers.ts +11 -0
- package/src/runtime/routes/oauth-request-hints.test.ts +246 -0
- package/src/runtime/routes/oauth-request-hints.ts +149 -0
- package/src/runtime/routes/platform-routes.ts +18 -15
- package/src/runtime/routes/plugins-routes.ts +83 -50
- package/src/runtime/routes/schedule-routes.ts +2 -0
- package/src/runtime/routes/secret-routes.ts +99 -119
- package/src/runtime/routes/workspace-routes.test.ts +55 -26
- package/src/runtime/routes/workspace-routes.ts +46 -63
- package/src/runtime/routes/workspace-tree-walk.test.ts +244 -0
- package/src/runtime/routes/workspace-tree-walk.ts +240 -0
- package/src/runtime/routes/workspace-utils.ts +17 -0
- package/src/runtime/services/__tests__/conversation-serializer.test.ts +28 -0
- package/src/runtime/services/conversation-serializer.ts +16 -3
- package/src/runtime/sync/resource-sync-events.ts +6 -1
- package/src/schedule/__tests__/worker-mcp-bootstrap.test.ts +107 -0
- package/src/schedule/__tests__/worker-mcp-readiness.test.ts +173 -0
- package/src/schedule/__tests__/worker-mcp-tools.test.ts +13 -47
- package/src/schedule/run-script.ts +4 -3
- package/src/schedule/schedule-store.ts +9 -2
- package/src/schedule/scheduler.ts +47 -11
- package/src/schedule/tool-surface-readiness.ts +65 -0
- package/src/schedule/worker-mcp.ts +85 -0
- package/src/schedule/worker.ts +9 -31
- package/src/security/__tests__/ces-http-record-backend.test.ts +137 -0
- package/src/security/ces-http-record-backend.ts +120 -0
- package/src/security/ces-rpc-record-backend.ts +7 -0
- package/src/security/secure-keys.ts +60 -48
- package/src/security/untrusted-content.ts +3 -1
- package/src/skills/__tests__/workspace-skill-md-signature.test.ts +60 -0
- package/src/skills/managed-store.ts +153 -31
- package/src/skills/workspace-skill-md-signature.ts +50 -0
- package/src/subagent/consult-context.ts +12 -25
- package/src/tools/__tests__/side-effect-abort-guard.test.ts +1 -0
- package/src/tools/acp/list-agents.test.ts +3 -1
- package/src/tools/acp/spawn.test.ts +146 -11
- package/src/tools/acp/spawn.ts +21 -3
- package/src/tools/acp/status.test.ts +349 -0
- package/src/tools/acp/status.ts +112 -9
- package/src/tools/app-control/skill-proxy-bridge.ts +2 -1
- package/src/tools/browser/browser-execution.ts +45 -14
- package/src/tools/browser/cdp-client/__tests__/factory.test.ts +69 -0
- package/src/tools/browser/cdp-client/accessibility-snapshot.ts +9 -3
- package/src/tools/browser/cdp-client/cdp-inspect/__tests__/ws-transport.test.ts +1 -0
- package/src/tools/browser/cdp-client/cdp-inspect/discovery.ts +2 -1
- package/src/tools/browser/cdp-client/cdp-inspect/ws-transport.ts +5 -0
- package/src/tools/browser/cdp-client/extension-cdp-client.ts +3 -2
- package/src/tools/browser/network-recorder.ts +2 -1
- package/src/tools/capability-offer.test.ts +22 -2
- package/src/tools/capability-offer.ts +16 -1
- package/src/tools/computer-use/definitions.ts +123 -5
- package/src/tools/computer-use/skill-proxy-bridge.ts +2 -1
- package/src/tools/credentials/metadata-store.ts +100 -0
- package/src/tools/filesystem/search.ts +2 -1
- package/src/tools/host-filesystem/edit.test.ts +2 -1
- package/src/tools/host-filesystem/edit.ts +2 -2
- package/src/tools/host-filesystem/read.test.ts +2 -1
- package/src/tools/host-filesystem/read.ts +2 -2
- package/src/tools/host-filesystem/transfer.test.ts +2 -1
- package/src/tools/host-filesystem/transfer.ts +2 -2
- package/src/tools/host-filesystem/write.test.ts +2 -1
- package/src/tools/host-filesystem/write.ts +2 -2
- package/src/tools/host-terminal/host-shell.test.ts +16 -0
- package/src/tools/host-terminal/host-shell.ts +16 -26
- package/src/tools/network/web-search.ts +6 -1
- package/src/tools/policy-context.ts +6 -7
- package/src/tools/shared/filesystem/file-ops-service.ts +1 -31
- package/src/tools/shared/shell-output.test.ts +10 -0
- package/src/tools/shared/shell-output.ts +14 -2
- package/src/tools/skills/find-similar-skills.test.ts +214 -2
- package/src/tools/skills/find-similar-skills.ts +62 -9
- package/src/tools/skills/resolve-execute-invocation.ts +24 -0
- package/src/tools/skills/sandbox-runner.ts +13 -2
- package/src/tools/skills/scaffold-managed.ts +78 -16
- package/src/tools/subagent/spawn.ts +5 -8
- package/src/tools/terminal/__tests__/safe-env.test.ts +33 -4
- package/src/tools/terminal/__tests__/sanitized-bash.test.ts +43 -0
- package/src/tools/terminal/safe-env.ts +35 -24
- package/src/tools/terminal/sanitized-bash.ts +118 -0
- package/src/tools/terminal/shell-launch.test.ts +162 -0
- package/src/tools/terminal/shell.test.ts +29 -0
- package/src/tools/terminal/shell.ts +13 -7
- package/src/tools/ui-surface/channel-variants.ts +10 -59
- package/src/tools/watch/watch-retro-report.ts +0 -8
- package/src/types/host-capabilities.ts +5 -3
- package/src/util/__tests__/language-subtag.test.ts +23 -1
- package/src/util/abort-reasons.ts +12 -1
- package/src/util/ca-bundle.ts +13 -0
- package/src/util/host-process.test.ts +17 -1
- package/src/util/host-process.ts +24 -0
- package/src/util/language-subtag.ts +16 -0
- package/src/util/platform.ts +8 -0
- package/src/util/plugin-manifest.ts +203 -0
- package/src/util/unicode.ts +44 -1
- package/src/watch/watch-retro.ts +6 -8
- package/src/watch/watch-timeline.ts +2 -2
- package/src/workflows/run-manager.ts +2 -1
- package/src/workspace/git-porcelain.ts +66 -0
- package/src/workspace/git-service.ts +6 -55
- package/src/workspace/migrations/154-repair-retired-codex-gpt-5-4-model-ids.ts +258 -0
- package/src/workspace/migrations/155-move-front-model-config-to-voice.ts +78 -0
- package/src/workspace/migrations/156-extract-workspace-mcp-json.ts +214 -0
- package/src/workspace/migrations/__tests__/155-move-front-model-config-to-voice.test.ts +102 -0
- package/src/workspace/migrations/__tests__/156-extract-workspace-mcp-json.test.ts +199 -0
- package/src/workspace/migrations/registry.ts +6 -0
- package/src/__tests__/app-source-watcher.test.ts +0 -218
- package/src/__tests__/config-watcher-skill-reseed.test.ts +0 -359
- package/src/__tests__/speaker-identification.test.ts +0 -56
- package/src/calls/speaker-identification.ts +0 -233
- package/src/daemon/app-source-watcher.ts +0 -218
- package/src/messaging/providers/whatsapp/client.ts +0 -29
- package/src/runtime/routes/debug-bash-routes.ts +0 -173
- package/src/util/fs-watcher-error.ts +0 -36
- /package/src/{live-voice → calls}/progress-phrases.ts +0 -0
|
@@ -42,9 +42,14 @@ const MAX_HISTORY_ENTRIES = 10;
|
|
|
42
42
|
const LOOP_DETECTION_WINDOW = 3;
|
|
43
43
|
const CONSECUTIVE_UNCHANGED_WARNING_THRESHOLD = 2;
|
|
44
44
|
|
|
45
|
+
const SEQUENCE_TOOL = "computer_use_sequence";
|
|
46
|
+
|
|
47
|
+
const SCREENSHOT_OMITTED_MESSAGE =
|
|
48
|
+
"Screenshot omitted: the accessibility tree above is current. Pass include_screenshot: true on computer_use_observe to see the screen.";
|
|
49
|
+
|
|
45
50
|
// computer_use_key combos that change only selection/cursor/clipboard state.
|
|
46
|
-
// The AX tree models none of these, so they always produce an empty diff
|
|
47
|
-
// exempt
|
|
51
|
+
// The AX tree models none of these, so they always produce an empty diff and
|
|
52
|
+
// are exempt from the unchanged-tree signal.
|
|
48
53
|
// Stored in canonical form (see canonicalizeKeyCombo): modifier aliases
|
|
49
54
|
// normalized and ordered, so `cmd + a`, `command+a`, `alt+tab`, `tab+shift`
|
|
50
55
|
// all match.
|
|
@@ -113,6 +118,8 @@ export interface CuObservationResult {
|
|
|
113
118
|
executionResult?: string;
|
|
114
119
|
executionError?: string;
|
|
115
120
|
userGuidance?: string;
|
|
121
|
+
/** Per-phase helper timings in milliseconds, absent on older helpers. */
|
|
122
|
+
timings?: Record<string, number>;
|
|
116
123
|
}
|
|
117
124
|
|
|
118
125
|
export interface ActionRecord {
|
|
@@ -122,12 +129,28 @@ export interface ActionRecord {
|
|
|
122
129
|
reasoning?: string;
|
|
123
130
|
}
|
|
124
131
|
|
|
132
|
+
// Steps whose empty diff says nothing about whether they worked: waiting and
|
|
133
|
+
// observing change nothing by design, and an AppleScript reports its own
|
|
134
|
+
// outcome in the execution result, often for an app whose content the tree
|
|
135
|
+
// cannot see. They neither warn nor move the unchanged streak.
|
|
136
|
+
const NO_AX_DIFF_TOOLS = new Set([
|
|
137
|
+
"computer_use_wait",
|
|
138
|
+
"computer_use_observe",
|
|
139
|
+
"computer_use_run_applescript",
|
|
140
|
+
]);
|
|
141
|
+
|
|
142
|
+
type ActionIdentity = Pick<ActionRecord, "toolName" | "input">;
|
|
143
|
+
|
|
144
|
+
function isNoDiffTool(action: ActionIdentity | undefined): boolean {
|
|
145
|
+
return action !== undefined && NO_AX_DIFF_TOOLS.has(action.toolName);
|
|
146
|
+
}
|
|
147
|
+
|
|
125
148
|
/**
|
|
126
149
|
* True when `action` is a computer_use_key press whose key only mutates
|
|
127
|
-
* selection/cursor/clipboard state
|
|
150
|
+
* selection/cursor/clipboard state: changes the AX tree cannot represent, so
|
|
128
151
|
* an empty diff is expected rather than a sign the action did nothing.
|
|
129
152
|
*/
|
|
130
|
-
function isNoDiffKeyAction(action:
|
|
153
|
+
function isNoDiffKeyAction(action: ActionIdentity | undefined): boolean {
|
|
131
154
|
if (action?.toolName !== "computer_use_key") {
|
|
132
155
|
return false;
|
|
133
156
|
}
|
|
@@ -161,6 +184,40 @@ function actionSignature(record: ActionRecord): string {
|
|
|
161
184
|
return `${record.toolName}:${JSON.stringify(record.input)}`;
|
|
162
185
|
}
|
|
163
186
|
|
|
187
|
+
/**
|
|
188
|
+
* The key a desktop's first look is tracked under. An untargeted request has
|
|
189
|
+
* no client id, so it gets a key no real id can take.
|
|
190
|
+
*/
|
|
191
|
+
function observedTargetKey(targetClientId: string | undefined): string {
|
|
192
|
+
return targetClientId === undefined
|
|
193
|
+
? "\u0000untargeted"
|
|
194
|
+
: `client:${targetClientId}`;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/** Whether `input` scopes the observation to one window or display. */
|
|
198
|
+
function hasCaptureTarget(input: Record<string, unknown>): boolean {
|
|
199
|
+
return (
|
|
200
|
+
Object.hasOwn(input, "capture_window_id") ||
|
|
201
|
+
Object.hasOwn(input, "captureWindowId") ||
|
|
202
|
+
Object.hasOwn(input, "captureDisplayId")
|
|
203
|
+
);
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
/**
|
|
207
|
+
* Whether the resolved client claimed `capability` on its connection. An
|
|
208
|
+
* untargeted request has no client to vouch for it, so it never qualifies.
|
|
209
|
+
*/
|
|
210
|
+
function clientAdvertises(
|
|
211
|
+
targetClientId: string | undefined,
|
|
212
|
+
capability: "host_cu_window_capture" | "host_cu_sequence",
|
|
213
|
+
): boolean {
|
|
214
|
+
const client =
|
|
215
|
+
targetClientId == null
|
|
216
|
+
? undefined
|
|
217
|
+
: assistantEventHub.getClientById(targetClientId);
|
|
218
|
+
return client?.capabilities.includes(capability) ?? false;
|
|
219
|
+
}
|
|
220
|
+
|
|
164
221
|
// ---------------------------------------------------------------------------
|
|
165
222
|
// HostCuProxy
|
|
166
223
|
// ---------------------------------------------------------------------------
|
|
@@ -172,8 +229,42 @@ export class HostCuProxy {
|
|
|
172
229
|
private _previousAXTree: string | undefined;
|
|
173
230
|
private _consecutiveUnchangedSteps = 0;
|
|
174
231
|
private _actionHistory: ActionRecord[] = [];
|
|
175
|
-
/**
|
|
176
|
-
|
|
232
|
+
/**
|
|
233
|
+
* Desktops an unscoped observation has come back from since the last reset,
|
|
234
|
+
* keyed by `observedTargetKey`. A first look is per desktop: having seen one
|
|
235
|
+
* machine says nothing about another the conversation switches to.
|
|
236
|
+
*/
|
|
237
|
+
private _observedTargets = new Set<string>();
|
|
238
|
+
/**
|
|
239
|
+
* Bumped on every reset. A request carries the value it was dispatched
|
|
240
|
+
* under, so an observation that lands after a reset cannot restore state
|
|
241
|
+
* that reset cleared.
|
|
242
|
+
*/
|
|
243
|
+
private _resetGeneration = 0;
|
|
244
|
+
/**
|
|
245
|
+
* Desktops a request has been sent to since the last reset, keyed by
|
|
246
|
+
* `observedTargetKey`, so the end of a task can reach each one.
|
|
247
|
+
*/
|
|
248
|
+
private _dispatchedTargets = new Map<string, string | undefined>();
|
|
249
|
+
/**
|
|
250
|
+
* Owned request IDs mapped to whether their observation is scoped, whether
|
|
251
|
+
* the helper was told to skip the screenshot, and when the request was
|
|
252
|
+
* dispatched. The dispatch time gives the round trip, which is the part of
|
|
253
|
+
* a step the helper's own timings cannot see.
|
|
254
|
+
*/
|
|
255
|
+
private _ownedRequests = new Map<
|
|
256
|
+
string,
|
|
257
|
+
{
|
|
258
|
+
scoped: boolean;
|
|
259
|
+
screenshotSkipped: boolean;
|
|
260
|
+
targetKey: string;
|
|
261
|
+
resetGeneration: number;
|
|
262
|
+
dispatchedAt: number;
|
|
263
|
+
toolName: string;
|
|
264
|
+
input: Record<string, unknown>;
|
|
265
|
+
step: number;
|
|
266
|
+
}
|
|
267
|
+
>();
|
|
177
268
|
|
|
178
269
|
constructor(maxSteps = loadConfig().maxStepsPerSession) {
|
|
179
270
|
this._maxSteps = maxSteps;
|
|
@@ -288,11 +379,7 @@ export class HostCuProxy {
|
|
|
288
379
|
}
|
|
289
380
|
// Never dispatch this option to a legacy executor: it would forward the
|
|
290
381
|
// unknown snake-case key and silently capture the entire desktop.
|
|
291
|
-
|
|
292
|
-
resolvedTargetClientId == null
|
|
293
|
-
? undefined
|
|
294
|
-
: assistantEventHub.getClientById(resolvedTargetClientId);
|
|
295
|
-
if (!client?.capabilities.includes("host_cu_window_capture")) {
|
|
382
|
+
if (!clientAdvertises(resolvedTargetClientId, "host_cu_window_capture")) {
|
|
296
383
|
return Promise.resolve({
|
|
297
384
|
content:
|
|
298
385
|
"Window-only observation requires a connected client advertising host_cu_window_capture; update the desktop app before retrying. No capture was requested.",
|
|
@@ -300,14 +387,40 @@ export class HostCuProxy {
|
|
|
300
387
|
});
|
|
301
388
|
}
|
|
302
389
|
}
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
390
|
+
// An older helper answers an unknown tool name as done, which would end
|
|
391
|
+
// the session with nothing run.
|
|
392
|
+
if (
|
|
393
|
+
toolName === SEQUENCE_TOOL &&
|
|
394
|
+
!clientAdvertises(resolvedTargetClientId, "host_cu_sequence")
|
|
395
|
+
) {
|
|
396
|
+
return Promise.resolve({
|
|
397
|
+
content:
|
|
398
|
+
"Batched actions require a desktop app that supports computer_use_sequence; update the desktop app, or send the actions one at a time. Nothing was run.",
|
|
399
|
+
isError: true,
|
|
400
|
+
});
|
|
401
|
+
}
|
|
402
|
+
const scopedObservation = hasCaptureTarget(input);
|
|
307
403
|
if (scopedObservation) {
|
|
308
404
|
this._previousAXTree = undefined;
|
|
309
405
|
this._consecutiveUnchangedSteps = 0;
|
|
310
406
|
}
|
|
407
|
+
const targetKey = observedTargetKey(resolvedTargetClientId);
|
|
408
|
+
// Pointing never reaches the helper's capture path, so its input goes out
|
|
409
|
+
// as given. Every other request carries the screenshot decision made now,
|
|
410
|
+
// at dispatch, and never the model-facing snake_case key.
|
|
411
|
+
let dispatchInput = input;
|
|
412
|
+
let screenshotSkipped = false;
|
|
413
|
+
if (toolName !== POINT_AT_PROXY_TOOL) {
|
|
414
|
+
const { include_screenshot: _includeScreenshot, ...rest } = input;
|
|
415
|
+
screenshotSkipped = !this.shouldAttachScreenshot(
|
|
416
|
+
toolName,
|
|
417
|
+
input,
|
|
418
|
+
targetKey,
|
|
419
|
+
);
|
|
420
|
+
dispatchInput = screenshotSkipped
|
|
421
|
+
? { ...rest, includeScreenshot: false }
|
|
422
|
+
: rest;
|
|
423
|
+
}
|
|
311
424
|
const requestId = uuid();
|
|
312
425
|
|
|
313
426
|
return new Promise<ToolExecutionResult>((resolve, reject) => {
|
|
@@ -351,7 +464,17 @@ export class HostCuProxy {
|
|
|
351
464
|
detachAbort = () => signal.removeEventListener("abort", onAbort);
|
|
352
465
|
}
|
|
353
466
|
|
|
354
|
-
this.
|
|
467
|
+
this._dispatchedTargets.set(targetKey, resolvedTargetClientId);
|
|
468
|
+
this._ownedRequests.set(requestId, {
|
|
469
|
+
scoped: scopedObservation,
|
|
470
|
+
screenshotSkipped,
|
|
471
|
+
targetKey,
|
|
472
|
+
resetGeneration: this._resetGeneration,
|
|
473
|
+
dispatchedAt: Date.now(),
|
|
474
|
+
toolName,
|
|
475
|
+
input,
|
|
476
|
+
step: stepNumber,
|
|
477
|
+
});
|
|
355
478
|
|
|
356
479
|
pendingInteractions.register(requestId, {
|
|
357
480
|
conversationId,
|
|
@@ -375,7 +498,7 @@ export class HostCuProxy {
|
|
|
375
498
|
requestId,
|
|
376
499
|
conversationId,
|
|
377
500
|
toolName,
|
|
378
|
-
input,
|
|
501
|
+
input: dispatchInput,
|
|
379
502
|
stepNumber,
|
|
380
503
|
reasoning,
|
|
381
504
|
...(resolvedTargetClientId != null
|
|
@@ -403,7 +526,8 @@ export class HostCuProxy {
|
|
|
403
526
|
requestId: string,
|
|
404
527
|
observation: CuObservationResult,
|
|
405
528
|
): ToolExecutionResult | undefined {
|
|
406
|
-
const
|
|
529
|
+
const owned = this._ownedRequests.get(requestId);
|
|
530
|
+
const scopedObservation = owned?.scoped ?? false;
|
|
407
531
|
this._ownedRequests.delete(requestId);
|
|
408
532
|
const interaction = pendingInteractions.resolve(requestId, "answered");
|
|
409
533
|
if (!interaction?.rpcResolve) {
|
|
@@ -411,20 +535,64 @@ export class HostCuProxy {
|
|
|
411
535
|
return undefined;
|
|
412
536
|
}
|
|
413
537
|
|
|
538
|
+
// Label the line from what this request was dispatched with, never from
|
|
539
|
+
// current proxy state. One model response can dispatch several CU tools,
|
|
540
|
+
// and the agent loop runs them concurrently, so a later call can advance
|
|
541
|
+
// the history and the step count before an earlier observation lands.
|
|
542
|
+
// Reading them here would file each measurement under whichever tool was
|
|
543
|
+
// dispatched last, and would mislabel computer_use_point_at, which never
|
|
544
|
+
// records an action at all.
|
|
545
|
+
if (owned) {
|
|
546
|
+
log.info(
|
|
547
|
+
{
|
|
548
|
+
requestId,
|
|
549
|
+
toolName: owned.toolName,
|
|
550
|
+
step: owned.step,
|
|
551
|
+
roundTripMs: Date.now() - owned.dispatchedAt,
|
|
552
|
+
...(observation.timings ? { helper: observation.timings } : {}),
|
|
553
|
+
},
|
|
554
|
+
"Host CU step timings",
|
|
555
|
+
);
|
|
556
|
+
}
|
|
557
|
+
|
|
414
558
|
// A targeted snapshot has no comparable action/diff baseline; neither it
|
|
415
|
-
// nor the first desktop observation after it can imply
|
|
416
|
-
|
|
559
|
+
// nor the first desktop observation after it can imply an unchanged tree.
|
|
560
|
+
// A response dispatched before the last reset belongs to a finished run,
|
|
561
|
+
// so it must not change any state the new run has started building.
|
|
562
|
+
const fromCurrentRun =
|
|
563
|
+
owned === undefined || owned.resetGeneration === this._resetGeneration;
|
|
564
|
+
if (scopedObservation && fromCurrentRun) {
|
|
417
565
|
this._previousAXTree = undefined;
|
|
418
566
|
this._consecutiveUnchangedSteps = 0;
|
|
419
567
|
}
|
|
420
568
|
const prevAXTree = this._previousAXTree;
|
|
569
|
+
// Judge the empty diff against the action this request carried, for the
|
|
570
|
+
// same reason the timings line does.
|
|
571
|
+
const action: ActionIdentity | undefined = owned
|
|
572
|
+
? { toolName: owned.toolName, input: owned.input }
|
|
573
|
+
: this.lastRecordedAction();
|
|
421
574
|
const comparableObservation = scopedObservation
|
|
422
575
|
? { ...observation, axDiff: undefined, secondaryWindows: undefined }
|
|
423
576
|
: observation;
|
|
424
|
-
|
|
425
|
-
|
|
577
|
+
// Pointing observes nothing, so its response leaves the screen state alone.
|
|
578
|
+
if (
|
|
579
|
+
fromCurrentRun &&
|
|
580
|
+
!scopedObservation &&
|
|
581
|
+
owned?.toolName !== POINT_AT_PROXY_TOOL
|
|
582
|
+
) {
|
|
583
|
+
this.updateStateFromObservation(comparableObservation, action);
|
|
584
|
+
// A desktop has had its first look only once pixels from it arrived,
|
|
585
|
+
// so a failed capture leaves the next request asking again.
|
|
586
|
+
if (owned && observation.screenshot) {
|
|
587
|
+
this._observedTargets.add(owned.targetKey);
|
|
588
|
+
}
|
|
426
589
|
}
|
|
427
|
-
const result = this.formatObservation(
|
|
590
|
+
const result = this.formatObservation(
|
|
591
|
+
comparableObservation,
|
|
592
|
+
prevAXTree,
|
|
593
|
+
owned?.screenshotSkipped ?? false,
|
|
594
|
+
action,
|
|
595
|
+
);
|
|
428
596
|
interaction.rpcResolve(result);
|
|
429
597
|
return result;
|
|
430
598
|
}
|
|
@@ -454,12 +622,65 @@ export class HostCuProxy {
|
|
|
454
622
|
}
|
|
455
623
|
}
|
|
456
624
|
|
|
625
|
+
/**
|
|
626
|
+
* Finish the task: tell every desktop this task drove that it is over, then
|
|
627
|
+
* reset. The notice rides `host_cu_cancel` with a fresh request ID, which
|
|
628
|
+
* matches nothing in flight; the helper treats any cancel as the run
|
|
629
|
+
* stopping and puts the pointer back where the user left it, instead of
|
|
630
|
+
* waiting out its idle fallback.
|
|
631
|
+
*/
|
|
632
|
+
endTask(conversationId: string): void {
|
|
633
|
+
for (const targetClientId of this._dispatchedTargets.values()) {
|
|
634
|
+
try {
|
|
635
|
+
broadcastMessage(
|
|
636
|
+
{
|
|
637
|
+
type: "host_cu_cancel",
|
|
638
|
+
requestId: uuid(),
|
|
639
|
+
conversationId,
|
|
640
|
+
...(targetClientId != null ? { targetClientId } : {}),
|
|
641
|
+
},
|
|
642
|
+
conversationId,
|
|
643
|
+
{ targetClientId },
|
|
644
|
+
);
|
|
645
|
+
} catch {
|
|
646
|
+
// Best-effort: the helper still returns the pointer after its idle delay.
|
|
647
|
+
}
|
|
648
|
+
}
|
|
649
|
+
this.reset();
|
|
650
|
+
}
|
|
651
|
+
|
|
457
652
|
/** Reset all CU state. Called on terminal tools (computer_use_done, etc.). */
|
|
458
653
|
reset(): void {
|
|
654
|
+
this._dispatchedTargets.clear();
|
|
459
655
|
this._stepCount = 0;
|
|
460
656
|
this._previousAXTree = undefined;
|
|
461
657
|
this._consecutiveUnchangedSteps = 0;
|
|
462
658
|
this._actionHistory = [];
|
|
659
|
+
this._observedTargets.clear();
|
|
660
|
+
this._resetGeneration++;
|
|
661
|
+
}
|
|
662
|
+
|
|
663
|
+
/**
|
|
664
|
+
* Whether the request about to be dispatched should carry a screenshot. The
|
|
665
|
+
* accessibility tree comes back every step. Pixels come back after every
|
|
666
|
+
* action, so the model sees what its action did, on the first look since
|
|
667
|
+
* the last reset, for a window- or display-scoped capture, and when the
|
|
668
|
+
* model asks for them. Only a plain observation leaves whether the tree is
|
|
669
|
+
* enough to the model.
|
|
670
|
+
*/
|
|
671
|
+
private shouldAttachScreenshot(
|
|
672
|
+
toolName: string,
|
|
673
|
+
input: Record<string, unknown>,
|
|
674
|
+
targetKey: string,
|
|
675
|
+
): boolean {
|
|
676
|
+
if (toolName !== "computer_use_observe") {
|
|
677
|
+
return true;
|
|
678
|
+
}
|
|
679
|
+
return (
|
|
680
|
+
!this._observedTargets.has(targetKey) ||
|
|
681
|
+
hasCaptureTarget(input) ||
|
|
682
|
+
input.include_screenshot === true
|
|
683
|
+
);
|
|
463
684
|
}
|
|
464
685
|
|
|
465
686
|
// ---------------------------------------------------------------------------
|
|
@@ -474,6 +695,8 @@ export class HostCuProxy {
|
|
|
474
695
|
formatObservation(
|
|
475
696
|
obs: CuObservationResult,
|
|
476
697
|
previousAXTree?: string,
|
|
698
|
+
screenshotSkipped = false,
|
|
699
|
+
action: ActionIdentity | undefined = this.lastRecordedAction(),
|
|
477
700
|
): ToolExecutionResult {
|
|
478
701
|
const prevTree = previousAXTree;
|
|
479
702
|
const parts: string[] = [];
|
|
@@ -492,24 +715,17 @@ export class HostCuProxy {
|
|
|
492
715
|
parts.push(obs.axDiff);
|
|
493
716
|
parts.push("");
|
|
494
717
|
} else if (prevTree != null && obs.axTree != null) {
|
|
495
|
-
|
|
496
|
-
this._actionHistory.length > 0
|
|
497
|
-
? this._actionHistory[this._actionHistory.length - 1]
|
|
498
|
-
: undefined;
|
|
499
|
-
const isWaitAction = lastAction?.toolName === "computer_use_wait";
|
|
500
|
-
const isNoDiffKey = isNoDiffKeyAction(lastAction);
|
|
501
|
-
|
|
502
|
-
if (!isWaitAction && !isNoDiffKey) {
|
|
718
|
+
if (!isNoDiffTool(action) && !isNoDiffKeyAction(action)) {
|
|
503
719
|
if (
|
|
504
720
|
this._consecutiveUnchangedSteps >=
|
|
505
721
|
CONSECUTIVE_UNCHANGED_WARNING_THRESHOLD
|
|
506
722
|
) {
|
|
507
723
|
parts.push(
|
|
508
|
-
`WARNING: ${this._consecutiveUnchangedSteps} consecutive actions
|
|
724
|
+
`WARNING: the accessibility tree did not change across ${this._consecutiveUnchangedSteps} consecutive actions. If the screenshot does not show them working either, try a different approach.`,
|
|
509
725
|
);
|
|
510
726
|
} else {
|
|
511
727
|
parts.push(
|
|
512
|
-
"
|
|
728
|
+
"The accessibility tree did not change after your last action. Apps that draw their own content (timelines, canvases, spreadsheet grids) change without it, so check the screenshot before deciding the action did nothing.",
|
|
513
729
|
);
|
|
514
730
|
}
|
|
515
731
|
parts.push("");
|
|
@@ -552,6 +768,15 @@ export class HostCuProxy {
|
|
|
552
768
|
parts.push(...screenshotMeta);
|
|
553
769
|
}
|
|
554
770
|
|
|
771
|
+
// Only a deliberate omission is announced, and only beside a tree the line
|
|
772
|
+
// can point at. A step that asked for pixels and got none is a capture
|
|
773
|
+
// failure, which this line must not disguise. A refused or failed step
|
|
774
|
+
// still carries the line, since that is where asking for pixels helps most.
|
|
775
|
+
const isError = obs.executionError != null;
|
|
776
|
+
if (screenshotSkipped && !obs.screenshot && obs.axTree) {
|
|
777
|
+
parts.push("", SCREENSHOT_OMITTED_MESSAGE);
|
|
778
|
+
}
|
|
779
|
+
|
|
555
780
|
const content = parts.join("\n").trim() || "Action executed";
|
|
556
781
|
|
|
557
782
|
const contentBlocks: ContentBlock[] = [];
|
|
@@ -566,8 +791,6 @@ export class HostCuProxy {
|
|
|
566
791
|
});
|
|
567
792
|
}
|
|
568
793
|
|
|
569
|
-
const isError = obs.executionError != null;
|
|
570
|
-
|
|
571
794
|
return {
|
|
572
795
|
content: isError
|
|
573
796
|
? `Action failed: ${obs.executionError}\n\n${content}`
|
|
@@ -617,19 +840,26 @@ export class HostCuProxy {
|
|
|
617
840
|
// Private helpers
|
|
618
841
|
// ---------------------------------------------------------------------------
|
|
619
842
|
|
|
620
|
-
private
|
|
843
|
+
private lastRecordedAction(): ActionRecord | undefined {
|
|
844
|
+
return this._actionHistory.at(-1);
|
|
845
|
+
}
|
|
846
|
+
|
|
847
|
+
private updateStateFromObservation(
|
|
848
|
+
obs: CuObservationResult,
|
|
849
|
+
action: ActionIdentity | undefined,
|
|
850
|
+
): void {
|
|
621
851
|
if (this._stepCount > 0) {
|
|
622
|
-
|
|
623
|
-
this._actionHistory.length > 0
|
|
624
|
-
? this._actionHistory[this._actionHistory.length - 1]
|
|
625
|
-
: undefined;
|
|
626
|
-
if (obs.axDiff != null || isNoDiffKeyAction(lastAction)) {
|
|
852
|
+
if (obs.axDiff != null || isNoDiffKeyAction(action)) {
|
|
627
853
|
// A real diff, or an exempt key whose effect is invisible by design,
|
|
628
|
-
// breaks the no-effect streak
|
|
629
|
-
// stale count
|
|
630
|
-
// into a false "consecutive" escalation.
|
|
854
|
+
// breaks the no-effect streak. Clearing it rather than preserving a
|
|
855
|
+
// stale count keeps an intervening cmd+a from bridging two no-op
|
|
856
|
+
// actions into a false "consecutive" escalation.
|
|
631
857
|
this._consecutiveUnchangedSteps = 0;
|
|
632
|
-
} else if (
|
|
858
|
+
} else if (
|
|
859
|
+
!isNoDiffTool(action) &&
|
|
860
|
+
this._previousAXTree != null &&
|
|
861
|
+
obs.axTree != null
|
|
862
|
+
) {
|
|
633
863
|
this._consecutiveUnchangedSteps++;
|
|
634
864
|
}
|
|
635
865
|
}
|
|
@@ -6,10 +6,10 @@ import {
|
|
|
6
6
|
broadcastMessage,
|
|
7
7
|
} from "../runtime/assistant-event-hub.js";
|
|
8
8
|
import {
|
|
9
|
-
ambiguousSameUserError,
|
|
10
9
|
enforceSameActorOrErrorResult,
|
|
11
|
-
|
|
10
|
+
pickMostRecentSameUserClient,
|
|
12
11
|
snapshotHostProxyActorPrincipalId,
|
|
12
|
+
unavailableSameUserClientError,
|
|
13
13
|
} from "../runtime/auth/same-actor.js";
|
|
14
14
|
import * as pendingInteractions from "../runtime/pending-interactions.js";
|
|
15
15
|
import { readAudioBase64 } from "../tools/shared/filesystem/audio-read.js";
|
|
@@ -102,19 +102,17 @@ export class HostFileProxy {
|
|
|
102
102
|
});
|
|
103
103
|
}
|
|
104
104
|
} else {
|
|
105
|
-
|
|
106
|
-
// (multi-machine) cases so a single targeted-style request cannot
|
|
107
|
-
// fan out across the user's machines.
|
|
108
|
-
const resolved = pickSameUserAutoResolve({
|
|
105
|
+
resolvedTargetClientId = pickMostRecentSameUserClient({
|
|
109
106
|
hub: assistantEventHub,
|
|
110
107
|
capability: "host_file",
|
|
111
108
|
sourceActorPrincipalId,
|
|
112
109
|
});
|
|
113
|
-
if (
|
|
114
|
-
|
|
110
|
+
if (
|
|
111
|
+
resolvedTargetClientId === undefined &&
|
|
112
|
+
sourceActorPrincipalId !== undefined
|
|
113
|
+
) {
|
|
114
|
+
return Promise.resolve(unavailableSameUserClientError("host_file"));
|
|
115
115
|
}
|
|
116
|
-
resolvedTargetClientId =
|
|
117
|
-
resolved.kind === "match" ? resolved.clientId : undefined;
|
|
118
116
|
}
|
|
119
117
|
|
|
120
118
|
// Same-user check: targeted host_file requests must be bound to the same
|
|
@@ -9,6 +9,8 @@
|
|
|
9
9
|
* - `daemon/process-message.ts` (create path, prepareConversationForMessage)
|
|
10
10
|
* - `daemon/conversation-process.ts` `drainSingleMessage` (re-add after dequeue)
|
|
11
11
|
* - `daemon/conversation-process.ts` `drainBatch` (re-add after dequeue)
|
|
12
|
+
* - `calls/voice-session-bridge.ts` (tool-capable legs of a live-voice
|
|
13
|
+
* session opened from the macOS desktop client)
|
|
12
14
|
*
|
|
13
15
|
* The create paths additionally instantiate the proxy itself; that
|
|
14
16
|
* instantiation logic is per-proxy-class and stays inline at each create
|
|
@@ -105,12 +107,13 @@ export const HOST_PROXY_SKILL_PREACTIVATIONS: ReadonlyArray<{
|
|
|
105
107
|
*/
|
|
106
108
|
/**
|
|
107
109
|
* Capabilities a client asks for on its connection rather than getting from
|
|
108
|
-
* what it is.
|
|
110
|
+
* what it is. Each rides the host_cu transport and is answered only by a
|
|
109
111
|
* macOS build new enough to send the header (`events-routes.ts`), so the
|
|
110
112
|
* interface alone cannot say whether one is really there.
|
|
111
113
|
*/
|
|
112
114
|
const NEGOTIATED_CAPABILITIES: ReadonlySet<HostProxyCapability> = new Set([
|
|
113
115
|
"host_cu_window_capture",
|
|
116
|
+
"host_cu_sequence",
|
|
114
117
|
"host_cu_annotate",
|
|
115
118
|
]);
|
|
116
119
|
|
|
@@ -10,10 +10,10 @@ import {
|
|
|
10
10
|
broadcastMessage,
|
|
11
11
|
} from "../runtime/assistant-event-hub.js";
|
|
12
12
|
import {
|
|
13
|
-
ambiguousSameUserError,
|
|
14
13
|
enforceSameActorOrErrorResult,
|
|
15
|
-
|
|
14
|
+
pickMostRecentSameUserClient,
|
|
16
15
|
snapshotHostProxyActorPrincipalId,
|
|
16
|
+
unavailableSameUserClientError,
|
|
17
17
|
} from "../runtime/auth/same-actor.js";
|
|
18
18
|
import * as pendingInteractions from "../runtime/pending-interactions.js";
|
|
19
19
|
import type { ToolExecutionResult } from "../tools/types.js";
|
|
@@ -162,19 +162,17 @@ export class HostTransferProxy {
|
|
|
162
162
|
});
|
|
163
163
|
}
|
|
164
164
|
} else {
|
|
165
|
-
|
|
166
|
-
// (multi-machine) cases so a single targeted-style transfer cannot
|
|
167
|
-
// fan out across the user's machines.
|
|
168
|
-
const resolved = pickSameUserAutoResolve({
|
|
165
|
+
resolvedTargetClientId = pickMostRecentSameUserClient({
|
|
169
166
|
hub: assistantEventHub,
|
|
170
167
|
capability: "host_file",
|
|
171
168
|
sourceActorPrincipalId,
|
|
172
169
|
});
|
|
173
|
-
if (
|
|
174
|
-
|
|
170
|
+
if (
|
|
171
|
+
resolvedTargetClientId === undefined &&
|
|
172
|
+
sourceActorPrincipalId !== undefined
|
|
173
|
+
) {
|
|
174
|
+
return Promise.resolve(unavailableSameUserClientError("host_file"));
|
|
175
175
|
}
|
|
176
|
-
resolvedTargetClientId =
|
|
177
|
-
resolved.kind === "match" ? resolved.clientId : undefined;
|
|
178
176
|
}
|
|
179
177
|
|
|
180
178
|
if (resolvedTargetClientId != null) {
|
|
@@ -364,19 +362,17 @@ export class HostTransferProxy {
|
|
|
364
362
|
});
|
|
365
363
|
}
|
|
366
364
|
} else {
|
|
367
|
-
|
|
368
|
-
// (multi-machine) cases so a single targeted-style transfer cannot
|
|
369
|
-
// fan out across the user's machines.
|
|
370
|
-
const resolved = pickSameUserAutoResolve({
|
|
365
|
+
resolvedTargetClientId = pickMostRecentSameUserClient({
|
|
371
366
|
hub: assistantEventHub,
|
|
372
367
|
capability: "host_file",
|
|
373
368
|
sourceActorPrincipalId,
|
|
374
369
|
});
|
|
375
|
-
if (
|
|
376
|
-
|
|
370
|
+
if (
|
|
371
|
+
resolvedTargetClientId === undefined &&
|
|
372
|
+
sourceActorPrincipalId !== undefined
|
|
373
|
+
) {
|
|
374
|
+
return Promise.resolve(unavailableSameUserClientError("host_file"));
|
|
377
375
|
}
|
|
378
|
-
resolvedTargetClientId =
|
|
379
|
-
resolved.kind === "match" ? resolved.clientId : undefined;
|
|
380
376
|
}
|
|
381
377
|
|
|
382
378
|
if (resolvedTargetClientId != null) {
|
package/src/daemon/lifecycle.ts
CHANGED
|
@@ -71,7 +71,6 @@ import { ensureCompleteCustomProfiles } from "../workspace/custom-profile-ensure
|
|
|
71
71
|
import { ensureDefaultProvider } from "../workspace/default-provider-ensure.js";
|
|
72
72
|
import { WORKSPACE_MIGRATIONS } from "../workspace/migrations/registry.js";
|
|
73
73
|
import { runWorkspaceMigrations } from "../workspace/migrations/runner.js";
|
|
74
|
-
import { startAppSourceWatcher } from "./app-source-watcher.js";
|
|
75
74
|
import { startConfigWatcher } from "./config-watcher.js";
|
|
76
75
|
import { startConversationEvictor } from "./conversation-evictor.js";
|
|
77
76
|
import { writePid } from "./daemon-control.js";
|
|
@@ -674,11 +673,11 @@ export async function runDaemon(): Promise<void> {
|
|
|
674
673
|
// blocked.
|
|
675
674
|
startConsentRefresh();
|
|
676
675
|
|
|
677
|
-
// Bring up the
|
|
676
|
+
// Bring up the assistant's CES connection (process + handshake + reconnect
|
|
678
677
|
// wiring). Blocks up to a 20s timeout so credential reads route through CES
|
|
679
|
-
// before provider init; non-fatal
|
|
680
|
-
// on failure.
|
|
681
|
-
// happens at the process level.
|
|
678
|
+
// before provider init; non-fatal, falls back to the direct credential store
|
|
679
|
+
// on failure. CES serves a multi-connection bootstrap socket, so this
|
|
680
|
+
// happens at the process level and child processes can connect independently.
|
|
682
681
|
await startCes(config);
|
|
683
682
|
|
|
684
683
|
// Bring up the plugin layer: install the runtime bridge, register the
|
|
@@ -706,10 +705,6 @@ export async function runDaemon(): Promise<void> {
|
|
|
706
705
|
// config, and broadcast the relevant resource-changed events to clients.
|
|
707
706
|
startConfigWatcher();
|
|
708
707
|
|
|
709
|
-
// Watch app source directories so edits recompile + refresh surfaces across
|
|
710
|
-
// all conversations.
|
|
711
|
-
startAppSourceWatcher();
|
|
712
|
-
|
|
713
708
|
// Start the CLI IPC server. Throws on EADDRINUSE to abort startup when another
|
|
714
709
|
// daemon already holds the socket, so this process never runs background jobs
|
|
715
710
|
// against the shared database as an unmanageable duplicate.
|