@vellumai/assistant 0.6.2 → 0.6.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +273 -10
- package/Dockerfile +2 -3
- package/bun.lock +41 -49
- package/bunfig.toml +3 -0
- package/docs/architecture/memory.md +1 -1
- package/docs/backup-troubleshooting.md +52 -0
- package/docs/browser-use-architecture-phase2.md +174 -0
- package/docs/stt-provider-onboarding.md +120 -0
- package/knip.json +12 -2
- package/node_modules/@vellumai/ces-contracts/bun.lock +8 -6
- package/node_modules/@vellumai/ces-contracts/package.json +3 -3
- package/node_modules/@vellumai/ces-contracts/src/rpc.ts +42 -0
- package/openapi.yaml +1111 -86
- package/package.json +40 -42
- package/scripts/generate-openapi.ts +0 -2
- package/scripts/test.sh +73 -18
- package/src/__tests__/acp-session.test.ts +43 -0
- package/src/__tests__/agent-image-optimize.test.ts +28 -0
- package/src/__tests__/agent-loop.test.ts +123 -0
- package/src/__tests__/anthropic-provider.test.ts +263 -10
- package/src/__tests__/app-builder-tool-scripts.test.ts +1 -0
- package/src/__tests__/app-executors.test.ts +1 -0
- package/src/__tests__/app-source-watcher.test.ts +37 -11
- package/src/__tests__/approval-routes-http.test.ts +178 -1
- package/src/__tests__/auto-analysis-end-to-end.test.ts +550 -0
- package/src/__tests__/auto-analysis-prompt.test.ts +50 -0
- package/src/__tests__/browser-fill-credential.test.ts +240 -94
- package/src/__tests__/browser-manager.test.ts +40 -27
- package/src/__tests__/browser-skill-baseline-tool-payload.test.ts +2 -2
- package/src/__tests__/browser-skill-endstate.test.ts +31 -7
- package/src/__tests__/btw-routes.test.ts +7 -0
- package/src/__tests__/call-controller.test.ts +581 -20
- package/src/__tests__/catalog-files.test.ts +1000 -0
- package/src/__tests__/channel-approvals.test.ts +53 -0
- package/src/__tests__/channel-invite-transport.test.ts +2 -2
- package/src/__tests__/channel-readiness-routes.test.ts +16 -20
- package/src/__tests__/channel-readiness-service.test.ts +12 -7
- package/src/__tests__/checker.test.ts +157 -10
- package/src/__tests__/clawhub-files.test.ts +347 -0
- package/src/__tests__/commit-message-enrichment-service.test.ts +36 -19
- package/src/__tests__/config-analysis.test.ts +100 -0
- package/src/__tests__/config-managed-gemini-defaults.test.ts +326 -0
- package/src/__tests__/config-schema-cmd.test.ts +2 -2
- package/src/__tests__/config-schema.test.ts +1248 -224
- package/src/__tests__/config-watcher-cleanup-throttle.test.ts +339 -0
- package/src/__tests__/config-watcher.test.ts +43 -8
- package/src/__tests__/confirmation-request-guardian-bridge.test.ts +23 -0
- package/src/__tests__/contact-store-user-file.test.ts +512 -0
- package/src/__tests__/contacts-write.test.ts +197 -0
- package/src/__tests__/context-overflow-approval.test.ts +16 -1
- package/src/__tests__/context-window-manager.test.ts +88 -0
- package/src/__tests__/conversation-abort-tool-results.test.ts +2 -0
- package/src/__tests__/conversation-agent-loop-overflow.test.ts +2 -1
- package/src/__tests__/conversation-agent-loop.test.ts +99 -3
- package/src/__tests__/conversation-analysis-routes.test.ts +2 -2
- package/src/__tests__/conversation-attachments.test.ts +80 -4
- package/src/__tests__/conversation-confirmation-signals.test.ts +290 -0
- package/src/__tests__/conversation-error.test.ts +70 -0
- package/src/__tests__/conversation-fork-crud.test.ts +17 -0
- package/src/__tests__/conversation-history-web-search.test.ts +12 -4
- package/src/__tests__/conversation-host-access-routes.test.ts +229 -0
- package/src/__tests__/conversation-init.benchmark.test.ts +6 -1
- package/src/__tests__/conversation-inject-context.test.ts +103 -0
- package/src/__tests__/conversation-launcher-skill-regression.test.ts +51 -0
- package/src/__tests__/conversation-list-source.test.ts +145 -0
- package/src/__tests__/conversation-pre-run-repair.test.ts +2 -0
- package/src/__tests__/conversation-provider-retry-repair.test.ts +2 -0
- package/src/__tests__/conversation-queue.test.ts +946 -62
- package/src/__tests__/conversation-routes-disk-view.test.ts +275 -0
- package/src/__tests__/conversation-routes-guardian-reply.test.ts +16 -0
- package/src/__tests__/conversation-routes-slash-commands.test.ts +1 -0
- package/src/__tests__/conversation-runtime-assembly.test.ts +324 -46
- package/src/__tests__/conversation-skill-tools.test.ts +7 -4
- package/src/__tests__/conversation-slash-commands.test.ts +33 -0
- package/src/__tests__/conversation-slash-queue.test.ts +89 -18
- package/src/__tests__/conversation-slash-unknown.test.ts +2 -0
- package/src/__tests__/conversation-starter-routes.test.ts +126 -0
- package/src/__tests__/conversation-starters-cadence.test.ts +161 -0
- package/src/__tests__/conversation-store.test.ts +195 -0
- package/src/__tests__/conversation-tool-setup-batch-authorized.test.ts +226 -0
- package/src/__tests__/conversation-workspace-cache-state.test.ts +193 -0
- package/src/__tests__/conversation-workspace-injection.test.ts +2 -0
- package/src/__tests__/conversation-workspace-tool-tracking.test.ts +2 -0
- package/src/__tests__/credential-execution-approval-bridge.test.ts +32 -1
- package/src/__tests__/credential-health-service.test.ts +352 -0
- package/src/__tests__/credential-security-invariants.test.ts +6 -3
- package/src/__tests__/credential-vault-unit.test.ts +383 -7
- package/src/__tests__/credential-vault.test.ts +152 -13
- package/src/__tests__/credentials-cli.test.ts +42 -18
- package/src/__tests__/cross-provider-web-search.test.ts +146 -35
- package/src/__tests__/date-context.test.ts +4 -4
- package/src/__tests__/deterministic-verification-control-plane.test.ts +10 -1
- package/src/__tests__/device-id.test.ts +112 -0
- package/src/__tests__/docker-signing-key-bootstrap.test.ts +167 -4
- package/src/__tests__/dynamic-skill-workflow-prompt.test.ts +1 -3
- package/src/__tests__/email-html-renderer.test.ts +71 -0
- package/src/__tests__/email-invite-adapter.test.ts +36 -32
- package/src/__tests__/embedding-managed-proxy-selection.test.ts +256 -0
- package/src/__tests__/emit-event-signal.test.ts +71 -0
- package/src/__tests__/extension-id-sync-guard.test.ts +222 -0
- package/src/__tests__/fixtures/mock-chrome-extension.ts +386 -0
- package/src/__tests__/gateway-only-enforcement.test.ts +206 -1
- package/src/__tests__/gateway-only-guard.test.ts +2 -0
- package/src/__tests__/gemini-provider.test.ts +66 -2
- package/src/__tests__/get-skill-detail-audit.test.ts +325 -0
- package/src/__tests__/gmail-archive-fallback.test.ts +193 -0
- package/src/__tests__/gmail-archive-gate.test.ts +246 -0
- package/src/__tests__/gmail-preferences.test.ts +117 -0
- package/src/__tests__/guardian-routing-invariants.test.ts +70 -2
- package/src/__tests__/headless-browser-interactions.test.ts +738 -359
- package/src/__tests__/headless-browser-mode.test.ts +614 -0
- package/src/__tests__/headless-browser-navigate.test.ts +528 -49
- package/src/__tests__/headless-browser-read-tools.test.ts +274 -100
- package/src/__tests__/headless-browser-snapshot.test.ts +250 -77
- package/src/__tests__/heartbeat-service.test.ts +70 -17
- package/src/__tests__/home-state-routes.test.ts +162 -0
- package/src/__tests__/host-bash-proxy.test.ts +145 -1
- package/src/__tests__/host-browser-e2e-cloud.test.ts +596 -0
- package/src/__tests__/host-browser-e2e-self-hosted-capability.test.ts +286 -0
- package/src/__tests__/host-browser-e2e-self-hosted.test.ts +374 -0
- package/src/__tests__/host-browser-event-routes.test.ts +350 -0
- package/src/__tests__/host-browser-proxy.test.ts +444 -0
- package/src/__tests__/host-browser-routes.test.ts +198 -0
- package/src/__tests__/host-browser-ws-events-e2e.test.ts +423 -0
- package/src/__tests__/host-cu-proxy.test.ts +166 -1
- package/src/__tests__/host-file-proxy.test.ts +185 -1
- package/src/__tests__/host-file-read-tool.test.ts +52 -0
- package/src/__tests__/host-proxy-interface.test.ts +165 -0
- package/src/__tests__/host-shell-tool.test.ts +1 -11
- package/src/__tests__/http-user-message-parity.test.ts +1 -0
- package/src/__tests__/identity-intro-cache.test.ts +40 -10
- package/src/__tests__/init-feature-flag-overrides.test.ts +38 -112
- package/src/__tests__/integration-status.test.ts +6 -7
- package/src/__tests__/jobs-store-upsert-debounced.test.ts +141 -0
- package/src/__tests__/list-messages-tool-merge.test.ts +37 -12
- package/src/__tests__/llm-context-normalization.test.ts +488 -0
- package/src/__tests__/llm-context-route-provider.test.ts +86 -5
- package/src/__tests__/llm-usage-store.test.ts +363 -0
- package/src/__tests__/mcp-client-auth.test.ts +40 -4
- package/src/__tests__/mcp-health-check.test.ts +10 -3
- package/src/__tests__/media-stream-output.test.ts +555 -0
- package/src/__tests__/media-stream-parser.test.ts +374 -0
- package/src/__tests__/media-stream-server-integration.test.ts +1234 -0
- package/src/__tests__/media-stream-stt-session.test.ts +588 -0
- package/src/__tests__/media-turn-detector.test.ts +440 -0
- package/src/__tests__/message-queue.test.ts +125 -0
- package/src/__tests__/migration-cross-version-compatibility.test.ts +3 -1
- package/src/__tests__/migration-export-http.test.ts +67 -8
- package/src/__tests__/migration-export-streaming.test.ts +66 -0
- package/src/__tests__/migration-import-commit-http.test.ts +109 -7
- package/src/__tests__/migration-import-preflight-http.test.ts +6 -5
- package/src/__tests__/migration-validate-http.test.ts +3 -3
- package/src/__tests__/mock-gateway-ipc.ts +151 -0
- package/src/__tests__/model-intents.test.ts +2 -2
- package/src/__tests__/native-host-marker-sync-guard.test.ts +157 -0
- package/src/__tests__/oauth-apps-routes.test.ts +18 -12
- package/src/__tests__/oauth-cli.test.ts +709 -60
- package/src/__tests__/oauth-connect-orchestrator.test.ts +118 -24
- package/src/__tests__/oauth-provider-seed-logos.test.ts +23 -0
- package/src/__tests__/oauth-provider-serializer.test.ts +147 -10
- package/src/__tests__/oauth-provider-visibility.test.ts +19 -21
- package/src/__tests__/oauth-providers-routes.test.ts +52 -14
- package/src/__tests__/oauth-store.test.ts +1465 -176
- package/src/__tests__/oauth2-gateway-transport.test.ts +460 -26
- package/src/__tests__/onboarding-template-contract.test.ts +81 -70
- package/src/__tests__/openai-provider.test.ts +178 -2
- package/src/__tests__/openai-responses-cutover-guard.test.ts +184 -0
- package/src/__tests__/openai-responses-provider.test.ts +1105 -0
- package/src/__tests__/openrouter-token-estimation.test.ts +100 -0
- package/src/__tests__/outlook-categories.test.ts +1 -1
- package/src/__tests__/outlook-client-automation.test.ts +1 -1
- package/src/__tests__/outlook-compose-tools.test.ts +1 -1
- package/src/__tests__/outlook-email-watcher.test.ts +1 -1
- package/src/__tests__/outlook-follow-up.test.ts +1 -1
- package/src/__tests__/outlook-messaging-provider.test.ts +2 -2
- package/src/__tests__/outlook-trash.test.ts +1 -1
- package/src/__tests__/outlook-unsubscribe.test.ts +32 -3
- package/src/__tests__/permission-checker-host-gate.test.ts +74 -14
- package/src/__tests__/permission-mode.test.ts +28 -56
- package/src/__tests__/persona-resolver.test.ts +251 -0
- package/src/__tests__/platform-bash-auto-approve.test.ts +4 -0
- package/src/__tests__/platform-callback-registration.test.ts +19 -0
- package/src/__tests__/platform.test.ts +92 -1
- package/src/__tests__/post-turn-tool-result-truncation.test.ts +343 -0
- package/src/__tests__/prechat-onboarding-contract.test.ts +267 -0
- package/src/__tests__/pricing.test.ts +174 -0
- package/src/__tests__/proxy-approval-callback.test.ts +18 -0
- package/src/__tests__/qdrant-manager.test.ts +29 -8
- package/src/__tests__/regenerate-fire-and-forget-trace.test.ts +194 -0
- package/src/__tests__/relationship-state-contract.test.ts +175 -0
- package/src/__tests__/relay-server.test.ts +423 -5
- package/src/__tests__/require-fresh-approval.test.ts +40 -1
- package/src/__tests__/sanitize-config-for-transfer.test.ts +132 -0
- package/src/__tests__/schedule-routes.test.ts +162 -0
- package/src/__tests__/search-skills-unified.test.ts +118 -0
- package/src/__tests__/secret-detection-handler.test.ts +84 -0
- package/src/__tests__/secret-ingress-http.test.ts +1 -0
- package/src/__tests__/secret-scanner-executor.test.ts +4 -0
- package/src/__tests__/secure-keys.test.ts +107 -0
- package/src/__tests__/send-endpoint-busy.test.ts +8 -1
- package/src/__tests__/sequence-store.test.ts +1 -1
- package/src/__tests__/server-history-render.test.ts +49 -0
- package/src/__tests__/set-permission-mode.test.ts +13 -250
- package/src/__tests__/settings-routes.test.ts +201 -0
- package/src/__tests__/skill-load-feature-flag.test.ts +1 -0
- package/src/__tests__/skills-file-content-endpoint.test.ts +801 -0
- package/src/__tests__/skills-files-catalog-fallback.test.ts +738 -0
- package/src/__tests__/skills.test.ts +5 -2
- package/src/__tests__/skillssh-files.test.ts +446 -0
- package/src/__tests__/slack-block-formatting.test.ts +110 -0
- package/src/__tests__/slack-channel-config.test.ts +576 -16
- package/src/__tests__/stt-catalog-parity.test.ts +282 -0
- package/src/__tests__/stt-stream-session.test.ts +535 -0
- package/src/__tests__/subagent-detail.test.ts +44 -2
- package/src/__tests__/subagent-disposal.test.ts +1 -0
- package/src/__tests__/subagent-fork-notifications.test.ts +291 -0
- package/src/__tests__/subagent-fork-spawn.test.ts +384 -0
- package/src/__tests__/subagent-manager-notify.test.ts +1 -0
- package/src/__tests__/subagent-notify-parent.test.ts +1 -0
- package/src/__tests__/subagent-spawn-tool-fork.test.ts +411 -0
- package/src/__tests__/subagent-tools.test.ts +1 -0
- package/src/__tests__/subagent-types.test.ts +1 -0
- package/src/__tests__/system-prompt-ask-mode.test.ts +27 -71
- package/src/__tests__/system-prompt.test.ts +184 -27
- package/src/__tests__/task-scheduler.test.ts +32 -6
- package/src/__tests__/telegram-config.test.ts +10 -13
- package/src/__tests__/telephony-stt-routing.test.ts +329 -0
- package/src/__tests__/terminal-tools.test.ts +25 -5
- package/src/__tests__/test-preload.ts +18 -0
- package/src/__tests__/test-support/browser-skill-harness.ts +4 -1
- package/src/__tests__/tool-approval-handler.test.ts +73 -0
- package/src/__tests__/tool-executor-lifecycle-events.test.ts +9 -5
- package/src/__tests__/tool-executor-shell-integration.test.ts +4 -0
- package/src/__tests__/tool-executor.test.ts +33 -24
- package/src/__tests__/tool-result-truncation.test.ts +36 -0
- package/src/__tests__/tool-side-effects-slack-dm.test.ts +22 -0
- package/src/__tests__/top-level-renderer.test.ts +73 -1
- package/src/__tests__/transport-hints-queue.test.ts +14 -29
- package/src/__tests__/trust-store.test.ts +7 -1
- package/src/__tests__/trusted-contact-approval-notifier.test.ts +1 -1
- package/src/__tests__/trusted-contact-inline-approval-integration.test.ts +109 -0
- package/src/__tests__/tts-catalog-parity.test.ts +345 -0
- package/src/__tests__/twilio-routes-twiml.test.ts +512 -114
- package/src/__tests__/twilio-routes.test.ts +376 -0
- package/src/__tests__/unicode.test.ts +293 -0
- package/src/__tests__/update-bulletin-format.test.ts +59 -0
- package/src/__tests__/update-bulletin.test.ts +206 -5
- package/src/__tests__/usage-routes.test.ts +25 -4
- package/src/__tests__/user-reference.test.ts +46 -61
- package/src/__tests__/v2-consent-policy.test.ts +103 -0
- package/src/__tests__/verification-control-plane-policy.test.ts +4 -0
- package/src/__tests__/voice-config-update.test.ts +403 -0
- package/src/__tests__/voice-quality.test.ts +434 -19
- package/src/__tests__/workspace-heartbeat-service.test.ts +7 -0
- package/src/__tests__/workspace-migration-033-stt-service-explicit-config.test.ts +547 -0
- package/src/__tests__/workspace-migration-034-remove-calls-voice-transcription-provider.test.ts +596 -0
- package/src/__tests__/workspace-migration-drop-user-md.test.ts +368 -0
- package/src/__tests__/workspace-migration-meets.test.ts +244 -0
- package/src/__tests__/workspace-migration-seed-device-id.test.ts +14 -20
- package/src/__tests__/workspace-policy.test.ts +2 -0
- package/src/acp/client-handler.ts +30 -4
- package/src/agent/image-optimize.ts +24 -12
- package/src/agent/loop.ts +55 -9
- package/src/approvals/guardian-request-resolvers.ts +21 -15
- package/src/backup/__tests__/backup-key.test.ts +152 -0
- package/src/backup/__tests__/backup-worker.test.ts +767 -0
- package/src/backup/__tests__/list-snapshots.test.ts +87 -0
- package/src/backup/__tests__/local-writer.test.ts +218 -0
- package/src/backup/__tests__/offsite-writer.test.ts +641 -0
- package/src/backup/__tests__/paths.test.ts +300 -0
- package/src/backup/__tests__/restore.test.ts +498 -0
- package/src/backup/__tests__/snapshot-lock.test.ts +352 -0
- package/src/backup/__tests__/stream-crypt.test.ts +228 -0
- package/src/backup/backup-key.ts +137 -0
- package/src/backup/backup-worker.ts +459 -0
- package/src/backup/list-snapshots.ts +147 -0
- package/src/backup/local-writer.ts +133 -0
- package/src/backup/offsite-writer.ts +222 -0
- package/src/backup/paths.ts +226 -0
- package/src/backup/restore.ts +322 -0
- package/src/backup/snapshot-lock.ts +431 -0
- package/src/backup/stream-crypt.ts +263 -0
- package/src/browser-session/__tests__/manager.test.ts +297 -0
- package/src/browser-session/backends/cdp-inspect.ts +30 -0
- package/src/browser-session/backends/extension.ts +26 -0
- package/src/browser-session/backends/local.ts +24 -0
- package/src/browser-session/events.ts +164 -0
- package/src/browser-session/index.ts +27 -0
- package/src/browser-session/manager.ts +159 -0
- package/src/browser-session/types.ts +28 -0
- package/src/bundler/package-resolver.ts +4 -0
- package/src/calls/audio-store.ts +11 -5
- package/src/calls/call-controller.ts +226 -71
- package/src/calls/call-domain.ts +9 -0
- package/src/calls/call-speech-output.ts +190 -0
- package/src/calls/call-transport.ts +77 -0
- package/src/calls/media-stream-audio-transcode.ts +173 -0
- package/src/calls/media-stream-output.ts +660 -0
- package/src/calls/media-stream-parser.ts +300 -0
- package/src/calls/media-stream-protocol.ts +166 -0
- package/src/calls/media-stream-server.ts +592 -0
- package/src/calls/media-stream-stt-session.ts +460 -0
- package/src/calls/media-turn-detector.ts +230 -0
- package/src/calls/relay-server.ts +90 -75
- package/src/calls/resolve-call-tts-provider.ts +136 -0
- package/src/calls/telephony-stt-routing.ts +145 -0
- package/src/calls/tts-call-strategy.ts +161 -0
- package/src/calls/tts-text-sanitizer.ts +32 -16
- package/src/calls/twilio-routes.ts +281 -17
- package/src/calls/voice-quality.ts +78 -35
- package/src/calls/voice-session-bridge.ts +8 -1
- package/src/channels/__tests__/types.test.ts +134 -0
- package/src/channels/types.ts +69 -3
- package/src/cli/__tests__/run-assistant-command.ts +11 -1
- package/src/cli/commands/__tests__/backup.test.ts +1165 -0
- package/src/cli/commands/__tests__/domain-register.test.ts +234 -0
- package/src/cli/commands/__tests__/domain-status.test.ts +132 -0
- package/src/cli/commands/__tests__/email-attachment.test.ts +422 -0
- package/src/cli/commands/__tests__/email-download.test.ts +16 -1
- package/src/cli/commands/__tests__/email-list.test.ts +22 -4
- package/src/cli/commands/__tests__/email-register.test.ts +4 -4
- package/src/cli/commands/__tests__/email-send.test.ts +37 -4
- package/src/cli/commands/__tests__/email-status.test.ts +5 -1
- package/src/cli/commands/__tests__/email-unregister.test.ts +34 -5
- package/src/cli/commands/backup.ts +993 -0
- package/src/cli/commands/conversations.ts +77 -0
- package/src/cli/commands/credentials.ts +3 -4
- package/src/cli/commands/domain.ts +210 -0
- package/src/cli/commands/email.ts +273 -16
- package/src/cli/commands/mcp.ts +16 -4
- package/src/cli/commands/oauth/__tests__/connect.test.ts +56 -44
- package/src/cli/commands/oauth/__tests__/disconnect.test.ts +21 -21
- package/src/cli/commands/oauth/__tests__/mode.test.ts +17 -17
- package/src/cli/commands/oauth/__tests__/ping.test.ts +16 -16
- package/src/cli/commands/oauth/__tests__/providers-delete.test.ts +32 -33
- package/src/cli/commands/oauth/__tests__/providers-register.test.ts +330 -0
- package/src/cli/commands/oauth/__tests__/providers-update.test.ts +117 -12
- package/src/cli/commands/oauth/__tests__/status.test.ts +10 -10
- package/src/cli/commands/oauth/__tests__/token.test.ts +7 -7
- package/src/cli/commands/oauth/apps.ts +7 -4
- package/src/cli/commands/oauth/connect.ts +6 -3
- package/src/cli/commands/oauth/disconnect.ts +1 -1
- package/src/cli/commands/oauth/mode.ts +12 -3
- package/src/cli/commands/oauth/providers.ts +215 -36
- package/src/cli/commands/oauth/shared.ts +7 -6
- package/src/cli/commands/platform/__tests__/callback-routes-list.test.ts +254 -0
- package/src/cli/commands/platform/__tests__/connect.test.ts +6 -0
- package/src/cli/commands/platform/__tests__/disconnect.test.ts +7 -1
- package/src/cli/commands/platform/__tests__/status.test.ts +6 -0
- package/src/cli/commands/platform/index.ts +107 -10
- package/src/cli/commands/usage.ts +10 -9
- package/src/cli/lib/daemon-credential-client.ts +4 -0
- package/src/cli/program.ts +30 -4
- package/src/config/__tests__/backup-schema.test.ts +134 -0
- package/src/config/assistant-feature-flags.ts +61 -62
- package/src/config/bundled-skills/app-builder/SKILL.md +26 -249
- package/src/config/bundled-skills/app-builder/references/CUSTOM_ROUTES.md +141 -0
- package/src/config/bundled-skills/app-builder/references/INTERACTION_HOOKS.md +56 -0
- package/src/config/bundled-skills/app-builder/references/WIDGETS.md +125 -0
- package/src/config/bundled-skills/browser/SKILL.md +30 -5
- package/src/config/bundled-skills/browser/TOOLS.json +123 -0
- package/src/config/bundled-skills/browser/tools/browser-attach.ts +12 -0
- package/src/config/bundled-skills/browser/tools/browser-detach.ts +12 -0
- package/src/config/bundled-skills/browser/tools/browser-status.ts +12 -0
- package/src/config/bundled-skills/browser/tools/browser-wait-for-download.ts +17 -0
- package/src/config/bundled-skills/contacts/SKILL.md +5 -2
- package/src/config/bundled-skills/document/SKILL.md +4 -0
- package/src/config/bundled-skills/gmail/SKILL.md +54 -8
- package/src/config/bundled-skills/gmail/TOOLS.json +33 -3
- package/src/config/bundled-skills/gmail/tools/gmail-archive.ts +116 -9
- package/src/config/bundled-skills/gmail/tools/gmail-outreach-scan.ts +138 -11
- package/src/config/bundled-skills/gmail/tools/gmail-preferences-tool.ts +59 -0
- package/src/config/bundled-skills/gmail/tools/gmail-preferences.ts +82 -0
- package/src/config/bundled-skills/gmail/tools/gmail-sender-digest.ts +113 -17
- package/src/config/bundled-skills/gmail/tools/gmail-unsubscribe.ts +2 -2
- package/src/config/bundled-skills/media-processing/SKILL.md +3 -9
- package/src/config/bundled-skills/media-processing/TOOLS.json +1 -6
- package/src/config/bundled-skills/media-processing/__tests__/audio-transcribe.test.ts +125 -0
- package/src/config/bundled-skills/media-processing/__tests__/extract-keyframes.test.ts +181 -0
- package/src/config/bundled-skills/media-processing/__tests__/preprocess-audio.test.ts +141 -0
- package/src/config/bundled-skills/media-processing/services/audio-transcribe.ts +32 -87
- package/src/config/bundled-skills/media-processing/services/preprocess.ts +8 -4
- package/src/config/bundled-skills/media-processing/tools/extract-keyframes.ts +0 -10
- package/src/config/bundled-skills/messaging/SKILL.md +3 -3
- package/src/config/bundled-skills/messaging/tools/messaging-archive-by-sender.ts +2 -2
- package/src/config/bundled-skills/outlook/SKILL.md +9 -2
- package/src/config/bundled-skills/outlook/tools/outlook-unsubscribe.ts +2 -2
- package/src/config/bundled-skills/phone-calls/SKILL.md +2 -2
- package/src/config/bundled-skills/phone-calls/references/CONFIG.md +27 -18
- package/src/config/bundled-skills/phone-calls/references/TROUBLESHOOTING.md +3 -3
- package/src/config/bundled-skills/settings/TOOLS.json +3 -3
- package/src/config/bundled-skills/settings/tools/voice-config-update.ts +26 -22
- package/src/config/bundled-skills/slack/SKILL.md +1 -0
- package/src/config/bundled-skills/subagent/SKILL.md +21 -0
- package/src/config/bundled-skills/subagent/TOOLS.json +8 -4
- package/src/config/bundled-skills/tasks/SKILL.md +5 -0
- package/src/config/bundled-skills/transcribe/SKILL.md +9 -14
- package/src/config/bundled-skills/transcribe/TOOLS.json +2 -7
- package/src/config/bundled-skills/transcribe/tools/transcribe-media.test.ts +256 -0
- package/src/config/bundled-skills/transcribe/tools/transcribe-media.ts +38 -188
- package/src/config/bundled-tool-registry.ts +8 -0
- package/src/config/env-registry.ts +38 -0
- package/src/config/env.ts +49 -4
- package/src/config/feature-flag-registry.json +85 -14
- package/src/config/loader.ts +82 -13
- package/src/config/sanitize-for-transfer.ts +47 -0
- package/src/config/schema.ts +81 -15
- package/src/config/schemas/__tests__/stt.test.ts +43 -0
- package/src/config/schemas/analysis.ts +51 -0
- package/src/config/schemas/backup.ts +72 -0
- package/src/config/schemas/calls.ts +1 -26
- package/src/config/schemas/elevenlabs.ts +0 -59
- package/src/config/schemas/filing.ts +47 -7
- package/src/config/schemas/heartbeat.ts +27 -5
- package/src/config/schemas/host-browser.ts +112 -0
- package/src/config/schemas/inference.ts +1 -1
- package/src/config/schemas/memory-lifecycle.ts +14 -2
- package/src/config/schemas/memory-retrieval.ts +103 -0
- package/src/config/schemas/security.ts +0 -6
- package/src/config/schemas/services.ts +52 -0
- package/src/config/schemas/stt.ts +59 -0
- package/src/config/schemas/tts.ts +230 -0
- package/src/config/schemas/updates.ts +14 -0
- package/src/config/skills.ts +4 -0
- package/src/config/types.ts +4 -1
- package/src/contacts/contact-store.ts +56 -11
- package/src/contacts/contacts-write.ts +38 -1
- package/src/context/post-turn-tool-result-truncation.ts +177 -0
- package/src/context/tool-result-truncation.ts +2 -1
- package/src/context/window-manager.ts +61 -10
- package/src/credential-execution/approval-bridge.ts +49 -15
- package/src/credential-execution/executable-discovery.ts +12 -2
- package/src/credential-execution/process-manager.ts +33 -2
- package/src/credential-health/credential-health-service.ts +366 -0
- package/src/daemon/__tests__/conversation-lifecycle-auto-analyze.test.ts +324 -0
- package/src/daemon/__tests__/conversation-surfaces-launch.test.ts +497 -0
- package/src/daemon/__tests__/conversation-tool-setup.test.ts +195 -0
- package/src/daemon/__tests__/lifecycle-startup-ordering.test.ts +127 -0
- package/src/daemon/app-source-watcher.ts +35 -0
- package/src/daemon/config-watcher.ts +99 -5
- package/src/daemon/context-overflow-approval.ts +5 -0
- package/src/daemon/conversation-agent-loop-handlers.ts +23 -2
- package/src/daemon/conversation-agent-loop.ts +153 -42
- package/src/daemon/conversation-attachments.ts +40 -0
- package/src/daemon/conversation-error.ts +11 -0
- package/src/daemon/conversation-history.ts +40 -6
- package/src/daemon/conversation-launch.ts +220 -0
- package/src/daemon/conversation-lifecycle.ts +59 -9
- package/src/daemon/conversation-messaging.ts +37 -3
- package/src/daemon/conversation-notifiers.ts +5 -0
- package/src/daemon/conversation-process.ts +622 -13
- package/src/daemon/conversation-queue-manager.ts +24 -0
- package/src/daemon/conversation-runtime-assembly.ts +128 -36
- package/src/daemon/conversation-slash.ts +36 -0
- package/src/daemon/conversation-surfaces.ts +131 -40
- package/src/daemon/conversation-tool-setup.ts +99 -8
- package/src/daemon/conversation-usage.ts +7 -4
- package/src/daemon/conversation-workspace.ts +12 -0
- package/src/daemon/conversation.ts +292 -16
- package/src/daemon/date-context.ts +10 -10
- package/src/daemon/first-greeting.ts +3 -2
- package/src/daemon/handlers/config-slack-channel.ts +269 -94
- package/src/daemon/handlers/conversations.ts +13 -141
- package/src/daemon/handlers/shared.ts +80 -0
- package/src/daemon/handlers/skills.ts +483 -44
- package/src/daemon/host-bash-proxy.ts +48 -13
- package/src/daemon/host-browser-proxy.ts +192 -0
- package/src/daemon/host-cu-proxy.ts +36 -11
- package/src/daemon/host-file-proxy.ts +57 -9
- package/src/daemon/lifecycle.ts +179 -28
- package/src/daemon/message-protocol.ts +13 -0
- package/src/daemon/message-types/conversations.ts +89 -14
- package/src/daemon/message-types/home.ts +40 -0
- package/src/daemon/message-types/host-browser.ts +100 -0
- package/src/daemon/message-types/meet.ts +143 -0
- package/src/daemon/message-types/messages.ts +19 -5
- package/src/daemon/message-types/schedules.ts +34 -2
- package/src/daemon/message-types/skills.ts +26 -0
- package/src/daemon/message-types/subagents.ts +2 -0
- package/src/daemon/message-types/surfaces.ts +2 -0
- package/src/daemon/server.ts +439 -14
- package/src/daemon/shutdown-handlers.ts +32 -4
- package/src/daemon/shutdown-registry.ts +40 -0
- package/src/daemon/tool-side-effects.ts +15 -0
- package/src/daemon/transport-hints.ts +5 -24
- package/src/email/html-renderer.ts +76 -0
- package/src/heartbeat/heartbeat-service.ts +93 -7
- package/src/home/__tests__/assistant-feed-authoring.test.ts +156 -0
- package/src/home/__tests__/emit-feed-event.test.ts +169 -0
- package/src/home/__tests__/feed-scheduler.test.ts +194 -0
- package/src/home/__tests__/feed-types.test.ts +275 -0
- package/src/home/__tests__/feed-writer.test.ts +688 -0
- package/src/home/__tests__/phase5-exit-criteria.test.ts +212 -0
- package/src/home/__tests__/platform-gmail-digest.test.ts +222 -0
- package/src/home/__tests__/progress-formula.test.ts +213 -0
- package/src/home/__tests__/relationship-state-writer.test.ts +740 -0
- package/src/home/__tests__/rollup-producer.test.ts +398 -0
- package/src/home/assistant-feed-authoring.ts +124 -0
- package/src/home/emit-feed-event.ts +158 -0
- package/src/home/feed-scheduler.ts +247 -0
- package/src/home/feed-types.ts +181 -0
- package/src/home/feed-writer.ts +469 -0
- package/src/home/platform-gmail-digest.ts +163 -0
- package/src/home/progress-formula.ts +86 -0
- package/src/home/relationship-state-writer.ts +824 -0
- package/src/home/relationship-state.ts +143 -0
- package/src/home/rollup-producer.ts +384 -0
- package/src/hooks/runner.ts +7 -0
- package/src/inbound/platform-callback-registration.ts +30 -20
- package/src/inbound/public-ingress-urls.ts +12 -0
- package/src/instrument.ts +1 -1
- package/src/ipc/__tests__/cli-ipc.test.ts +200 -0
- package/src/ipc/cli-client.ts +151 -0
- package/src/ipc/cli-server.ts +234 -0
- package/src/ipc/gateway-client.ts +180 -0
- package/src/ipc/routes/index.ts +5 -0
- package/src/ipc/routes/wake-conversation.ts +19 -0
- package/src/mcp/client.ts +59 -24
- package/src/memory/__tests__/auto-analysis-enqueue.test.ts +356 -0
- package/src/memory/__tests__/auto-analysis-guard.test.ts +57 -0
- package/src/memory/__tests__/conversation-analyze-job.test.ts +232 -0
- package/src/memory/__tests__/find-analysis-conversation.test.ts +196 -0
- package/src/memory/app-store.ts +31 -1
- package/src/memory/attachments-store.ts +70 -0
- package/src/memory/auto-analysis-enqueue.ts +127 -0
- package/src/memory/auto-analysis-guard.ts +27 -0
- package/src/memory/cleanup-schedule-state.ts +37 -0
- package/src/memory/conversation-analyze-job.ts +73 -0
- package/src/memory/conversation-crud.ts +122 -0
- package/src/memory/conversation-disk-view.ts +7 -0
- package/src/memory/conversation-group-migration.ts +34 -2
- package/src/memory/conversation-queries.ts +6 -5
- package/src/memory/conversation-starters-cadence.ts +76 -0
- package/src/memory/conversation-title-service.ts +5 -2
- package/src/memory/db-init.ts +18 -0
- package/src/memory/db-maintenance.ts +108 -0
- package/src/memory/db.ts +1 -0
- package/src/memory/embedding-backend.test.ts +75 -0
- package/src/memory/embedding-backend.ts +131 -5
- package/src/memory/embedding-gemini.test.ts +54 -0
- package/src/memory/embedding-gemini.ts +20 -9
- package/src/memory/embedding-local.ts +176 -17
- package/src/memory/graph/consolidation.ts +10 -23
- package/src/memory/graph/conversation-graph-memory.ts +15 -0
- package/src/memory/graph/extraction-job.ts +15 -0
- package/src/memory/graph/extraction.test.ts +23 -0
- package/src/memory/graph/extraction.ts +8 -0
- package/src/memory/graph/retriever.ts +67 -40
- package/src/memory/graph/scoring.test.ts +186 -0
- package/src/memory/graph/scoring.ts +31 -1
- package/src/memory/graph/store.test.ts +7 -3
- package/src/memory/graph/store.ts +47 -12
- package/src/memory/graph/tools.ts +1 -1
- package/src/memory/group-crud.ts +6 -1
- package/src/memory/indexer.ts +95 -16
- package/src/memory/job-handlers/cleanup.ts +11 -8
- package/src/memory/job-handlers/conversation-starters.ts +16 -10
- package/src/memory/jobs-store.ts +64 -4
- package/src/memory/jobs-worker.ts +22 -9
- package/src/memory/llm-usage-store.ts +137 -60
- package/src/memory/migrations/213-oauth-providers-scope-separator.ts +13 -0
- package/src/memory/migrations/214-oauth-providers-refresh-url.ts +11 -0
- package/src/memory/migrations/215-oauth-providers-revoke.ts +14 -0
- package/src/memory/migrations/216-oauth-providers-token-auth-method.ts +30 -0
- package/src/memory/migrations/217-conversation-host-access.ts +40 -0
- package/src/memory/migrations/218-oauth-providers-logo-url.ts +11 -0
- package/src/memory/migrations/219-oauth-providers-token-exchange-body-format.ts +15 -0
- package/src/memory/migrations/220-normalize-user-file-by-principal.ts +190 -0
- package/src/memory/migrations/221-conversations-archived-at.ts +16 -0
- package/src/memory/migrations/index.ts +12 -0
- package/src/memory/migrations/registry.ts +16 -0
- package/src/memory/qdrant-manager.ts +43 -16
- package/src/memory/schema/conversations.ts +3 -0
- package/src/memory/schema/oauth.ts +21 -13
- package/src/memory/usage-buckets.ts +396 -0
- package/src/messaging/providers/gmail/client.ts +57 -6
- package/src/messaging/providers/slack/__tests__/adapter-token-routing.test.ts +282 -0
- package/src/messaging/providers/slack/adapter.ts +143 -38
- package/src/messaging/providers/slack/client.ts +16 -0
- package/src/messaging/providers/slack/types.ts +4 -0
- package/src/notifications/decision-engine.ts +3 -3
- package/src/notifications/signal.ts +5 -0
- package/src/oauth/AGENTS.md +76 -0
- package/src/oauth/__tests__/identity-verifier.test.ts +25 -19
- package/src/oauth/__tests__/seed-providers-managed.test.ts +32 -0
- package/src/oauth/byo-connection.test.ts +26 -9
- package/src/oauth/byo-connection.ts +10 -8
- package/src/oauth/connect-orchestrator.ts +25 -21
- package/src/oauth/connect-types.ts +3 -3
- package/src/oauth/connection-resolver.test.ts +17 -4
- package/src/oauth/connection-resolver.ts +22 -18
- package/src/oauth/connection.ts +3 -1
- package/src/oauth/manual-token-connection.ts +13 -13
- package/src/oauth/oauth-store.ts +223 -100
- package/src/oauth/platform-connection.test.ts +101 -3
- package/src/oauth/platform-connection.ts +56 -35
- package/src/oauth/provider-serializer.ts +31 -5
- package/src/oauth/revoke.ts +76 -0
- package/src/oauth/seed-providers.ts +133 -87
- package/src/oauth/token-persistence.ts +1 -1
- package/src/permissions/checker.ts +16 -6
- package/src/permissions/defaults.ts +49 -1
- package/src/permissions/permission-mode.ts +4 -11
- package/src/permissions/prompter.ts +13 -1
- package/src/permissions/trust-store.ts +3 -3
- package/src/permissions/v2-consent-policy.ts +87 -0
- package/src/permissions/workspace-policy.ts +3 -0
- package/src/platform/client.test.ts +10 -0
- package/src/platform/sync-identity.ts +129 -0
- package/src/prompts/persona-resolver.ts +126 -2
- package/src/prompts/system-prompt.ts +76 -38
- package/src/prompts/templates/BOOTSTRAP-REFERENCE.md +3 -65
- package/src/prompts/templates/BOOTSTRAP.md +59 -105
- package/src/prompts/templates/SOUL.md +3 -1
- package/src/prompts/templates/UPDATES.md +12 -0
- package/src/prompts/templates/channels/slack.md +20 -0
- package/src/prompts/update-bulletin-format.ts +26 -9
- package/src/prompts/update-bulletin.ts +34 -23
- package/src/prompts/user-reference.ts +20 -17
- package/src/providers/__tests__/provider-secret-catalog.test.ts +42 -0
- package/src/providers/anthropic/client.ts +157 -60
- package/src/providers/fireworks/client.ts +2 -2
- package/src/providers/gemini/client.ts +9 -1
- package/src/providers/model-catalog.ts +6 -0
- package/src/providers/model-intents.ts +4 -4
- package/src/providers/ollama/client.ts +2 -2
- package/src/providers/openai/chat-completions-provider.ts +474 -0
- package/src/providers/openai/client.ts +25 -440
- package/src/providers/openai/responses-provider.ts +502 -0
- package/src/providers/openrouter/client.ts +101 -4
- package/src/providers/provider-secret-catalog.ts +139 -0
- package/src/providers/registry.ts +2 -2
- package/src/providers/retry.ts +14 -3
- package/src/providers/speech-to-text/__tests__/provider-catalog.test.ts +251 -0
- package/src/providers/speech-to-text/__tests__/resolve.test.ts +828 -0
- package/src/providers/speech-to-text/deepgram-realtime.test.ts +980 -0
- package/src/providers/speech-to-text/deepgram-realtime.ts +767 -0
- package/src/providers/speech-to-text/deepgram.test.ts +332 -0
- package/src/providers/speech-to-text/deepgram.ts +115 -0
- package/src/providers/speech-to-text/google-gemini-live-stream.test.ts +743 -0
- package/src/providers/speech-to-text/google-gemini-live-stream.ts +625 -0
- package/src/providers/speech-to-text/google-gemini.test.ts +226 -0
- package/src/providers/speech-to-text/google-gemini.ts +101 -0
- package/src/providers/speech-to-text/openai-whisper-stream.test.ts +564 -0
- package/src/providers/speech-to-text/openai-whisper-stream.ts +381 -0
- package/src/providers/speech-to-text/openai-whisper.test.ts +1 -37
- package/src/providers/speech-to-text/openai-whisper.ts +63 -33
- package/src/providers/speech-to-text/provider-catalog.ts +306 -0
- package/src/providers/speech-to-text/resolve.ts +386 -6
- package/src/providers/types.ts +10 -1
- package/src/runtime/AGENTS.md +65 -0
- package/src/runtime/__tests__/agent-wake.test.ts +831 -0
- package/src/runtime/__tests__/browser-extension-pair-routes.test.ts +715 -0
- package/src/runtime/__tests__/capability-tokens.test.ts +258 -0
- package/src/runtime/__tests__/chrome-extension-registry.test.ts +518 -0
- package/src/runtime/__tests__/runtime-mode.test.ts +62 -0
- package/src/runtime/__tests__/slack-block-formatting.test.ts +481 -0
- package/src/runtime/agent-wake.ts +512 -0
- package/src/runtime/assistant-event-hub.ts +2 -2
- package/src/runtime/auth/__tests__/guard-tests.test.ts +1 -0
- package/src/runtime/auth/__tests__/middleware.test.ts +116 -1
- package/src/runtime/auth/__tests__/route-policy.test.ts +48 -0
- package/src/runtime/auth/middleware.ts +98 -0
- package/src/runtime/auth/route-policy.ts +33 -9
- package/src/runtime/auth/token-service.ts +56 -1
- package/src/runtime/btw-sidechain.ts +2 -0
- package/src/runtime/capability-tokens.ts +414 -0
- package/src/runtime/channel-approvals.ts +18 -5
- package/src/runtime/channel-invite-transport.ts +1 -1
- package/src/runtime/channel-invite-transports/email.ts +14 -6
- package/src/runtime/channel-readiness-service.ts +12 -22
- package/src/runtime/chrome-extension-registry.ts +368 -0
- package/src/runtime/confirmation-request-guardian-bridge.ts +6 -0
- package/src/runtime/guardian-decision-types.ts +7 -0
- package/src/runtime/http-server.ts +815 -75
- package/src/runtime/http-types.ts +6 -2
- package/src/runtime/migrations/__tests__/rebind-secrets-credentials.test.ts +172 -0
- package/src/runtime/migrations/__tests__/vbundle-builder-credentials.test.ts +276 -0
- package/src/runtime/migrations/__tests__/vbundle-import-credentials.test.ts +198 -0
- package/src/runtime/migrations/__tests__/vbundle-legacy-user-md.test.ts +360 -0
- package/src/runtime/migrations/migration-transport.ts +7 -0
- package/src/runtime/migrations/migration-wizard.ts +23 -2
- package/src/runtime/migrations/rebind-secrets-screen.ts +76 -15
- package/src/runtime/migrations/vbundle-builder.ts +145 -38
- package/src/runtime/migrations/vbundle-import-analyzer.ts +96 -1
- package/src/runtime/migrations/vbundle-importer.ts +89 -5
- package/src/runtime/pending-interactions.ts +18 -13
- package/src/runtime/routes/__tests__/backup-routes.test.ts +967 -0
- package/src/runtime/routes/__tests__/home-feed-routes.test.ts +507 -0
- package/src/runtime/routes/__tests__/migration-import-credential-filter.test.ts +208 -0
- package/src/runtime/routes/__tests__/stt-routes.test.ts +406 -0
- package/src/runtime/routes/__tests__/tts-routes.test.ts +474 -0
- package/src/runtime/routes/__tests__/user-route-dispatcher.test.ts +148 -17
- package/src/runtime/routes/app-management-routes.ts +12 -18
- package/src/runtime/routes/approval-routes.ts +90 -16
- package/src/runtime/routes/attachment-routes.test.ts +9 -3
- package/src/runtime/routes/attachment-routes.ts +216 -17
- package/src/runtime/routes/backup-routes.ts +519 -0
- package/src/runtime/routes/browser-extension-pair-routes.ts +556 -0
- package/src/runtime/routes/btw-routes.ts +8 -6
- package/src/runtime/routes/contact-routes.test.ts +298 -0
- package/src/runtime/routes/contact-routes.ts +132 -5
- package/src/runtime/routes/conversation-analysis-routes.ts +22 -141
- package/src/runtime/routes/conversation-management-routes.ts +223 -0
- package/src/runtime/routes/conversation-routes.ts +598 -103
- package/src/runtime/routes/conversation-starter-routes.ts +78 -16
- package/src/runtime/routes/filing-routes.ts +93 -0
- package/src/runtime/routes/guardian-action-routes.ts +24 -13
- package/src/runtime/routes/home-feed-routes.ts +334 -0
- package/src/runtime/routes/home-state-routes.ts +138 -0
- package/src/runtime/routes/host-browser-routes.ts +268 -0
- package/src/runtime/routes/host-file-routes.ts +9 -1
- package/src/runtime/routes/identity-intro-cache.ts +7 -3
- package/src/runtime/routes/identity-routes.ts +262 -33
- package/src/runtime/routes/inbound-stages/transcribe-audio.test.ts +46 -39
- package/src/runtime/routes/inbound-stages/transcribe-audio.ts +15 -15
- package/src/runtime/routes/integrations/slack/__tests__/channel.test.ts +137 -0
- package/src/runtime/routes/integrations/slack/__tests__/share.test.ts +179 -0
- package/src/runtime/routes/integrations/slack/channel.ts +11 -3
- package/src/runtime/routes/integrations/slack/share.ts +45 -7
- package/src/runtime/routes/llm-context-normalization.ts +303 -0
- package/src/runtime/routes/log-export-routes.ts +42 -22
- package/src/runtime/routes/memory-item-routes.test.ts +3 -2
- package/src/runtime/routes/memory-item-routes.ts +1 -7
- package/src/runtime/routes/migration-routes.ts +122 -2
- package/src/runtime/routes/oauth-apps.ts +15 -17
- package/src/runtime/routes/oauth-providers.ts +4 -0
- package/src/runtime/routes/schedule-routes.ts +24 -11
- package/src/runtime/routes/settings-routes.ts +31 -102
- package/src/runtime/routes/skills-routes.ts +128 -9
- package/src/runtime/routes/stt-routes.ts +233 -0
- package/src/runtime/routes/subagents-routes.ts +14 -10
- package/src/runtime/routes/surface-action-routes.ts +41 -2
- package/src/runtime/routes/tts-routes.ts +108 -24
- package/src/runtime/routes/usage-routes.ts +38 -9
- package/src/runtime/routes/user-route-dispatcher.ts +50 -5
- package/src/runtime/routes/user-routes.ts +13 -1
- package/src/runtime/routes/work-items-routes.ts +8 -1
- package/src/runtime/routes/workspace-routes.test.ts +22 -0
- package/src/runtime/routes/workspace-routes.ts +8 -1
- package/src/runtime/routes/workspace-utils.ts +2 -0
- package/src/runtime/runtime-mode.ts +33 -0
- package/src/runtime/services/__tests__/analyze-conversation.test.ts +444 -0
- package/src/runtime/services/__tests__/analyze-deps-singleton.test.ts +67 -0
- package/src/runtime/services/__tests__/auto-analysis-prompt.test.ts +53 -0
- package/src/runtime/services/__tests__/manual-analysis-prompt.test.ts +41 -0
- package/src/runtime/services/analyze-conversation.ts +344 -0
- package/src/runtime/services/analyze-deps-singleton.ts +32 -0
- package/src/runtime/services/auto-analysis-prompt.ts +55 -0
- package/src/runtime/skill-route-registry.ts +49 -0
- package/src/runtime/slack-block-formatting.ts +437 -10
- package/src/schedule/scheduler.ts +57 -5
- package/src/security/ces-credential-client.ts +20 -0
- package/src/security/ces-rpc-credential-backend.ts +17 -0
- package/src/security/credential-backend.ts +5 -0
- package/src/security/oauth2.ts +68 -29
- package/src/security/secure-keys.ts +143 -27
- package/src/security/token-manager.ts +31 -10
- package/src/sequence/engine.ts +23 -0
- package/src/sequence/types.ts +1 -1
- package/src/skills/catalog-files.ts +554 -0
- package/src/skills/category-inference.ts +122 -0
- package/src/skills/clawhub-files.ts +213 -0
- package/src/skills/clawhub.ts +84 -23
- package/src/skills/skill-file-provider.ts +40 -0
- package/src/skills/skillssh-files.ts +395 -0
- package/src/skills/skillssh-registry.ts +4 -4
- package/src/stt/__tests__/daemon-batch-transcriber.test.ts +392 -0
- package/src/stt/__tests__/types.test.ts +89 -0
- package/src/stt/daemon-batch-transcriber.ts +195 -0
- package/src/stt/stt-stream-session.ts +499 -0
- package/src/stt/types.ts +330 -0
- package/src/stt/wav-encoder.test.ts +373 -0
- package/src/stt/wav-encoder.ts +175 -0
- package/src/subagent/manager.ts +169 -40
- package/src/subagent/types.ts +19 -0
- package/src/tools/apps/executors.ts +11 -2
- package/src/tools/browser/__tests__/auth-detector.test.ts +202 -108
- package/src/tools/browser/__tests__/browser-mode.test.ts +119 -0
- package/src/tools/browser/__tests__/browser-status.test.ts +123 -0
- package/src/tools/browser/auth-detector.ts +43 -12
- package/src/tools/browser/browser-execution.ts +1787 -342
- package/src/tools/browser/browser-manager.ts +81 -12
- package/src/tools/browser/browser-mode-constants.ts +12 -0
- package/src/tools/browser/browser-mode.ts +92 -0
- package/src/tools/browser/browser-status-constants.ts +33 -0
- package/src/tools/browser/cdp-client/__tests__/accessibility-snapshot.test.ts +318 -0
- package/src/tools/browser/cdp-client/__tests__/cdp-dom-helpers.test.ts +1175 -0
- package/src/tools/browser/cdp-client/__tests__/cdp-inspect-client.test.ts +1263 -0
- package/src/tools/browser/cdp-client/__tests__/extension-cdp-client.test.ts +359 -0
- package/src/tools/browser/cdp-client/__tests__/factory.test.ts +1993 -0
- package/src/tools/browser/cdp-client/__tests__/fixtures/ax-tree-nested-frames.json +64 -0
- package/src/tools/browser/cdp-client/__tests__/fixtures/ax-tree-simple.json +69 -0
- package/src/tools/browser/cdp-client/__tests__/local-cdp-client.test.ts +310 -0
- package/src/tools/browser/cdp-client/__tests__/types.test.ts +96 -0
- package/src/tools/browser/cdp-client/accessibility-snapshot.ts +387 -0
- package/src/tools/browser/cdp-client/cdp-dom-helpers.ts +695 -0
- package/src/tools/browser/cdp-client/cdp-inspect/__tests__/discovery.test.ts +1007 -0
- package/src/tools/browser/cdp-client/cdp-inspect/__tests__/ws-transport.test.ts +580 -0
- package/src/tools/browser/cdp-client/cdp-inspect/discovery.ts +744 -0
- package/src/tools/browser/cdp-client/cdp-inspect/ws-transport.ts +579 -0
- package/src/tools/browser/cdp-client/cdp-inspect-client.ts +868 -0
- package/src/tools/browser/cdp-client/errors.ts +49 -0
- package/src/tools/browser/cdp-client/extension-cdp-client.ts +148 -0
- package/src/tools/browser/cdp-client/factory.ts +914 -0
- package/src/tools/browser/cdp-client/index.ts +28 -0
- package/src/tools/browser/cdp-client/local-cdp-client.ts +187 -0
- package/src/tools/browser/cdp-client/types.ts +120 -0
- package/src/tools/credentials/vault.ts +35 -6
- package/src/tools/filesystem/edit.ts +1 -1
- package/src/tools/filesystem/list.ts +1 -1
- package/src/tools/filesystem/read.ts +1 -1
- package/src/tools/filesystem/write.ts +2 -1
- package/src/tools/host-filesystem/edit.ts +1 -1
- package/src/tools/host-filesystem/read.ts +12 -15
- package/src/tools/host-filesystem/write.ts +1 -1
- package/src/tools/host-terminal/host-shell.ts +21 -16
- package/src/tools/network/web-fetch.ts +5 -2
- package/src/tools/network/web-search.ts +5 -2
- package/src/tools/permission-checker.ts +77 -82
- package/src/tools/registry.ts +0 -2
- package/src/tools/secret-detection-handler.ts +34 -0
- package/src/tools/shared/filesystem/image-read.ts +61 -40
- package/src/tools/shared/shell-output.ts +3 -1
- package/src/tools/side-effects.ts +2 -0
- package/src/tools/skills/sandbox-runner.ts +3 -2
- package/src/tools/subagent/spawn.ts +47 -3
- package/src/tools/subagent/status.ts +2 -0
- package/src/tools/system/register.ts +2 -16
- package/src/tools/terminal/safe-env.ts +15 -0
- package/src/tools/terminal/shell.ts +36 -20
- package/src/tools/tool-approval-handler.ts +48 -2
- package/src/tools/tool-manifest.ts +21 -0
- package/src/tools/types.ts +19 -0
- package/src/tools/ui-surface/definitions.ts +6 -1
- package/src/tts/__tests__/provider-adapters.test.ts +834 -0
- package/src/tts/__tests__/provider-catalog-consistency.test.ts +196 -0
- package/src/tts/__tests__/provider-catalog.test.ts +183 -0
- package/src/tts/__tests__/provider-registry.test.ts +90 -0
- package/src/tts/provider-catalog.ts +201 -0
- package/src/tts/provider-registry.ts +73 -0
- package/src/tts/providers/deepgram-provider.ts +219 -0
- package/src/tts/providers/elevenlabs-provider.ts +211 -0
- package/src/tts/providers/fish-audio-provider.ts +183 -0
- package/src/tts/providers/index.ts +42 -0
- package/src/tts/providers/register-builtins.ts +130 -0
- package/src/tts/synthesize-text.ts +110 -0
- package/src/tts/tts-config-resolver.ts +78 -0
- package/src/tts/types.ts +153 -0
- package/src/types/onboarding-context.ts +7 -0
- package/src/util/abort-reasons.ts +58 -0
- package/src/util/device-id.ts +32 -16
- package/src/util/errors.ts +9 -1
- package/src/util/platform.ts +63 -24
- package/src/util/pricing.ts +66 -3
- package/src/util/spawn.ts +1 -1
- package/src/util/truncate.ts +4 -2
- package/src/util/unicode.ts +201 -0
- package/src/version.ts +19 -24
- package/src/watcher/engine.ts +23 -0
- package/src/watcher/watcher-store.ts +31 -0
- package/src/workspace/migrations/003-seed-device-id.ts +9 -3
- package/src/workspace/migrations/017-seed-persona-dirs.ts +68 -4
- package/src/workspace/migrations/029-seed-pkb.ts +1 -1
- package/src/workspace/migrations/031-drop-user-md.ts +317 -0
- package/src/workspace/migrations/031-llm-log-retention-zero-to-null.ts +73 -0
- package/src/workspace/migrations/032-tts-provider-unification.ts +227 -0
- package/src/workspace/migrations/033-stt-service-explicit-config.ts +122 -0
- package/src/workspace/migrations/034-remove-calls-voice-transcription-provider.ts +215 -0
- package/src/workspace/migrations/035-seed-slack-channel-persona.ts +50 -0
- package/src/workspace/migrations/036-update-pkb-index-bar.ts +37 -0
- package/src/workspace/migrations/037-create-meets-dir.ts +61 -0
- package/src/workspace/migrations/registry.ts +16 -0
- package/src/workspace/top-level-renderer.ts +31 -1
- package/src/workspace/turn-commit.ts +31 -0
- package/src/__tests__/chrome-cdp.test.ts +0 -419
- package/src/__tests__/email-cli.test.ts +0 -297
- package/src/__tests__/email-service-config-fallback.test.ts +0 -102
- package/src/__tests__/permission-mode-sse.test.ts +0 -418
- package/src/__tests__/permission-mode-store.test.ts +0 -277
- package/src/browser-extension-relay/protocol.ts +0 -63
- package/src/browser-extension-relay/server.ts +0 -203
- package/src/cli/commands/browser-relay.ts +0 -536
- package/src/config/schemas/sandbox.ts +0 -14
- package/src/email/guardrails.ts +0 -221
- package/src/email/provider.ts +0 -117
- package/src/email/providers/agentmail.ts +0 -361
- package/src/email/providers/index.ts +0 -65
- package/src/email/service.ts +0 -384
- package/src/email/types.ts +0 -126
- package/src/permissions/permission-mode-store.ts +0 -180
- package/src/prompts/templates/USER.md +0 -13
- package/src/providers/speech-to-text/types.ts +0 -17
- package/src/tools/browser/chrome-cdp.ts +0 -239
- package/src/tools/system/set-permission-mode.ts +0 -103
|
@@ -1,6 +1,8 @@
|
|
|
1
|
+
import { getConfig } from "../../config/loader.js";
|
|
1
2
|
import type { ImageContent } from "../../providers/types.js";
|
|
2
3
|
import { getLogger } from "../../util/logger.js";
|
|
3
4
|
import { truncate } from "../../util/truncate.js";
|
|
5
|
+
import { safeStringSlice } from "../../util/unicode.js";
|
|
4
6
|
import { credentialBroker } from "../credentials/broker.js";
|
|
5
7
|
import {
|
|
6
8
|
isPrivateOrLocalHost,
|
|
@@ -15,14 +17,58 @@ import {
|
|
|
15
17
|
detectCaptchaChallenge,
|
|
16
18
|
formatAuthChallenge,
|
|
17
19
|
} from "./auth-detector.js";
|
|
18
|
-
import type {
|
|
20
|
+
import type { RouteHandler } from "./browser-manager.js";
|
|
19
21
|
import { browserManager } from "./browser-manager.js";
|
|
22
|
+
import { type BrowserMode, normalizeBrowserMode } from "./browser-mode.js";
|
|
23
|
+
import { BROWSER_MODE } from "./browser-mode-constants.js";
|
|
20
24
|
import {
|
|
21
25
|
ensureScreencast,
|
|
22
26
|
getSender,
|
|
23
27
|
stopAllScreencasts,
|
|
24
28
|
stopBrowserScreencast,
|
|
25
29
|
} from "./browser-screencast.js";
|
|
30
|
+
import {
|
|
31
|
+
BROWSER_STATUS_INPUT_FIELD,
|
|
32
|
+
BROWSER_STATUS_MODE,
|
|
33
|
+
BROWSER_STATUS_MODES,
|
|
34
|
+
type BrowserStatusMode,
|
|
35
|
+
CDP_INSPECT_STATUS_DISCOVERY_CODE,
|
|
36
|
+
EXTENSION_STATUS_ERROR_MARKER,
|
|
37
|
+
} from "./browser-status-constants.js";
|
|
38
|
+
import {
|
|
39
|
+
formatAxSnapshot,
|
|
40
|
+
transformAxTree,
|
|
41
|
+
} from "./cdp-client/accessibility-snapshot.js";
|
|
42
|
+
import {
|
|
43
|
+
captureScreenshotJpeg,
|
|
44
|
+
dispatchClickAt,
|
|
45
|
+
dispatchHoverAt,
|
|
46
|
+
dispatchInsertText,
|
|
47
|
+
dispatchKeyPress,
|
|
48
|
+
dispatchWheelScroll,
|
|
49
|
+
evaluateExpression,
|
|
50
|
+
focusElement,
|
|
51
|
+
getCenterPoint,
|
|
52
|
+
getCurrentUrl,
|
|
53
|
+
getPageTitle,
|
|
54
|
+
navigateAndWait,
|
|
55
|
+
querySelectorBackendNodeId,
|
|
56
|
+
scrollIntoViewIfNeeded,
|
|
57
|
+
waitForSelector as cdpWaitForSelector,
|
|
58
|
+
waitForText as cdpWaitForText,
|
|
59
|
+
} from "./cdp-client/cdp-dom-helpers.js";
|
|
60
|
+
import { CdpError } from "./cdp-client/errors.js";
|
|
61
|
+
import {
|
|
62
|
+
buildCandidateList,
|
|
63
|
+
getCdpClient,
|
|
64
|
+
isDesktopAutoCooldownActive,
|
|
65
|
+
} from "./cdp-client/factory.js";
|
|
66
|
+
import type {
|
|
67
|
+
AttemptDiagnostic,
|
|
68
|
+
CdpClient,
|
|
69
|
+
CdpClientKind,
|
|
70
|
+
} from "./cdp-client/types.js";
|
|
71
|
+
import { checkBrowserRuntime } from "./runtime-check.js";
|
|
26
72
|
|
|
27
73
|
const log = getLogger("headless-browser");
|
|
28
74
|
|
|
@@ -32,43 +78,433 @@ export const NAVIGATE_TIMEOUT_MS = 15_000;
|
|
|
32
78
|
|
|
33
79
|
export const ACTION_TIMEOUT_MS = 10_000;
|
|
34
80
|
|
|
35
|
-
export const MAX_SNAPSHOT_ELEMENTS = 150;
|
|
36
|
-
|
|
37
|
-
export const INTERACTIVE_SELECTOR = [
|
|
38
|
-
"a[href]",
|
|
39
|
-
"button",
|
|
40
|
-
"input",
|
|
41
|
-
"select",
|
|
42
|
-
"textarea",
|
|
43
|
-
'[role="button"]',
|
|
44
|
-
'[role="link"]',
|
|
45
|
-
'[role="checkbox"]',
|
|
46
|
-
'[role="radio"]',
|
|
47
|
-
'[role="tab"]',
|
|
48
|
-
'[role="menuitem"]',
|
|
49
|
-
'[role="option"]',
|
|
50
|
-
'[role="combobox"]',
|
|
51
|
-
'[role="listbox"]',
|
|
52
|
-
'[contenteditable="true"]',
|
|
53
|
-
].join(", ");
|
|
54
|
-
|
|
55
|
-
export type SnapshotElement = {
|
|
56
|
-
eid: string;
|
|
57
|
-
tag: string;
|
|
58
|
-
attrs: Record<string, string>;
|
|
59
|
-
text: string;
|
|
60
|
-
};
|
|
61
|
-
|
|
62
81
|
export const MAX_WAIT_MS = 30_000;
|
|
63
82
|
|
|
64
83
|
export const MAX_EXTRACT_LENGTH = 50_000;
|
|
65
84
|
|
|
85
|
+
type StatusCheckMode = BrowserStatusMode;
|
|
86
|
+
|
|
87
|
+
const MODE_TRADEOFFS: Record<StatusCheckMode, string[]> = {
|
|
88
|
+
[BROWSER_STATUS_MODE.EXTENSION]: [
|
|
89
|
+
"Controls the user's existing Chrome profile and tabs.",
|
|
90
|
+
"Requires the Vellum extension to be paired and actively connected.",
|
|
91
|
+
"Best when the user wants the assistant to operate in their real browser session.",
|
|
92
|
+
],
|
|
93
|
+
[BROWSER_STATUS_MODE.CDP_INSPECT]: [
|
|
94
|
+
"Controls an existing Chrome instance launched with --remote-debugging-port.",
|
|
95
|
+
"Does not require the extension to be connected.",
|
|
96
|
+
"Requires remote debugging to stay enabled on localhost, which is more manual to maintain.",
|
|
97
|
+
],
|
|
98
|
+
[BROWSER_STATUS_MODE.LOCAL]: [
|
|
99
|
+
"Runs a dedicated Playwright-managed Chromium profile.",
|
|
100
|
+
"Most isolated and reliable fallback when extension/CDP inspect are unavailable.",
|
|
101
|
+
"Does not use the user's existing browser profile, so sessions/cookies may differ.",
|
|
102
|
+
],
|
|
103
|
+
};
|
|
104
|
+
|
|
105
|
+
interface BrowserStatusModeResult {
|
|
106
|
+
mode: StatusCheckMode;
|
|
107
|
+
available: boolean;
|
|
108
|
+
verified: "active_probe" | "preflight";
|
|
109
|
+
autoCandidate: boolean;
|
|
110
|
+
summary: string;
|
|
111
|
+
userActions: string[];
|
|
112
|
+
tradeoffs: string[];
|
|
113
|
+
details: Record<string, unknown>;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* IIFE evaluated inside the page via `Runtime.evaluate` to auto-dismiss
|
|
118
|
+
* common blocker modals (regulatory notices, cookie banners) that
|
|
119
|
+
* aren't exposed in the accessibility tree. Runs silently - if no
|
|
120
|
+
* matching modal is present the expression is a no-op.
|
|
121
|
+
*/
|
|
122
|
+
const DISMISS_MODALS_EXPRESSION = `(() => {
|
|
123
|
+
const dismissPatterns = /^(got it|accept|ok|dismiss|i understand|close)$/i;
|
|
124
|
+
const buttons = document.querySelectorAll('button, [role="button"], input[type="submit"]');
|
|
125
|
+
for (const btn of buttons) {
|
|
126
|
+
const text = (btn.textContent || '').trim();
|
|
127
|
+
if (dismissPatterns.test(text)) {
|
|
128
|
+
const modal = btn.closest('[role="dialog"], [class*="modal"], [class*="Modal"], [class*="overlay"], [class*="Overlay"]');
|
|
129
|
+
if (modal) {
|
|
130
|
+
btn.click();
|
|
131
|
+
break;
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
})()`;
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* IIFE evaluated by {@link executeBrowserExtract} when `include_links`
|
|
139
|
+
* is true. Walks `document.querySelectorAll('a[href]')`, caps at 200
|
|
140
|
+
* anchors, and shapes each entry as `{ text, href }`. Extracted to a
|
|
141
|
+
* module-level constant so the expression is shared between the
|
|
142
|
+
* runtime call site and any future refactors / tests that need to
|
|
143
|
+
* reason about the evaluated source.
|
|
144
|
+
*/
|
|
145
|
+
export const EXTRACT_LINKS_EXPRESSION = `
|
|
146
|
+
(() => {
|
|
147
|
+
const anchors = Array.from(document.querySelectorAll('a[href]'));
|
|
148
|
+
return anchors.slice(0, 200).map(a => ({
|
|
149
|
+
text: (a.textContent || '').trim().slice(0, 80),
|
|
150
|
+
href: a.href,
|
|
151
|
+
}));
|
|
152
|
+
})()
|
|
153
|
+
`;
|
|
154
|
+
|
|
155
|
+
// ── browser_mode parsing ─────────────────────────────────────────────
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* Parse the `browser_mode` field from a tool input map. Returns either
|
|
159
|
+
* a normalized {@link BrowserMode} or a pre-formatted error string
|
|
160
|
+
* suitable for returning directly in a tool response.
|
|
161
|
+
*
|
|
162
|
+
* When the value is absent, undefined, or empty the default `"auto"`
|
|
163
|
+
* is returned. Invalid values produce a descriptive error listing
|
|
164
|
+
* accepted values and aliases.
|
|
165
|
+
*/
|
|
166
|
+
export function parseBrowserMode(
|
|
167
|
+
input: Record<string, unknown>,
|
|
168
|
+
): { ok: true; mode: BrowserMode } | { ok: false; error: string } {
|
|
169
|
+
const raw = input.browser_mode;
|
|
170
|
+
const result = normalizeBrowserMode(raw);
|
|
171
|
+
if ("error" in result) {
|
|
172
|
+
return { ok: false, error: `Error: ${result.error}` };
|
|
173
|
+
}
|
|
174
|
+
return { ok: true, mode: result.mode };
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
// ── Mode-selection failure formatter ─────────────────────────────────
|
|
178
|
+
|
|
179
|
+
/**
|
|
180
|
+
* Remediation hints keyed by (candidateKind, discoveryCode | errorCode).
|
|
181
|
+
* Discovery codes come from DevToolsDiscoveryError; error codes come
|
|
182
|
+
* from CdpError. The formatter walks these in priority order: exact
|
|
183
|
+
* (kind, discoveryCode) first, then (kind, errorCode), then a generic
|
|
184
|
+
* per-kind fallback.
|
|
185
|
+
*/
|
|
186
|
+
const REMEDIATION_HINTS: Record<string, string[]> = {
|
|
187
|
+
// Extension backend
|
|
188
|
+
"extension:transport_error": [
|
|
189
|
+
"Ensure the Vellum browser extension is installed and enabled.",
|
|
190
|
+
"Check that the extension WebSocket connection is active (extension popup → status).",
|
|
191
|
+
"Try reconnecting the extension or reloading the extension service worker.",
|
|
192
|
+
],
|
|
193
|
+
// cdp-inspect backend — discovery-level failures
|
|
194
|
+
"cdp-inspect:unreachable": [
|
|
195
|
+
"Ensure Chrome/Chromium is running with --remote-debugging-port=9222.",
|
|
196
|
+
"Verify no firewall or antivirus is blocking localhost:9222.",
|
|
197
|
+
"Try: /Applications/Google\\ Chrome.app/Contents/MacOS/Google\\ Chrome --remote-debugging-port=9222",
|
|
198
|
+
],
|
|
199
|
+
"cdp-inspect:non_chrome": [
|
|
200
|
+
"The process listening on the configured port is not Chrome/Chromium.",
|
|
201
|
+
"Check if another application (dev server, proxy) is using port 9222.",
|
|
202
|
+
"Ensure Chrome is launched with --remote-debugging-port=9222.",
|
|
203
|
+
],
|
|
204
|
+
"cdp-inspect:timeout": [
|
|
205
|
+
"Chrome DevTools endpoint did not respond within the probe timeout.",
|
|
206
|
+
"Ensure Chrome is running and listening on the configured port.",
|
|
207
|
+
"Try increasing hostBrowser.cdpInspect.probeTimeoutMs in config.",
|
|
208
|
+
],
|
|
209
|
+
"cdp-inspect:no_targets": [
|
|
210
|
+
"Chrome is reachable but has no open page targets.",
|
|
211
|
+
"Open at least one browser tab, then retry.",
|
|
212
|
+
],
|
|
213
|
+
"cdp-inspect:non_loopback": [
|
|
214
|
+
"CDP inspect only allows loopback hosts (localhost, 127.0.0.1, ::1).",
|
|
215
|
+
"Update hostBrowser.cdpInspect.host in config to a loopback address.",
|
|
216
|
+
],
|
|
217
|
+
"cdp-inspect:transport_error": [
|
|
218
|
+
"CDP endpoint unreachable. Ensure Chrome is running with --remote-debugging-port.",
|
|
219
|
+
"Verify the configured host:port matches Chrome's DevTools listener.",
|
|
220
|
+
"Consider using browser_mode: 'extension' or 'local' as an alternative.",
|
|
221
|
+
],
|
|
222
|
+
// Local/Playwright backend
|
|
223
|
+
"local:transport_error": [
|
|
224
|
+
"The local Playwright-managed browser failed to start or connect.",
|
|
225
|
+
"Check that the Playwright browser binary is downloaded (bun run install).",
|
|
226
|
+
"Try closing any stale Chromium processes and retrying.",
|
|
227
|
+
],
|
|
228
|
+
};
|
|
229
|
+
|
|
230
|
+
/**
|
|
231
|
+
* Build a human-readable, tool-response-ready error string from a
|
|
232
|
+
* pinned-mode failure. Includes:
|
|
233
|
+
* - the requested mode
|
|
234
|
+
* - ordered attempted modes with exact failure reasons
|
|
235
|
+
* - a remediation checklist tailored by backend and failure code
|
|
236
|
+
*
|
|
237
|
+
* Exported for testing.
|
|
238
|
+
*/
|
|
239
|
+
export function formatModeSelectionFailure(
|
|
240
|
+
requestedMode: BrowserMode,
|
|
241
|
+
error: CdpError,
|
|
242
|
+
): string {
|
|
243
|
+
const lines: string[] = [];
|
|
244
|
+
lines.push(`Error: Browser mode "${requestedMode}" failed.`);
|
|
245
|
+
lines.push("");
|
|
246
|
+
|
|
247
|
+
const diagnostics: readonly AttemptDiagnostic[] =
|
|
248
|
+
error.attemptDiagnostics ?? [];
|
|
249
|
+
|
|
250
|
+
if (diagnostics.length > 0) {
|
|
251
|
+
lines.push("Attempted backends:");
|
|
252
|
+
for (const diag of diagnostics) {
|
|
253
|
+
const status =
|
|
254
|
+
diag.stage === "success" ? "OK" : `FAILED at ${diag.stage}`;
|
|
255
|
+
lines.push(` - ${diag.candidateKind}: ${status}`);
|
|
256
|
+
if (diag.errorMessage) {
|
|
257
|
+
lines.push(` Reason: ${diag.errorMessage}`);
|
|
258
|
+
}
|
|
259
|
+
if (diag.discoveryCode) {
|
|
260
|
+
lines.push(` Discovery code: ${diag.discoveryCode}`);
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
lines.push("");
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
// Collect remediation hints
|
|
267
|
+
const hints = collectRemediationHints(diagnostics, error);
|
|
268
|
+
if (hints.length > 0) {
|
|
269
|
+
lines.push("Remediation:");
|
|
270
|
+
for (const hint of hints) {
|
|
271
|
+
lines.push(` - ${hint}`);
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
return lines.join("\n");
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
/**
|
|
279
|
+
* Gather remediation hints based on attempt diagnostics and the error.
|
|
280
|
+
* Walks each diagnostic and looks up hints by (kind, discoveryCode),
|
|
281
|
+
* then (kind, errorCode), then generic kind-level fallback.
|
|
282
|
+
*/
|
|
283
|
+
function collectRemediationHints(
|
|
284
|
+
diagnostics: readonly AttemptDiagnostic[],
|
|
285
|
+
error: CdpError,
|
|
286
|
+
): string[] {
|
|
287
|
+
const seen = new Set<string>();
|
|
288
|
+
const hints: string[] = [];
|
|
289
|
+
|
|
290
|
+
const addHints = (key: string) => {
|
|
291
|
+
const list = REMEDIATION_HINTS[key];
|
|
292
|
+
if (!list) return;
|
|
293
|
+
for (const hint of list) {
|
|
294
|
+
if (!seen.has(hint)) {
|
|
295
|
+
seen.add(hint);
|
|
296
|
+
hints.push(hint);
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
};
|
|
300
|
+
|
|
301
|
+
for (const diag of diagnostics) {
|
|
302
|
+
if (diag.stage === "success") continue;
|
|
303
|
+
if (diag.discoveryCode) {
|
|
304
|
+
addHints(`${diag.candidateKind}:${diag.discoveryCode}`);
|
|
305
|
+
}
|
|
306
|
+
if (diag.errorCode) {
|
|
307
|
+
addHints(`${diag.candidateKind}:${diag.errorCode}`);
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
// Fallback: if no diagnostics but we have a top-level error, use
|
|
312
|
+
// the error code with a generic candidate kind derived from the mode.
|
|
313
|
+
if (diagnostics.length === 0 && error.code) {
|
|
314
|
+
// Try to infer the candidate kind from the error message
|
|
315
|
+
for (const kind of BROWSER_STATUS_MODES) {
|
|
316
|
+
if (error.message.toLowerCase().includes(kind)) {
|
|
317
|
+
addHints(`${kind}:${error.code}`);
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
return hints;
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
/**
|
|
326
|
+
* Parse browser_mode from input and acquire a CdpClient. Returns
|
|
327
|
+
* either a `{ cdp, browserMode }` pair on success or a pre-formatted
|
|
328
|
+
* `{ errorResult }` on failure (invalid mode or pinned-mode
|
|
329
|
+
* precondition not met).
|
|
330
|
+
*
|
|
331
|
+
* This is the single integration point for all CDP-backed tool
|
|
332
|
+
* functions. Using it ensures every tool:
|
|
333
|
+
* - normalizes aliases (`cdp-debugger` -> `cdp-inspect`, etc.)
|
|
334
|
+
* - passes the mode preference to the factory
|
|
335
|
+
* - surfaces a remediation-rich error on pinned-mode failures
|
|
336
|
+
*
|
|
337
|
+
* Per-conversation stickiness: when the incoming `browser_mode` is
|
|
338
|
+
* `"auto"` and the conversation has already resolved to a backend
|
|
339
|
+
* kind on a prior call, the factory is pinned to that kind instead
|
|
340
|
+
* of re-running the auto priority list. This prevents
|
|
341
|
+
* `browser_navigate` (e.g. pinned to `local`) and `browser_screenshot`
|
|
342
|
+
* (default auto) in the same conversation from landing on different
|
|
343
|
+
* Chrome instances. Explicit non-auto modes override and update the
|
|
344
|
+
* memo; teardown via browser_close / browser_detach clears it.
|
|
345
|
+
*
|
|
346
|
+
* The returned client is wrapped so its first successful `send()`
|
|
347
|
+
* writes the resolved kind back to the conversation memo.
|
|
348
|
+
*/
|
|
349
|
+
export function acquireCdpClientWithMode(
|
|
350
|
+
input: Record<string, unknown>,
|
|
351
|
+
context: ToolContext,
|
|
352
|
+
):
|
|
353
|
+
| {
|
|
354
|
+
cdp: ReturnType<typeof getCdpClient>;
|
|
355
|
+
browserMode: BrowserMode;
|
|
356
|
+
errorResult?: never;
|
|
357
|
+
}
|
|
358
|
+
| { cdp?: never; browserMode?: never; errorResult: ToolExecutionResult } {
|
|
359
|
+
const modeResult = parseBrowserMode(input);
|
|
360
|
+
if (!modeResult.ok) {
|
|
361
|
+
return {
|
|
362
|
+
errorResult: { content: modeResult.error, isError: true },
|
|
363
|
+
};
|
|
364
|
+
}
|
|
365
|
+
const browserMode = modeResult.mode;
|
|
366
|
+
|
|
367
|
+
const rememberedKind = browserManager.getPreferredBackendKind(
|
|
368
|
+
context.conversationId,
|
|
369
|
+
);
|
|
370
|
+
const effectiveMode: BrowserMode =
|
|
371
|
+
browserMode === "auto" && rememberedKind !== null
|
|
372
|
+
? rememberedKind
|
|
373
|
+
: browserMode;
|
|
374
|
+
|
|
375
|
+
try {
|
|
376
|
+
const raw = getCdpClient(context, { mode: effectiveMode });
|
|
377
|
+
const cdp = wrapWithKindMemo(raw, context.conversationId);
|
|
378
|
+
return { cdp, browserMode };
|
|
379
|
+
} catch (err) {
|
|
380
|
+
// Sticky-mode fallback: the caller requested "auto" but we pinned to
|
|
381
|
+
// a remembered backend kind that has since become unavailable. Drop
|
|
382
|
+
// the stale memo and retry with fresh auto selection so a dead
|
|
383
|
+
// sticky preference doesn't surface as a hard failure.
|
|
384
|
+
if (browserMode === "auto" && effectiveMode !== "auto") {
|
|
385
|
+
browserManager.clearPreferredBackendKind(context.conversationId);
|
|
386
|
+
try {
|
|
387
|
+
const raw = getCdpClient(context, { mode: "auto" });
|
|
388
|
+
const cdp = wrapWithKindMemo(raw, context.conversationId);
|
|
389
|
+
return { cdp, browserMode };
|
|
390
|
+
} catch (retryErr) {
|
|
391
|
+
if (retryErr instanceof CdpError) {
|
|
392
|
+
return {
|
|
393
|
+
errorResult: {
|
|
394
|
+
content: formatModeSelectionFailure("auto", retryErr),
|
|
395
|
+
isError: true,
|
|
396
|
+
},
|
|
397
|
+
};
|
|
398
|
+
}
|
|
399
|
+
throw retryErr;
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
if (err instanceof CdpError && browserMode !== "auto") {
|
|
403
|
+
return {
|
|
404
|
+
errorResult: {
|
|
405
|
+
content: formatModeSelectionFailure(browserMode, err),
|
|
406
|
+
isError: true,
|
|
407
|
+
},
|
|
408
|
+
};
|
|
409
|
+
}
|
|
410
|
+
throw err;
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
/**
|
|
415
|
+
* Wrap a {@link ScopedCdpClient} so the first successful `send()`
|
|
416
|
+
* records the resolved backend kind in the conversation's
|
|
417
|
+
* `preferredBackendKinds` memo. Subsequent sends are no-ops for the
|
|
418
|
+
* memo; dispose() delegates to the underlying client.
|
|
419
|
+
*/
|
|
420
|
+
function wrapWithKindMemo(
|
|
421
|
+
inner: ReturnType<typeof getCdpClient>,
|
|
422
|
+
conversationId: string,
|
|
423
|
+
): ReturnType<typeof getCdpClient> {
|
|
424
|
+
let recorded = false;
|
|
425
|
+
return {
|
|
426
|
+
get kind() {
|
|
427
|
+
return inner.kind;
|
|
428
|
+
},
|
|
429
|
+
conversationId: inner.conversationId,
|
|
430
|
+
async send<T = unknown>(
|
|
431
|
+
method: string,
|
|
432
|
+
params?: Record<string, unknown>,
|
|
433
|
+
signal?: AbortSignal,
|
|
434
|
+
): Promise<T> {
|
|
435
|
+
const result = await inner.send<T>(method, params, signal);
|
|
436
|
+
if (!recorded) {
|
|
437
|
+
browserManager.setPreferredBackendKind(conversationId, inner.kind);
|
|
438
|
+
recorded = true;
|
|
439
|
+
}
|
|
440
|
+
return result;
|
|
441
|
+
},
|
|
442
|
+
dispose(): void {
|
|
443
|
+
inner.dispose();
|
|
444
|
+
},
|
|
445
|
+
};
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
// ── CDP error diagnostics helper ─────────────────────────────────────
|
|
449
|
+
|
|
450
|
+
/**
|
|
451
|
+
* Check whether a caught error is a {@link CdpError} carrying
|
|
452
|
+
* {@link AttemptDiagnostic attempt diagnostics} from the factory's
|
|
453
|
+
* failover walk. When the browser_mode is pinned (not "auto") and
|
|
454
|
+
* diagnostics are present, format the error with the full remediation
|
|
455
|
+
* checklist via {@link formatModeSelectionFailure}. Otherwise return
|
|
456
|
+
* `null` so the caller falls through to its generic error message.
|
|
457
|
+
*
|
|
458
|
+
* This handles the case where pinned-mode unavailability is surfaced
|
|
459
|
+
* on the first `cdp.send()` (via `sendWithFailover`) rather than
|
|
460
|
+
* during client construction (which `acquireCdpClientWithMode` already
|
|
461
|
+
* covers).
|
|
462
|
+
*/
|
|
463
|
+
function formatCdpSendDiagnostics(
|
|
464
|
+
err: unknown,
|
|
465
|
+
browserMode: BrowserMode,
|
|
466
|
+
): string | null {
|
|
467
|
+
if (
|
|
468
|
+
err instanceof CdpError &&
|
|
469
|
+
browserMode !== "auto" &&
|
|
470
|
+
err.code === "transport_error" &&
|
|
471
|
+
err.attemptDiagnostics
|
|
472
|
+
) {
|
|
473
|
+
return formatModeSelectionFailure(browserMode, err);
|
|
474
|
+
}
|
|
475
|
+
return null;
|
|
476
|
+
}
|
|
477
|
+
|
|
66
478
|
// ── Shared element resolution ────────────────────────────────────────
|
|
67
479
|
|
|
68
|
-
|
|
480
|
+
/**
|
|
481
|
+
* Discriminated union returned by {@link resolveElement}. The
|
|
482
|
+
* `"backend"` variant is produced when an `element_id` from the most
|
|
483
|
+
* recent AX-tree snapshot is resolved to a CDP `backendNodeId`; the
|
|
484
|
+
* `"selector"` variant is produced when the caller passed a raw CSS
|
|
485
|
+
* `selector` that should be resolved via `DOM.querySelector` at
|
|
486
|
+
* send-time by the individual tool.
|
|
487
|
+
*
|
|
488
|
+
* Consumed by CDP-native interaction tools (click, hover, type, …)
|
|
489
|
+
* that talk to CDP directly.
|
|
490
|
+
*/
|
|
491
|
+
export type ResolvedElement =
|
|
492
|
+
| { kind: "backend"; backendNodeId: number; eid: string }
|
|
493
|
+
| { kind: "selector"; selector: string };
|
|
494
|
+
|
|
495
|
+
/**
|
|
496
|
+
* Resolve an element reference (either `element_id` from a prior
|
|
497
|
+
* snapshot or a raw `selector`) for CDP-native tools. Returns a
|
|
498
|
+
* {@link ResolvedElement} discriminated union so callers can branch
|
|
499
|
+
* on whether a backendNodeId was recovered from the snapshot map.
|
|
500
|
+
* Returns `{ resolved: null, error: "Error: …" }` on invalid input
|
|
501
|
+
* or when an `element_id` is provided but the snapshot map is
|
|
502
|
+
* empty/stale.
|
|
503
|
+
*/
|
|
504
|
+
export function resolveElement(
|
|
69
505
|
conversationId: string,
|
|
70
506
|
input: Record<string, unknown>,
|
|
71
|
-
): {
|
|
507
|
+
): { resolved: ResolvedElement | null; error: string | null } {
|
|
72
508
|
const elementId =
|
|
73
509
|
typeof input.element_id === "string" ? input.element_id : null;
|
|
74
510
|
const rawSelector =
|
|
@@ -76,26 +512,32 @@ export function resolveSelector(
|
|
|
76
512
|
|
|
77
513
|
if (!elementId && !rawSelector) {
|
|
78
514
|
return {
|
|
79
|
-
|
|
515
|
+
resolved: null,
|
|
80
516
|
error: "Error: Either element_id or selector is required.",
|
|
81
517
|
};
|
|
82
518
|
}
|
|
83
519
|
|
|
84
520
|
if (elementId) {
|
|
85
|
-
const
|
|
521
|
+
const backendNodeId = browserManager.resolveSnapshotBackendNodeId(
|
|
86
522
|
conversationId,
|
|
87
523
|
elementId,
|
|
88
524
|
);
|
|
89
|
-
if (
|
|
525
|
+
if (backendNodeId !== null) {
|
|
90
526
|
return {
|
|
91
|
-
|
|
92
|
-
error:
|
|
527
|
+
resolved: { kind: "backend", backendNodeId, eid: elementId },
|
|
528
|
+
error: null,
|
|
93
529
|
};
|
|
94
530
|
}
|
|
95
|
-
return {
|
|
531
|
+
return {
|
|
532
|
+
resolved: null,
|
|
533
|
+
error: `Error: element_id "${elementId}" not found. Run browser_snapshot first to get current element IDs.`,
|
|
534
|
+
};
|
|
96
535
|
}
|
|
97
536
|
|
|
98
|
-
return {
|
|
537
|
+
return {
|
|
538
|
+
resolved: { kind: "selector", selector: rawSelector! },
|
|
539
|
+
error: null,
|
|
540
|
+
};
|
|
99
541
|
}
|
|
100
542
|
|
|
101
543
|
// ── browser_navigate ─────────────────────────────────────────────────
|
|
@@ -108,6 +550,8 @@ export async function executeBrowserNavigate(
|
|
|
108
550
|
return { content: "Error: operation was cancelled", isError: true };
|
|
109
551
|
}
|
|
110
552
|
|
|
553
|
+
// Pre-flight URL validation runs before CDP acquisition so we fail
|
|
554
|
+
// fast on obviously invalid URLs without opening a browser session.
|
|
111
555
|
const parsedUrl = parseUrl(input.url);
|
|
112
556
|
if (!parsedUrl) {
|
|
113
557
|
return {
|
|
@@ -122,7 +566,8 @@ export async function executeBrowserNavigate(
|
|
|
122
566
|
const allowPrivateNetwork = input.allow_private_network === true;
|
|
123
567
|
const safeRequestedUrl = sanitizeUrlForOutput(parsedUrl);
|
|
124
568
|
|
|
125
|
-
// Block private/local targets by default
|
|
569
|
+
// Block private/local targets by default. Runs before any CDP session
|
|
570
|
+
// is opened so we fail fast on obviously invalid URLs.
|
|
126
571
|
if (!allowPrivateNetwork && isPrivateOrLocalHost(parsedUrl.hostname)) {
|
|
127
572
|
return {
|
|
128
573
|
content: `Error: Refusing to navigate to local/private network target (${parsedUrl.hostname}). Set allow_private_network=true if you explicitly need it.`,
|
|
@@ -130,7 +575,7 @@ export async function executeBrowserNavigate(
|
|
|
130
575
|
};
|
|
131
576
|
}
|
|
132
577
|
|
|
133
|
-
// DNS resolution check for non-literal hostnames
|
|
578
|
+
// DNS resolution check for non-literal hostnames.
|
|
134
579
|
if (!allowPrivateNetwork) {
|
|
135
580
|
const resolution = await resolveRequestAddress(
|
|
136
581
|
parsedUrl.hostname,
|
|
@@ -145,29 +590,40 @@ export async function executeBrowserNavigate(
|
|
|
145
590
|
}
|
|
146
591
|
}
|
|
147
592
|
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
593
|
+
// URL validation passed — acquire the CDP client.
|
|
594
|
+
const acquired = acquireCdpClientWithMode(input, context);
|
|
595
|
+
if (acquired.errorResult) return acquired.errorResult;
|
|
596
|
+
const { cdp, browserMode } = acquired;
|
|
597
|
+
|
|
598
|
+
// Screencast + handoff are Playwright-backed and only meaningful
|
|
599
|
+
// for the local sacrificial-profile path. On the extension path the
|
|
600
|
+
// user already has their own Chrome window, so both are no-ops.
|
|
601
|
+
const sender =
|
|
602
|
+
cdp.kind === "local" ? getSender(context.conversationId) : null;
|
|
603
|
+
if (cdp.kind === "local" && sender) {
|
|
154
604
|
await ensureScreencast(context.conversationId);
|
|
155
605
|
}
|
|
156
606
|
|
|
607
|
+
// SSRF route interception uses the Playwright page.route() API to
|
|
608
|
+
// block redirect-time requests to private networks. This only works
|
|
609
|
+
// on the local path where Playwright manages the browser; on the
|
|
610
|
+
// extension/cdp-inspect paths, CDP navigates a different browser so
|
|
611
|
+
// the Playwright route handler would be a no-op. The post-navigation
|
|
612
|
+
// final URL check below provides defense-in-depth for all paths.
|
|
613
|
+
let routeHandler: RouteHandler | null = null;
|
|
614
|
+
let blockedUrl: string | null = null;
|
|
615
|
+
|
|
157
616
|
try {
|
|
158
|
-
const page = await browserManager.getOrCreateSessionPage(
|
|
159
|
-
context.conversationId,
|
|
160
|
-
);
|
|
161
617
|
log.debug(
|
|
162
618
|
{ url: safeRequestedUrl, conversationId: context.conversationId },
|
|
163
619
|
"Navigating",
|
|
164
620
|
);
|
|
165
621
|
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
622
|
+
if (
|
|
623
|
+
cdp.kind === "local" &&
|
|
624
|
+
!allowPrivateNetwork &&
|
|
625
|
+
browserManager.supportsRouteInterception
|
|
626
|
+
) {
|
|
171
627
|
// Cache DNS results per-hostname to avoid redundant lookups on subrequests
|
|
172
628
|
// (heavy sites like DoorDash fire hundreds of requests to the same CDN hostnames).
|
|
173
629
|
// Use a short TTL to mitigate DNS rebinding attacks where a hostname first
|
|
@@ -242,47 +698,107 @@ export async function executeBrowserNavigate(
|
|
|
242
698
|
);
|
|
243
699
|
}
|
|
244
700
|
};
|
|
701
|
+
// Bridge through browserManager to reach the Playwright Page for
|
|
702
|
+
// route installation. The route handler intercepts redirect-time
|
|
703
|
+
// requests before Page.navigate's network fetches can hit them.
|
|
704
|
+
const page = await browserManager.getOrCreateSessionPage(
|
|
705
|
+
context.conversationId,
|
|
706
|
+
);
|
|
245
707
|
await page.route("**/*", routeHandler);
|
|
246
708
|
}
|
|
247
709
|
|
|
248
|
-
//
|
|
249
|
-
// the page
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
710
|
+
// Read the current URL BEFORE calling navigateAndWait so we can
|
|
711
|
+
// detect the "page never moved" case on timeout.
|
|
712
|
+
const urlBeforeNav = await getCurrentUrl(cdp, context.signal);
|
|
713
|
+
|
|
714
|
+
// Navigate via CDP Page.navigate + document.readyState polling.
|
|
715
|
+
// navigateAndWait returns { finalUrl, timedOut }; HTTP status is
|
|
716
|
+
// not available on the CDP path because Page.navigate does not
|
|
717
|
+
// surface the response status.
|
|
718
|
+
const { finalUrl, timedOut: navigationTimedOut } = await navigateAndWait(
|
|
719
|
+
cdp,
|
|
720
|
+
parsedUrl.href,
|
|
721
|
+
{ timeoutMs: NAVIGATE_TIMEOUT_MS },
|
|
722
|
+
context.signal,
|
|
723
|
+
);
|
|
724
|
+
|
|
725
|
+
// Defense-in-depth: check the final URL after navigation completes.
|
|
726
|
+
// This catches redirect-based SSRF even when Playwright route
|
|
727
|
+
// interception is unavailable (e.g. extension-backed sessions where
|
|
728
|
+
// the CDP transport is separate from the Playwright page).
|
|
729
|
+
if (!allowPrivateNetwork) {
|
|
730
|
+
const finalParsed = parseUrl(finalUrl);
|
|
731
|
+
if (
|
|
732
|
+
finalParsed &&
|
|
733
|
+
(isPrivateOrLocalHost(finalParsed.hostname) ||
|
|
734
|
+
(
|
|
735
|
+
await resolveRequestAddress(
|
|
736
|
+
finalParsed.hostname,
|
|
737
|
+
resolveHostAddresses,
|
|
738
|
+
false,
|
|
739
|
+
)
|
|
740
|
+
).blockedAddress)
|
|
741
|
+
) {
|
|
742
|
+
// Navigate the page away from the private target to prevent
|
|
743
|
+
// follow-up tool calls (e.g. browser_snapshot) from reading
|
|
744
|
+
// the already-loaded private content.
|
|
745
|
+
try {
|
|
746
|
+
await navigateAndWait(
|
|
747
|
+
cdp,
|
|
748
|
+
"about:blank",
|
|
749
|
+
{ timeoutMs: 3_000 },
|
|
750
|
+
context.signal,
|
|
751
|
+
);
|
|
752
|
+
} catch {
|
|
753
|
+
// Best-effort — if the reset fails, the CDP session will be
|
|
754
|
+
// disposed in the finally block anyway.
|
|
266
755
|
}
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
756
|
+
// Clean up the route handler before returning to avoid leaking
|
|
757
|
+
// a stale interception handler on the session page.
|
|
758
|
+
if (routeHandler) {
|
|
759
|
+
const page = await browserManager.getOrCreateSessionPage(
|
|
760
|
+
context.conversationId,
|
|
761
|
+
);
|
|
762
|
+
await page.unroute("**/*", routeHandler);
|
|
763
|
+
routeHandler = null;
|
|
764
|
+
}
|
|
765
|
+
return {
|
|
766
|
+
content: `Error: Navigation blocked. Final URL resolved to a local/private network target (${sanitizeUrlForOutput(finalParsed)}). Set allow_private_network=true if you explicitly need it.`,
|
|
767
|
+
isError: true,
|
|
768
|
+
};
|
|
769
|
+
}
|
|
770
|
+
}
|
|
771
|
+
if (navigationTimedOut) {
|
|
772
|
+
// If the page URL never changed from before navigation, the page
|
|
773
|
+
// never actually loaded - re-throw instead of reporting success.
|
|
774
|
+
if (finalUrl === urlBeforeNav && urlBeforeNav !== parsedUrl.href) {
|
|
775
|
+
throw new Error(
|
|
776
|
+
`Navigation to ${parsedUrl.href} timed out after ${NAVIGATE_TIMEOUT_MS}ms`,
|
|
271
777
|
);
|
|
272
|
-
} else {
|
|
273
|
-
throw navErr;
|
|
274
778
|
}
|
|
779
|
+
log.info(
|
|
780
|
+
{ url: safeRequestedUrl },
|
|
781
|
+
"Navigation timed out waiting for document.readyState, continuing with partial load",
|
|
782
|
+
);
|
|
275
783
|
}
|
|
276
784
|
|
|
277
|
-
// Remove the route handler now that navigation is
|
|
785
|
+
// Remove the Playwright route handler now that navigation is
|
|
786
|
+
// complete (local path only — route interception is gated above).
|
|
278
787
|
if (routeHandler) {
|
|
788
|
+
const page = await browserManager.getOrCreateSessionPage(
|
|
789
|
+
context.conversationId,
|
|
790
|
+
);
|
|
279
791
|
await page.unroute("**/*", routeHandler);
|
|
280
792
|
routeHandler = null;
|
|
281
793
|
}
|
|
282
794
|
|
|
283
|
-
//
|
|
284
|
-
//
|
|
285
|
-
|
|
795
|
+
// Window positioning is a Playwright-internal affordance - on the
|
|
796
|
+
// extension path the user owns their Chrome window, so positioning
|
|
797
|
+
// is a no-op.
|
|
798
|
+
if (
|
|
799
|
+
cdp.kind === "local" &&
|
|
800
|
+
!browserManager.isInteractive(context.conversationId)
|
|
801
|
+
) {
|
|
286
802
|
await browserManager.positionWindowSidebar();
|
|
287
803
|
}
|
|
288
804
|
|
|
@@ -293,38 +809,34 @@ export async function executeBrowserNavigate(
|
|
|
293
809
|
};
|
|
294
810
|
}
|
|
295
811
|
|
|
296
|
-
// Navigation changed the page content, so clear stale snapshot
|
|
297
|
-
//
|
|
298
|
-
//
|
|
299
|
-
|
|
300
|
-
browserManager.clearSnapshotMap(context.conversationId);
|
|
812
|
+
// Navigation changed the page content, so clear stale snapshot
|
|
813
|
+
// mappings regardless of backend. The backendNodeId map is shared
|
|
814
|
+
// per-conversation state that needs to be invalidated on any nav.
|
|
815
|
+
browserManager.clearSnapshotBackendNodeMap(context.conversationId);
|
|
301
816
|
|
|
302
|
-
// Auto-dismiss common blocker modals (regulatory notices, cookie
|
|
303
|
-
// that aren't exposed in the accessibility tree. Runs
|
|
304
|
-
// modal is present the evaluate is a no-op.
|
|
817
|
+
// Auto-dismiss common blocker modals (regulatory notices, cookie
|
|
818
|
+
// banners) that aren't exposed in the accessibility tree. Runs
|
|
819
|
+
// silently - if no modal is present the evaluate is a no-op.
|
|
305
820
|
try {
|
|
306
|
-
await
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
const modal = btn.closest('[role="dialog"], [class*="modal"], [class*="Modal"], [class*="overlay"], [class*="Overlay"]');
|
|
313
|
-
if (modal) {
|
|
314
|
-
btn.click();
|
|
315
|
-
break;
|
|
316
|
-
}
|
|
317
|
-
}
|
|
318
|
-
}
|
|
319
|
-
})()`);
|
|
821
|
+
await evaluateExpression(
|
|
822
|
+
cdp,
|
|
823
|
+
DISMISS_MODALS_EXPRESSION,
|
|
824
|
+
{},
|
|
825
|
+
context.signal,
|
|
826
|
+
);
|
|
320
827
|
} catch {
|
|
321
828
|
// Page may have navigated during evaluate - safe to ignore
|
|
322
829
|
}
|
|
323
830
|
|
|
324
|
-
const finalUrl = page.url();
|
|
325
831
|
const safeFinalUrl = sanitizeUrlForOutput(new URL(finalUrl));
|
|
326
|
-
const title = await
|
|
327
|
-
|
|
832
|
+
const title = await getPageTitle(cdp, context.signal);
|
|
833
|
+
// HTTP status is not available on the CDP path: `Page.navigate`
|
|
834
|
+
// resolves the frame id and (on failure) an error text, but does
|
|
835
|
+
// not carry the response status code. Both the local and extension
|
|
836
|
+
// paths therefore print "unknown" here. A future phase may subscribe
|
|
837
|
+
// to `Network.responseReceived` events during the navigation window
|
|
838
|
+
// if the status is needed again.
|
|
839
|
+
const status: number | null = null;
|
|
328
840
|
|
|
329
841
|
const lines: string[] = [
|
|
330
842
|
`Requested URL: ${safeRequestedUrl}`,
|
|
@@ -335,7 +847,7 @@ export async function executeBrowserNavigate(
|
|
|
335
847
|
|
|
336
848
|
if (navigationTimedOut) {
|
|
337
849
|
lines.push(
|
|
338
|
-
`Note: Page is still loading (
|
|
850
|
+
`Note: Page is still loading (document.readyState timed out). The page should still be interactive - use browser_snapshot to check.`,
|
|
339
851
|
);
|
|
340
852
|
}
|
|
341
853
|
|
|
@@ -343,10 +855,14 @@ export async function executeBrowserNavigate(
|
|
|
343
855
|
lines.push(`Note: Page redirected from the requested URL.`);
|
|
344
856
|
}
|
|
345
857
|
|
|
346
|
-
// Detect auth challenges (login pages, 2FA, OAuth consent) and CAPTCHA
|
|
858
|
+
// Detect auth challenges (login pages, 2FA, OAuth consent) and CAPTCHA
|
|
859
|
+
// challenges via the CDP-migrated auth-detector helpers.
|
|
347
860
|
try {
|
|
348
|
-
const authChallenge = await detectAuthChallenge(
|
|
349
|
-
const captchaChallenge = await detectCaptchaChallenge(
|
|
861
|
+
const authChallenge = await detectAuthChallenge(cdp, context.signal);
|
|
862
|
+
const captchaChallenge = await detectCaptchaChallenge(
|
|
863
|
+
cdp,
|
|
864
|
+
context.signal,
|
|
865
|
+
);
|
|
350
866
|
// CAPTCHA takes priority - it blocks all interaction including login
|
|
351
867
|
let challenge = captchaChallenge ?? authChallenge;
|
|
352
868
|
|
|
@@ -359,12 +875,12 @@ export async function executeBrowserNavigate(
|
|
|
359
875
|
return { content: "Navigation cancelled.", isError: true };
|
|
360
876
|
}
|
|
361
877
|
await new Promise((r) => setTimeout(r, 1000));
|
|
362
|
-
const still = await detectCaptchaChallenge(
|
|
878
|
+
const still = await detectCaptchaChallenge(cdp, context.signal);
|
|
363
879
|
if (!still) {
|
|
364
880
|
log.info("CAPTCHA auto-resolved");
|
|
365
881
|
// Re-check for auth challenge now that CAPTCHA is gone -
|
|
366
882
|
// the page may have loaded a login form behind it.
|
|
367
|
-
challenge = await detectAuthChallenge(
|
|
883
|
+
challenge = await detectAuthChallenge(cdp, context.signal);
|
|
368
884
|
break;
|
|
369
885
|
}
|
|
370
886
|
}
|
|
@@ -373,7 +889,11 @@ export async function executeBrowserNavigate(
|
|
|
373
889
|
if (challenge) {
|
|
374
890
|
if (challenge.type === "captcha") {
|
|
375
891
|
// CAPTCHA persisted after auto-resolve wait - hand off to user
|
|
376
|
-
|
|
892
|
+
// only when we have a local Playwright-managed Chrome window
|
|
893
|
+
// AND a sender is registered. The extension path falls back
|
|
894
|
+
// to the text-only "solve manually" branch because the user
|
|
895
|
+
// already owns their Chrome window.
|
|
896
|
+
if (cdp.kind === "local" && sender) {
|
|
377
897
|
const { startHandoff } = await import("./browser-handoff.js");
|
|
378
898
|
await startHandoff(context.conversationId, {
|
|
379
899
|
reason: "captcha",
|
|
@@ -381,15 +901,18 @@ export async function executeBrowserNavigate(
|
|
|
381
901
|
"Cloudflare verification detected. Please solve the CAPTCHA in the Chrome window. The browser will automatically detect when you're done and resume.",
|
|
382
902
|
bringToFront: true,
|
|
383
903
|
});
|
|
384
|
-
const newUrl =
|
|
385
|
-
const newTitle = await
|
|
904
|
+
const newUrl = await getCurrentUrl(cdp, context.signal);
|
|
905
|
+
const newTitle = await getPageTitle(cdp, context.signal);
|
|
386
906
|
lines.push("");
|
|
387
907
|
lines.push(
|
|
388
908
|
`CAPTCHA solved by user. Current page: ${newTitle} (${newUrl})`,
|
|
389
909
|
);
|
|
390
910
|
|
|
391
911
|
// Re-check for auth challenges - the page behind the CAPTCHA may have a login form
|
|
392
|
-
const postCaptchaAuth = await detectAuthChallenge(
|
|
912
|
+
const postCaptchaAuth = await detectAuthChallenge(
|
|
913
|
+
cdp,
|
|
914
|
+
context.signal,
|
|
915
|
+
);
|
|
393
916
|
if (postCaptchaAuth) {
|
|
394
917
|
lines.push("");
|
|
395
918
|
lines.push(formatAuthChallenge(postCaptchaAuth));
|
|
@@ -448,7 +971,7 @@ export async function executeBrowserNavigate(
|
|
|
448
971
|
|
|
449
972
|
return { content: lines.join("\n"), isError: false };
|
|
450
973
|
} catch (err) {
|
|
451
|
-
// Best-effort cleanup of route handler on error
|
|
974
|
+
// Best-effort cleanup of route handler on error (local path only)
|
|
452
975
|
if (routeHandler) {
|
|
453
976
|
try {
|
|
454
977
|
const page = await browserManager.getOrCreateSessionPage(
|
|
@@ -461,8 +984,8 @@ export async function executeBrowserNavigate(
|
|
|
461
984
|
}
|
|
462
985
|
|
|
463
986
|
// If the route handler blocked a redirect to a private network address,
|
|
464
|
-
//
|
|
465
|
-
// raw
|
|
987
|
+
// Page.navigate throws. Return the clear security message instead of
|
|
988
|
+
// the raw underlying error (which could leak credentials from the URL).
|
|
466
989
|
if (blockedUrl) {
|
|
467
990
|
return {
|
|
468
991
|
content: `Error: Navigation blocked. A request targeted a local/private network address (${blockedUrl}). Set allow_private_network=true if you explicitly need it.`,
|
|
@@ -470,9 +993,16 @@ export async function executeBrowserNavigate(
|
|
|
470
993
|
};
|
|
471
994
|
}
|
|
472
995
|
|
|
996
|
+
const diagnosticMessage = formatCdpSendDiagnostics(err, browserMode);
|
|
997
|
+
if (diagnosticMessage) {
|
|
998
|
+
return { content: diagnosticMessage, isError: true };
|
|
999
|
+
}
|
|
1000
|
+
|
|
473
1001
|
const msg = err instanceof Error ? err.message : String(err);
|
|
474
1002
|
log.error({ err, url: safeRequestedUrl }, "Navigation failed");
|
|
475
1003
|
return { content: `Error: Navigation failed: ${msg}`, isError: true };
|
|
1004
|
+
} finally {
|
|
1005
|
+
cdp.dispose();
|
|
476
1006
|
}
|
|
477
1007
|
}
|
|
478
1008
|
|
|
@@ -482,79 +1012,49 @@ export async function executeBrowserSnapshot(
|
|
|
482
1012
|
_input: Record<string, unknown>,
|
|
483
1013
|
context: ToolContext,
|
|
484
1014
|
): Promise<ToolExecutionResult> {
|
|
1015
|
+
const acquired = acquireCdpClientWithMode(_input, context);
|
|
1016
|
+
if (acquired.errorResult) return acquired.errorResult;
|
|
1017
|
+
const { cdp, browserMode } = acquired;
|
|
1018
|
+
|
|
485
1019
|
try {
|
|
486
|
-
const
|
|
487
|
-
|
|
1020
|
+
const currentUrl = await getCurrentUrl(cdp, context.signal);
|
|
1021
|
+
const title = await getPageTitle(cdp, context.signal);
|
|
1022
|
+
|
|
1023
|
+
// Pull the full accessibility tree via CDP and fold it into typed
|
|
1024
|
+
// interactive elements + an `eid → backendNodeId` map. Interaction
|
|
1025
|
+
// tools (click, hover, type, …) resolve element_id against this map
|
|
1026
|
+
// and jump straight to CDP DOM commands without another round-trip
|
|
1027
|
+
// through any selector engine.
|
|
1028
|
+
await cdp.send("Accessibility.enable", {}, context.signal);
|
|
1029
|
+
const rawTree = await cdp.send(
|
|
1030
|
+
"Accessibility.getFullAXTree",
|
|
1031
|
+
{},
|
|
1032
|
+
context.signal,
|
|
488
1033
|
);
|
|
489
|
-
const
|
|
490
|
-
const title = await page.title();
|
|
491
|
-
|
|
492
|
-
const elements = (await page.evaluate(`
|
|
493
|
-
(() => {
|
|
494
|
-
const SELECTOR = ${JSON.stringify(INTERACTIVE_SELECTOR)};
|
|
495
|
-
const MAX = ${MAX_SNAPSHOT_ELEMENTS};
|
|
496
|
-
// Clear stale eid attributes from previous snapshots
|
|
497
|
-
document.querySelectorAll('[data-vellum-eid]').forEach(el => el.removeAttribute('data-vellum-eid'));
|
|
498
|
-
const els = Array.from(document.querySelectorAll(SELECTOR));
|
|
499
|
-
const visible = els.filter(el => {
|
|
500
|
-
const rect = el.getBoundingClientRect();
|
|
501
|
-
return rect.width > 0 && rect.height > 0;
|
|
502
|
-
});
|
|
503
|
-
return visible.slice(0, MAX).map((el, i) => {
|
|
504
|
-
const eid = 'e' + (i + 1);
|
|
505
|
-
el.setAttribute('data-vellum-eid', eid);
|
|
506
|
-
const tag = el.tagName.toLowerCase();
|
|
507
|
-
const attrs = {};
|
|
508
|
-
for (const attr of ['type', 'name', 'placeholder', 'href', 'value', 'role', 'aria-label', 'id']) {
|
|
509
|
-
if (el.hasAttribute(attr)) attrs[attr] = el.getAttribute(attr);
|
|
510
|
-
}
|
|
511
|
-
const text = (el.textContent || '').trim().slice(0, 80);
|
|
512
|
-
return { eid, tag, attrs, text };
|
|
513
|
-
});
|
|
514
|
-
})()
|
|
515
|
-
`)) as SnapshotElement[];
|
|
516
|
-
|
|
517
|
-
// Build and store selector map
|
|
518
|
-
const selectorMap = new Map<string, string>();
|
|
519
|
-
for (const el of elements) {
|
|
520
|
-
selectorMap.set(el.eid, `[data-vellum-eid="${el.eid}"]`);
|
|
521
|
-
}
|
|
522
|
-
browserManager.storeSnapshotMap(context.conversationId, selectorMap);
|
|
523
|
-
|
|
524
|
-
// Format output
|
|
525
|
-
const lines: string[] = [
|
|
526
|
-
`URL: ${currentUrl}`,
|
|
527
|
-
`Title: ${title || "(none)"}`,
|
|
528
|
-
"",
|
|
529
|
-
];
|
|
1034
|
+
const { elements, selectorMap: backendNodeMap } = transformAxTree(rawTree);
|
|
530
1035
|
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
let desc = `<${el.tag}`;
|
|
536
|
-
for (const [key, val] of Object.entries(el.attrs)) {
|
|
537
|
-
desc += ` ${key}="${val}"`;
|
|
538
|
-
}
|
|
539
|
-
desc += ">";
|
|
540
|
-
if (el.text) {
|
|
541
|
-
desc += ` ${el.text}`;
|
|
542
|
-
}
|
|
543
|
-
lines.push(`[${el.eid}] ${desc}`);
|
|
544
|
-
}
|
|
545
|
-
lines.push("");
|
|
546
|
-
lines.push(
|
|
547
|
-
`${elements.length} interactive element${
|
|
548
|
-
elements.length === 1 ? "" : "s"
|
|
549
|
-
} found.`,
|
|
550
|
-
);
|
|
551
|
-
}
|
|
1036
|
+
browserManager.storeSnapshotBackendNodeMap(
|
|
1037
|
+
context.conversationId,
|
|
1038
|
+
backendNodeMap,
|
|
1039
|
+
);
|
|
552
1040
|
|
|
553
|
-
return {
|
|
1041
|
+
return {
|
|
1042
|
+
content: formatAxSnapshot(
|
|
1043
|
+
{ elements, selectorMap: backendNodeMap },
|
|
1044
|
+
{ url: currentUrl, title },
|
|
1045
|
+
),
|
|
1046
|
+
isError: false,
|
|
1047
|
+
};
|
|
554
1048
|
} catch (err) {
|
|
1049
|
+
const diagnosticMessage = formatCdpSendDiagnostics(err, browserMode);
|
|
1050
|
+
if (diagnosticMessage) {
|
|
1051
|
+
return { content: diagnosticMessage, isError: true };
|
|
1052
|
+
}
|
|
555
1053
|
const msg = err instanceof Error ? err.message : String(err);
|
|
556
1054
|
log.error({ err }, "Snapshot failed");
|
|
557
1055
|
return { content: `Error: Snapshot failed: ${msg}`, isError: true };
|
|
1056
|
+
} finally {
|
|
1057
|
+
cdp.dispose();
|
|
558
1058
|
}
|
|
559
1059
|
}
|
|
560
1060
|
|
|
@@ -564,17 +1064,17 @@ export async function executeBrowserScreenshot(
|
|
|
564
1064
|
input: Record<string, unknown>,
|
|
565
1065
|
context: ToolContext,
|
|
566
1066
|
): Promise<ToolExecutionResult> {
|
|
1067
|
+
const acquired = acquireCdpClientWithMode(input, context);
|
|
1068
|
+
if (acquired.errorResult) return acquired.errorResult;
|
|
1069
|
+
const { cdp, browserMode } = acquired;
|
|
567
1070
|
const fullPage = input.full_page === true;
|
|
568
1071
|
|
|
569
1072
|
try {
|
|
570
|
-
const
|
|
571
|
-
|
|
1073
|
+
const buffer = await captureScreenshotJpeg(
|
|
1074
|
+
cdp,
|
|
1075
|
+
{ quality: 80, fullPage },
|
|
1076
|
+
context.signal,
|
|
572
1077
|
);
|
|
573
|
-
const buffer = await page.screenshot({
|
|
574
|
-
type: "jpeg",
|
|
575
|
-
quality: 80,
|
|
576
|
-
fullPage,
|
|
577
|
-
});
|
|
578
1078
|
const base64Data = buffer.toString("base64");
|
|
579
1079
|
|
|
580
1080
|
const imageBlock: ImageContent = {
|
|
@@ -594,9 +1094,118 @@ export async function executeBrowserScreenshot(
|
|
|
594
1094
|
contentBlocks: [imageBlock],
|
|
595
1095
|
};
|
|
596
1096
|
} catch (err) {
|
|
1097
|
+
const diagnosticMessage = formatCdpSendDiagnostics(err, browserMode);
|
|
1098
|
+
if (diagnosticMessage) {
|
|
1099
|
+
return { content: diagnosticMessage, isError: true };
|
|
1100
|
+
}
|
|
597
1101
|
const msg = err instanceof Error ? err.message : String(err);
|
|
598
1102
|
log.error({ err }, "Screenshot failed");
|
|
599
1103
|
return { content: `Error: Screenshot failed: ${msg}`, isError: true };
|
|
1104
|
+
} finally {
|
|
1105
|
+
cdp.dispose();
|
|
1106
|
+
}
|
|
1107
|
+
}
|
|
1108
|
+
|
|
1109
|
+
// ── browser_attach ───────────────────────────────────────────────────
|
|
1110
|
+
|
|
1111
|
+
export async function executeBrowserAttach(
|
|
1112
|
+
_input: Record<string, unknown>,
|
|
1113
|
+
context: ToolContext,
|
|
1114
|
+
): Promise<ToolExecutionResult> {
|
|
1115
|
+
const acquired = acquireCdpClientWithMode(_input, context);
|
|
1116
|
+
if (acquired.errorResult) return acquired.errorResult;
|
|
1117
|
+
const cdp = acquired.cdp;
|
|
1118
|
+
try {
|
|
1119
|
+
if (cdp.kind === "extension") {
|
|
1120
|
+
// Extension path: explicitly attach the debugger via a synthetic
|
|
1121
|
+
// Vellum.attach command so the debugging session is established
|
|
1122
|
+
// before any navigation or interaction.
|
|
1123
|
+
const result = await cdp.send<{ attached?: boolean; target?: unknown }>(
|
|
1124
|
+
"Vellum.attach",
|
|
1125
|
+
{},
|
|
1126
|
+
context.signal,
|
|
1127
|
+
);
|
|
1128
|
+
log.debug(
|
|
1129
|
+
{ conversationId: context.conversationId, result },
|
|
1130
|
+
"Browser debugger attached (extension)",
|
|
1131
|
+
);
|
|
1132
|
+
return {
|
|
1133
|
+
content: "Browser debugger attached.",
|
|
1134
|
+
isError: false,
|
|
1135
|
+
};
|
|
1136
|
+
}
|
|
1137
|
+
|
|
1138
|
+
// Non-extension backends (local / cdp-inspect): explicit attach is
|
|
1139
|
+
// not required — the backend manages its own connection lifecycle.
|
|
1140
|
+
// Return a deterministic no-op success.
|
|
1141
|
+
return {
|
|
1142
|
+
content:
|
|
1143
|
+
"Browser session ready. (Explicit attach is not required on this backend.)",
|
|
1144
|
+
isError: false,
|
|
1145
|
+
};
|
|
1146
|
+
} catch (err) {
|
|
1147
|
+
const diagnosticMessage = formatCdpSendDiagnostics(
|
|
1148
|
+
err,
|
|
1149
|
+
acquired.browserMode,
|
|
1150
|
+
);
|
|
1151
|
+
if (diagnosticMessage) {
|
|
1152
|
+
return { content: diagnosticMessage, isError: true };
|
|
1153
|
+
}
|
|
1154
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1155
|
+
log.error({ err }, "Attach failed");
|
|
1156
|
+
return { content: `Error: Attach failed: ${msg}`, isError: true };
|
|
1157
|
+
} finally {
|
|
1158
|
+
cdp.dispose();
|
|
1159
|
+
}
|
|
1160
|
+
}
|
|
1161
|
+
|
|
1162
|
+
// ── browser_detach ──────────────────────────────────────────────────
|
|
1163
|
+
|
|
1164
|
+
export async function executeBrowserDetach(
|
|
1165
|
+
_input: Record<string, unknown>,
|
|
1166
|
+
context: ToolContext,
|
|
1167
|
+
): Promise<ToolExecutionResult> {
|
|
1168
|
+
const acquired = acquireCdpClientWithMode(_input, context);
|
|
1169
|
+
if (acquired.errorResult) return acquired.errorResult;
|
|
1170
|
+
const cdp = acquired.cdp;
|
|
1171
|
+
try {
|
|
1172
|
+
if (cdp.kind === "extension") {
|
|
1173
|
+
// Extension path: explicitly detach the debugger via a synthetic
|
|
1174
|
+
// Vellum.detach command so the Chrome debugging banner clears.
|
|
1175
|
+
const result = await cdp.send<{ detached?: boolean; target?: unknown }>(
|
|
1176
|
+
"Vellum.detach",
|
|
1177
|
+
{},
|
|
1178
|
+
context.signal,
|
|
1179
|
+
);
|
|
1180
|
+
log.debug(
|
|
1181
|
+
{ conversationId: context.conversationId, result },
|
|
1182
|
+
"Browser debugger detached (extension)",
|
|
1183
|
+
);
|
|
1184
|
+
}
|
|
1185
|
+
|
|
1186
|
+
return {
|
|
1187
|
+
content: "Browser debugger detached and snapshot state cleared.",
|
|
1188
|
+
isError: false,
|
|
1189
|
+
};
|
|
1190
|
+
} catch (err) {
|
|
1191
|
+
const diagnosticMessage = formatCdpSendDiagnostics(
|
|
1192
|
+
err,
|
|
1193
|
+
acquired.browserMode,
|
|
1194
|
+
);
|
|
1195
|
+
if (diagnosticMessage) {
|
|
1196
|
+
return { content: diagnosticMessage, isError: true };
|
|
1197
|
+
}
|
|
1198
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1199
|
+
log.error({ err }, "Detach failed");
|
|
1200
|
+
return { content: `Error: Detach failed: ${msg}`, isError: true };
|
|
1201
|
+
} finally {
|
|
1202
|
+
// Always reset conversation-scoped browser state, even if the
|
|
1203
|
+
// Vellum.detach round-trip failed (target gone, transport dropped).
|
|
1204
|
+
// browser_detach is the user's recovery path — leaving a stale
|
|
1205
|
+
// sticky backend or snapshot map behind would defeat its purpose.
|
|
1206
|
+
browserManager.clearSnapshotBackendNodeMap(context.conversationId);
|
|
1207
|
+
browserManager.clearPreferredBackendKind(context.conversationId);
|
|
1208
|
+
cdp.dispose();
|
|
600
1209
|
}
|
|
601
1210
|
}
|
|
602
1211
|
|
|
@@ -606,29 +1215,62 @@ export async function executeBrowserClose(
|
|
|
606
1215
|
input: Record<string, unknown>,
|
|
607
1216
|
context: ToolContext,
|
|
608
1217
|
): Promise<ToolExecutionResult> {
|
|
1218
|
+
const acquired = acquireCdpClientWithMode(input, context);
|
|
1219
|
+
if (acquired.errorResult) return acquired.errorResult;
|
|
1220
|
+
const cdp = acquired.cdp;
|
|
609
1221
|
try {
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
1222
|
+
if (cdp.kind === "local") {
|
|
1223
|
+
// Local/sacrificial-profile path: tear down the Playwright page,
|
|
1224
|
+
// screencast, and associated CDP state for this conversation.
|
|
1225
|
+
const sender = getSender(context.conversationId);
|
|
1226
|
+
if (sender) {
|
|
1227
|
+
await stopBrowserScreencast(context.conversationId);
|
|
1228
|
+
}
|
|
614
1229
|
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
1230
|
+
if (input.close_all_pages === true) {
|
|
1231
|
+
await stopAllScreencasts();
|
|
1232
|
+
await browserManager.closeAllPages();
|
|
1233
|
+
return {
|
|
1234
|
+
content: "All browser pages and context closed.",
|
|
1235
|
+
isError: false,
|
|
1236
|
+
};
|
|
1237
|
+
}
|
|
1238
|
+
await browserManager.closeSessionPage(context.conversationId);
|
|
618
1239
|
return {
|
|
619
|
-
content: "
|
|
1240
|
+
content: "Browser page closed for this conversation.",
|
|
620
1241
|
isError: false,
|
|
621
1242
|
};
|
|
622
1243
|
}
|
|
623
|
-
|
|
1244
|
+
|
|
1245
|
+
// Extension path: the user owns their Chrome tab — we must not
|
|
1246
|
+
// close it. Detach the debugger (so the Chrome debugging banner
|
|
1247
|
+
// clears promptly) and drop the cached snapshot state so stale
|
|
1248
|
+
// eids from prior snapshots cannot be resolved by later tool calls.
|
|
1249
|
+
try {
|
|
1250
|
+
await cdp.send("Vellum.detach", {}, context.signal);
|
|
1251
|
+
} catch {
|
|
1252
|
+
// Tolerate detach failures (already detached, tab closed, etc.)
|
|
1253
|
+
}
|
|
1254
|
+
browserManager.clearSnapshotBackendNodeMap(context.conversationId);
|
|
1255
|
+
browserManager.clearPreferredBackendKind(context.conversationId);
|
|
624
1256
|
return {
|
|
625
|
-
content:
|
|
1257
|
+
content:
|
|
1258
|
+
"Browser session cleared. (Your Chrome tab was not closed — close it yourself if desired.)",
|
|
626
1259
|
isError: false,
|
|
627
1260
|
};
|
|
628
1261
|
} catch (err) {
|
|
1262
|
+
const diagnosticMessage = formatCdpSendDiagnostics(
|
|
1263
|
+
err,
|
|
1264
|
+
acquired.browserMode,
|
|
1265
|
+
);
|
|
1266
|
+
if (diagnosticMessage) {
|
|
1267
|
+
return { content: diagnosticMessage, isError: true };
|
|
1268
|
+
}
|
|
629
1269
|
const msg = err instanceof Error ? err.message : String(err);
|
|
630
1270
|
log.error({ err }, "Close failed");
|
|
631
1271
|
return { content: `Error: Close failed: ${msg}`, isError: true };
|
|
1272
|
+
} finally {
|
|
1273
|
+
cdp.dispose();
|
|
632
1274
|
}
|
|
633
1275
|
}
|
|
634
1276
|
|
|
@@ -638,32 +1280,115 @@ export async function executeBrowserClick(
|
|
|
638
1280
|
input: Record<string, unknown>,
|
|
639
1281
|
context: ToolContext,
|
|
640
1282
|
): Promise<ToolExecutionResult> {
|
|
641
|
-
const {
|
|
1283
|
+
const { resolved, error } = resolveElement(context.conversationId, input);
|
|
642
1284
|
if (error) return { content: error, isError: true };
|
|
643
1285
|
|
|
644
|
-
const
|
|
645
|
-
|
|
646
|
-
|
|
1286
|
+
const acquired = acquireCdpClientWithMode(input, context);
|
|
1287
|
+
if (acquired.errorResult) return acquired.errorResult;
|
|
1288
|
+
const cdp = acquired.cdp;
|
|
647
1289
|
try {
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
1290
|
+
let backendNodeId: number;
|
|
1291
|
+
if (resolved!.kind === "backend") {
|
|
1292
|
+
backendNodeId = resolved!.backendNodeId;
|
|
1293
|
+
} else {
|
|
1294
|
+
// Wait until the selector matches a visible element. Mirrors
|
|
1295
|
+
// Playwright's `page.click(selector, { timeout })` semantics
|
|
1296
|
+
// and lets click work on async-hydrated pages where the
|
|
1297
|
+
// target may not yet exist when the tool is invoked.
|
|
1298
|
+
// cdpWaitForSelector returns the backendNodeId so we don't
|
|
1299
|
+
// need a separate querySelectorBackendNodeId round-trip.
|
|
1300
|
+
backendNodeId = await cdpWaitForSelector(
|
|
1301
|
+
cdp,
|
|
1302
|
+
resolved!.selector,
|
|
1303
|
+
ACTION_TIMEOUT_MS,
|
|
1304
|
+
context.signal,
|
|
1305
|
+
);
|
|
1306
|
+
}
|
|
1307
|
+
await scrollIntoViewIfNeeded(cdp, backendNodeId, context.signal);
|
|
1308
|
+
const point = await getCenterPoint(cdp, backendNodeId, context.signal);
|
|
1309
|
+
await dispatchClickAt(cdp, point, context.signal);
|
|
1310
|
+
const desc =
|
|
1311
|
+
resolved!.kind === "backend"
|
|
1312
|
+
? `eid=${resolved!.eid}`
|
|
1313
|
+
: resolved!.selector;
|
|
1314
|
+
return { content: `Clicked element: ${desc}`, isError: false };
|
|
653
1315
|
} catch (err) {
|
|
1316
|
+
const diagnosticMessage = formatCdpSendDiagnostics(
|
|
1317
|
+
err,
|
|
1318
|
+
acquired.browserMode,
|
|
1319
|
+
);
|
|
1320
|
+
if (diagnosticMessage) {
|
|
1321
|
+
return { content: diagnosticMessage, isError: true };
|
|
1322
|
+
}
|
|
654
1323
|
const msg = err instanceof Error ? err.message : String(err);
|
|
655
|
-
log.error({ err
|
|
1324
|
+
log.error({ err }, "Click failed");
|
|
656
1325
|
return { content: `Error: Click failed: ${msg}`, isError: true };
|
|
1326
|
+
} finally {
|
|
1327
|
+
cdp.dispose();
|
|
657
1328
|
}
|
|
658
1329
|
}
|
|
659
1330
|
|
|
1331
|
+
// ── Shared input helpers ─────────────────────────────────────────────
|
|
1332
|
+
|
|
1333
|
+
/**
|
|
1334
|
+
* Focus an element, clear its existing value (handling both
|
|
1335
|
+
* `<input>`/`<textarea>` and `contentEditable` targets), re-focus
|
|
1336
|
+
* (sites sometimes blur on a programmatic value reset), and insert
|
|
1337
|
+
* the requested text via `Input.insertText`.
|
|
1338
|
+
*
|
|
1339
|
+
* Used by both `executeBrowserType` and `executeBrowserFillCredential`
|
|
1340
|
+
* so credential fills cannot append to autofilled / pre-populated
|
|
1341
|
+
* fields — appending would leak the existing value into the broker
|
|
1342
|
+
* payload and corrupt the resulting password.
|
|
1343
|
+
*/
|
|
1344
|
+
async function clearAndInsertText(
|
|
1345
|
+
cdp: CdpClient,
|
|
1346
|
+
backendNodeId: number,
|
|
1347
|
+
value: string,
|
|
1348
|
+
signal?: AbortSignal,
|
|
1349
|
+
): Promise<void> {
|
|
1350
|
+
await focusElement(cdp, backendNodeId, signal);
|
|
1351
|
+
|
|
1352
|
+
// Resolve the node to a Runtime.RemoteObject so we can invoke a
|
|
1353
|
+
// function on the element itself via Runtime.callFunctionOn. This
|
|
1354
|
+
// is more reliable than a keyboard select-all + delete sequence
|
|
1355
|
+
// across input, textarea, and contenteditable targets.
|
|
1356
|
+
const { object } = await cdp.send<{ object: { objectId: string } }>(
|
|
1357
|
+
"DOM.resolveNode",
|
|
1358
|
+
{ backendNodeId },
|
|
1359
|
+
signal,
|
|
1360
|
+
);
|
|
1361
|
+
await cdp.send(
|
|
1362
|
+
"Runtime.callFunctionOn",
|
|
1363
|
+
{
|
|
1364
|
+
objectId: object.objectId,
|
|
1365
|
+
functionDeclaration: `function() {
|
|
1366
|
+
if (typeof this.value === "string") {
|
|
1367
|
+
this.value = "";
|
|
1368
|
+
} else if (this.isContentEditable) {
|
|
1369
|
+
this.textContent = "";
|
|
1370
|
+
}
|
|
1371
|
+
this.dispatchEvent(new Event("input", { bubbles: true }));
|
|
1372
|
+
}`,
|
|
1373
|
+
arguments: [],
|
|
1374
|
+
},
|
|
1375
|
+
signal,
|
|
1376
|
+
);
|
|
1377
|
+
|
|
1378
|
+
// Re-focus after clearing — some sites move focus when the value
|
|
1379
|
+
// property is reassigned programmatically.
|
|
1380
|
+
await focusElement(cdp, backendNodeId, signal);
|
|
1381
|
+
|
|
1382
|
+
await dispatchInsertText(cdp, value, signal);
|
|
1383
|
+
}
|
|
1384
|
+
|
|
660
1385
|
// ── browser_type ─────────────────────────────────────────────────────
|
|
661
1386
|
|
|
662
1387
|
export async function executeBrowserType(
|
|
663
1388
|
input: Record<string, unknown>,
|
|
664
1389
|
context: ToolContext,
|
|
665
1390
|
): Promise<ToolExecutionResult> {
|
|
666
|
-
const {
|
|
1391
|
+
const { resolved, error } = resolveElement(context.conversationId, input);
|
|
667
1392
|
if (error) return { content: error, isError: true };
|
|
668
1393
|
|
|
669
1394
|
const text = typeof input.text === "string" ? input.text : "";
|
|
@@ -674,40 +1399,54 @@ export async function executeBrowserType(
|
|
|
674
1399
|
const clearFirst = input.clear_first !== false; // default true
|
|
675
1400
|
const pressEnter = input.press_enter === true;
|
|
676
1401
|
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
1402
|
+
const targetDescription =
|
|
1403
|
+
resolved!.kind === "backend"
|
|
1404
|
+
? `element_id "${resolved!.eid}"`
|
|
1405
|
+
: resolved!.selector;
|
|
681
1406
|
|
|
682
|
-
|
|
683
|
-
|
|
1407
|
+
const acquired = acquireCdpClientWithMode(input, context);
|
|
1408
|
+
if (acquired.errorResult) return acquired.errorResult;
|
|
1409
|
+
const cdp = acquired.cdp;
|
|
1410
|
+
try {
|
|
1411
|
+
let backendNodeId: number;
|
|
1412
|
+
if (resolved!.kind === "backend") {
|
|
1413
|
+
backendNodeId = resolved!.backendNodeId;
|
|
1414
|
+
} else {
|
|
1415
|
+
backendNodeId = await querySelectorBackendNodeId(
|
|
1416
|
+
cdp,
|
|
1417
|
+
resolved!.selector,
|
|
1418
|
+
context.signal,
|
|
1419
|
+
);
|
|
1420
|
+
}
|
|
684
1421
|
|
|
685
1422
|
if (clearFirst) {
|
|
686
|
-
await
|
|
1423
|
+
await clearAndInsertText(cdp, backendNodeId, text, context.signal);
|
|
687
1424
|
} else {
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
// visual line breaks from <br> and block elements, unlike textContent).
|
|
691
|
-
const currentValue = (await page.evaluate(
|
|
692
|
-
`(() => { const el = document.querySelector(${JSON.stringify(
|
|
693
|
-
selector!,
|
|
694
|
-
)}); if (!el) return ''; if (typeof el.value === 'string') return el.value; return el.innerText ?? ''; })()`,
|
|
695
|
-
)) as string;
|
|
696
|
-
await page.fill(selector!, currentValue + text, { timeout: fillTimeout });
|
|
1425
|
+
await focusElement(cdp, backendNodeId, context.signal);
|
|
1426
|
+
await dispatchInsertText(cdp, text, context.signal);
|
|
697
1427
|
}
|
|
698
1428
|
|
|
699
1429
|
if (pressEnter) {
|
|
700
|
-
await
|
|
1430
|
+
await dispatchKeyPress(cdp, "Enter", context.signal);
|
|
701
1431
|
}
|
|
702
1432
|
|
|
703
|
-
const lines = [`Typed into element: ${
|
|
1433
|
+
const lines = [`Typed into element: ${targetDescription}`];
|
|
704
1434
|
if (clearFirst) lines.push("(cleared existing content first)");
|
|
705
1435
|
if (pressEnter) lines.push("(pressed Enter after typing)");
|
|
706
1436
|
return { content: lines.join("\n"), isError: false };
|
|
707
1437
|
} catch (err) {
|
|
1438
|
+
const diagnosticMessage = formatCdpSendDiagnostics(
|
|
1439
|
+
err,
|
|
1440
|
+
acquired.browserMode,
|
|
1441
|
+
);
|
|
1442
|
+
if (diagnosticMessage) {
|
|
1443
|
+
return { content: diagnosticMessage, isError: true };
|
|
1444
|
+
}
|
|
708
1445
|
const msg = err instanceof Error ? err.message : String(err);
|
|
709
|
-
log.error({ err,
|
|
1446
|
+
log.error({ err, target: targetDescription }, "Type failed");
|
|
710
1447
|
return { content: `Error: Type failed: ${msg}`, isError: true };
|
|
1448
|
+
} finally {
|
|
1449
|
+
cdp.dispose();
|
|
711
1450
|
}
|
|
712
1451
|
}
|
|
713
1452
|
|
|
@@ -722,39 +1461,65 @@ export async function executeBrowserPressKey(
|
|
|
722
1461
|
return { content: "Error: key is required.", isError: true };
|
|
723
1462
|
}
|
|
724
1463
|
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
1464
|
+
const elementId =
|
|
1465
|
+
typeof input.element_id === "string" ? input.element_id : null;
|
|
1466
|
+
const rawSelector =
|
|
1467
|
+
typeof input.selector === "string" ? input.selector : null;
|
|
1468
|
+
const hasTarget = elementId !== null || rawSelector !== null;
|
|
1469
|
+
|
|
1470
|
+
let targetDescription: string | null = null;
|
|
1471
|
+
let resolved: ResolvedElement | null = null;
|
|
1472
|
+
if (hasTarget) {
|
|
1473
|
+
const res = resolveElement(context.conversationId, input);
|
|
1474
|
+
if (res.error) {
|
|
1475
|
+
return { content: res.error, isError: true };
|
|
1476
|
+
}
|
|
1477
|
+
resolved = res.resolved;
|
|
1478
|
+
targetDescription =
|
|
1479
|
+
resolved!.kind === "backend"
|
|
1480
|
+
? `element_id "${resolved!.eid}"`
|
|
1481
|
+
: resolved!.selector;
|
|
1482
|
+
}
|
|
735
1483
|
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
1484
|
+
const acquired = acquireCdpClientWithMode(input, context);
|
|
1485
|
+
if (acquired.errorResult) return acquired.errorResult;
|
|
1486
|
+
const cdp = acquired.cdp;
|
|
1487
|
+
try {
|
|
1488
|
+
if (resolved) {
|
|
1489
|
+
let backendNodeId: number;
|
|
1490
|
+
if (resolved.kind === "backend") {
|
|
1491
|
+
backendNodeId = resolved.backendNodeId;
|
|
1492
|
+
} else {
|
|
1493
|
+
backendNodeId = await querySelectorBackendNodeId(
|
|
1494
|
+
cdp,
|
|
1495
|
+
resolved.selector,
|
|
1496
|
+
context.signal,
|
|
1497
|
+
);
|
|
743
1498
|
}
|
|
744
|
-
await
|
|
1499
|
+
await focusElement(cdp, backendNodeId, context.signal);
|
|
1500
|
+
await dispatchKeyPress(cdp, key, context.signal);
|
|
745
1501
|
return {
|
|
746
|
-
content: `Pressed "${key}" on element: ${
|
|
1502
|
+
content: `Pressed "${key}" on element: ${targetDescription}`,
|
|
747
1503
|
isError: false,
|
|
748
1504
|
};
|
|
749
1505
|
}
|
|
750
1506
|
|
|
751
|
-
// No target -> press key on the
|
|
752
|
-
await
|
|
1507
|
+
// No target -> press key on the currently focused element
|
|
1508
|
+
await dispatchKeyPress(cdp, key, context.signal);
|
|
753
1509
|
return { content: `Pressed "${key}"`, isError: false };
|
|
754
1510
|
} catch (err) {
|
|
1511
|
+
const diagnosticMessage = formatCdpSendDiagnostics(
|
|
1512
|
+
err,
|
|
1513
|
+
acquired.browserMode,
|
|
1514
|
+
);
|
|
1515
|
+
if (diagnosticMessage) {
|
|
1516
|
+
return { content: diagnosticMessage, isError: true };
|
|
1517
|
+
}
|
|
755
1518
|
const msg = err instanceof Error ? err.message : String(err);
|
|
756
1519
|
log.error({ err, key }, "Press key failed");
|
|
757
1520
|
return { content: `Error: Press key failed: ${msg}`, isError: true };
|
|
1521
|
+
} finally {
|
|
1522
|
+
cdp.dispose();
|
|
758
1523
|
}
|
|
759
1524
|
}
|
|
760
1525
|
|
|
@@ -776,35 +1541,58 @@ export async function executeBrowserScroll(
|
|
|
776
1541
|
const amount =
|
|
777
1542
|
typeof input.amount === "number" ? Math.abs(input.amount) : 500;
|
|
778
1543
|
|
|
1544
|
+
let deltaX = 0;
|
|
1545
|
+
let deltaY = 0;
|
|
1546
|
+
switch (direction) {
|
|
1547
|
+
case "up":
|
|
1548
|
+
deltaY = -amount;
|
|
1549
|
+
break;
|
|
1550
|
+
case "down":
|
|
1551
|
+
deltaY = amount;
|
|
1552
|
+
break;
|
|
1553
|
+
case "left":
|
|
1554
|
+
deltaX = -amount;
|
|
1555
|
+
break;
|
|
1556
|
+
case "right":
|
|
1557
|
+
deltaX = amount;
|
|
1558
|
+
break;
|
|
1559
|
+
}
|
|
1560
|
+
|
|
1561
|
+
const acquired = acquireCdpClientWithMode(input, context);
|
|
1562
|
+
if (acquired.errorResult) return acquired.errorResult;
|
|
1563
|
+
const cdp = acquired.cdp;
|
|
779
1564
|
try {
|
|
780
|
-
|
|
781
|
-
|
|
1565
|
+
// Fetch viewport dimensions so we can dispatch the wheel event at
|
|
1566
|
+
// the viewport center — scrolling from (0, 0) misses sticky
|
|
1567
|
+
// headers and overflow containers on many sites.
|
|
1568
|
+
const { w, h } = await evaluateExpression<{ w: number; h: number }>(
|
|
1569
|
+
cdp,
|
|
1570
|
+
"({ w: window.innerWidth, h: window.innerHeight })",
|
|
1571
|
+
{},
|
|
1572
|
+
context.signal,
|
|
782
1573
|
);
|
|
783
1574
|
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
case "down":
|
|
791
|
-
deltaY = amount;
|
|
792
|
-
break;
|
|
793
|
-
case "left":
|
|
794
|
-
deltaX = -amount;
|
|
795
|
-
break;
|
|
796
|
-
case "right":
|
|
797
|
-
deltaX = amount;
|
|
798
|
-
break;
|
|
799
|
-
}
|
|
800
|
-
|
|
801
|
-
await page.mouse.wheel(deltaX, deltaY);
|
|
1575
|
+
await dispatchWheelScroll(
|
|
1576
|
+
cdp,
|
|
1577
|
+
{ x: w / 2, y: h / 2 },
|
|
1578
|
+
{ deltaX, deltaY },
|
|
1579
|
+
context.signal,
|
|
1580
|
+
);
|
|
802
1581
|
|
|
803
1582
|
return { content: `Scrolled ${direction} by ${amount}px`, isError: false };
|
|
804
1583
|
} catch (err) {
|
|
1584
|
+
const diagnosticMessage = formatCdpSendDiagnostics(
|
|
1585
|
+
err,
|
|
1586
|
+
acquired.browserMode,
|
|
1587
|
+
);
|
|
1588
|
+
if (diagnosticMessage) {
|
|
1589
|
+
return { content: diagnosticMessage, isError: true };
|
|
1590
|
+
}
|
|
805
1591
|
const msg = err instanceof Error ? err.message : String(err);
|
|
806
1592
|
log.error({ err, direction }, "Scroll failed");
|
|
807
1593
|
return { content: `Error: Scroll failed: ${msg}`, isError: true };
|
|
1594
|
+
} finally {
|
|
1595
|
+
cdp.dispose();
|
|
808
1596
|
}
|
|
809
1597
|
}
|
|
810
1598
|
|
|
@@ -814,7 +1602,7 @@ export async function executeBrowserSelectOption(
|
|
|
814
1602
|
input: Record<string, unknown>,
|
|
815
1603
|
context: ToolContext,
|
|
816
1604
|
): Promise<ToolExecutionResult> {
|
|
817
|
-
const {
|
|
1605
|
+
const { resolved, error } = resolveElement(context.conversationId, input);
|
|
818
1606
|
if (error) return { content: error, isError: true };
|
|
819
1607
|
|
|
820
1608
|
const value = typeof input.value === "string" ? input.value : undefined;
|
|
@@ -828,32 +1616,115 @@ export async function executeBrowserSelectOption(
|
|
|
828
1616
|
};
|
|
829
1617
|
}
|
|
830
1618
|
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
1619
|
+
const targetDescription =
|
|
1620
|
+
resolved!.kind === "backend"
|
|
1621
|
+
? `element_id "${resolved!.eid}"`
|
|
1622
|
+
: resolved!.selector;
|
|
835
1623
|
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
1624
|
+
const acquired = acquireCdpClientWithMode(input, context);
|
|
1625
|
+
if (acquired.errorResult) return acquired.errorResult;
|
|
1626
|
+
const cdp = acquired.cdp;
|
|
1627
|
+
try {
|
|
1628
|
+
let backendNodeId: number;
|
|
1629
|
+
if (resolved!.kind === "backend") {
|
|
1630
|
+
backendNodeId = resolved!.backendNodeId;
|
|
1631
|
+
} else {
|
|
1632
|
+
backendNodeId = await querySelectorBackendNodeId(
|
|
1633
|
+
cdp,
|
|
1634
|
+
resolved!.selector,
|
|
1635
|
+
context.signal,
|
|
1636
|
+
);
|
|
1637
|
+
}
|
|
840
1638
|
|
|
841
|
-
|
|
1639
|
+
// CDP does not expose a native "set select value" command, so we
|
|
1640
|
+
// resolve the node to a Runtime.RemoteObject and invoke a function
|
|
1641
|
+
// on it that applies value/label/index and dispatches `input`
|
|
1642
|
+
// followed by `change` (HTML spec order — Angular's
|
|
1643
|
+
// DefaultValueAccessor listens for `input`, so missing it breaks
|
|
1644
|
+
// form bindings on Angular sites).
|
|
1645
|
+
const { object } = await cdp.send<{ object: { objectId: string } }>(
|
|
1646
|
+
"DOM.resolveNode",
|
|
1647
|
+
{ backendNodeId },
|
|
1648
|
+
context.signal,
|
|
1649
|
+
);
|
|
1650
|
+
const callResult = await cdp.send<{
|
|
1651
|
+
result?: { value?: boolean };
|
|
1652
|
+
}>(
|
|
1653
|
+
"Runtime.callFunctionOn",
|
|
1654
|
+
{
|
|
1655
|
+
objectId: object.objectId,
|
|
1656
|
+
functionDeclaration: `function(value, label, index) {
|
|
1657
|
+
let matched = false;
|
|
1658
|
+
if (value !== null && value !== undefined) {
|
|
1659
|
+
for (const opt of this.options) {
|
|
1660
|
+
if (opt.value === value) {
|
|
1661
|
+
this.value = value;
|
|
1662
|
+
matched = true;
|
|
1663
|
+
break;
|
|
1664
|
+
}
|
|
1665
|
+
}
|
|
1666
|
+
} else if (label !== null && label !== undefined) {
|
|
1667
|
+
for (const opt of this.options) {
|
|
1668
|
+
if (opt.label === label) {
|
|
1669
|
+
this.value = opt.value;
|
|
1670
|
+
matched = true;
|
|
1671
|
+
break;
|
|
1672
|
+
}
|
|
1673
|
+
}
|
|
1674
|
+
} else if (index !== null && index !== undefined) {
|
|
1675
|
+
if (index >= 0 && index < this.options.length) {
|
|
1676
|
+
this.selectedIndex = index;
|
|
1677
|
+
matched = true;
|
|
1678
|
+
}
|
|
1679
|
+
}
|
|
1680
|
+
if (matched) {
|
|
1681
|
+
this.dispatchEvent(new Event("input", { bubbles: true }));
|
|
1682
|
+
this.dispatchEvent(new Event("change", { bubbles: true }));
|
|
1683
|
+
}
|
|
1684
|
+
return matched;
|
|
1685
|
+
}`,
|
|
1686
|
+
arguments: [
|
|
1687
|
+
{ value: value ?? null },
|
|
1688
|
+
{ value: label ?? null },
|
|
1689
|
+
{ value: index ?? null },
|
|
1690
|
+
],
|
|
1691
|
+
returnByValue: true,
|
|
1692
|
+
},
|
|
1693
|
+
context.signal,
|
|
1694
|
+
);
|
|
842
1695
|
|
|
1696
|
+
const matched = callResult?.result?.value === true;
|
|
843
1697
|
const desc =
|
|
844
1698
|
value !== undefined
|
|
845
1699
|
? `value="${value}"`
|
|
846
1700
|
: label !== undefined
|
|
847
1701
|
? `label="${label}"`
|
|
848
1702
|
: `index=${index}`;
|
|
1703
|
+
|
|
1704
|
+
if (!matched) {
|
|
1705
|
+
return {
|
|
1706
|
+
content: `Error: Select option failed: no option matched ${desc} on ${targetDescription}.`,
|
|
1707
|
+
isError: true,
|
|
1708
|
+
};
|
|
1709
|
+
}
|
|
1710
|
+
|
|
849
1711
|
return {
|
|
850
|
-
content: `Selected option (${desc}) on element: ${
|
|
1712
|
+
content: `Selected option (${desc}) on element: ${targetDescription}`,
|
|
851
1713
|
isError: false,
|
|
852
1714
|
};
|
|
853
1715
|
} catch (err) {
|
|
1716
|
+
const diagnosticMessage = formatCdpSendDiagnostics(
|
|
1717
|
+
err,
|
|
1718
|
+
acquired.browserMode,
|
|
1719
|
+
);
|
|
1720
|
+
if (diagnosticMessage) {
|
|
1721
|
+
return { content: diagnosticMessage, isError: true };
|
|
1722
|
+
}
|
|
854
1723
|
const msg = err instanceof Error ? err.message : String(err);
|
|
855
|
-
log.error({ err,
|
|
1724
|
+
log.error({ err, target: targetDescription }, "Select option failed");
|
|
856
1725
|
return { content: `Error: Select option failed: ${msg}`, isError: true };
|
|
1726
|
+
} finally {
|
|
1727
|
+
cdp.dispose();
|
|
857
1728
|
}
|
|
858
1729
|
}
|
|
859
1730
|
|
|
@@ -863,20 +1734,48 @@ export async function executeBrowserHover(
|
|
|
863
1734
|
input: Record<string, unknown>,
|
|
864
1735
|
context: ToolContext,
|
|
865
1736
|
): Promise<ToolExecutionResult> {
|
|
866
|
-
const {
|
|
1737
|
+
const { resolved, error } = resolveElement(context.conversationId, input);
|
|
867
1738
|
if (error) return { content: error, isError: true };
|
|
868
1739
|
|
|
1740
|
+
const acquired = acquireCdpClientWithMode(input, context);
|
|
1741
|
+
if (acquired.errorResult) return acquired.errorResult;
|
|
1742
|
+
const cdp = acquired.cdp;
|
|
869
1743
|
try {
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
1744
|
+
let backendNodeId: number;
|
|
1745
|
+
if (resolved!.kind === "backend") {
|
|
1746
|
+
backendNodeId = resolved!.backendNodeId;
|
|
1747
|
+
} else {
|
|
1748
|
+
// Wait until the selector matches a visible element. See the
|
|
1749
|
+
// matching note in executeBrowserClick — async-hydrated pages
|
|
1750
|
+
// need this to behave like Playwright's hover-with-timeout.
|
|
1751
|
+
backendNodeId = await cdpWaitForSelector(
|
|
1752
|
+
cdp,
|
|
1753
|
+
resolved!.selector,
|
|
1754
|
+
ACTION_TIMEOUT_MS,
|
|
1755
|
+
context.signal,
|
|
1756
|
+
);
|
|
1757
|
+
}
|
|
1758
|
+
await scrollIntoViewIfNeeded(cdp, backendNodeId, context.signal);
|
|
1759
|
+
const point = await getCenterPoint(cdp, backendNodeId, context.signal);
|
|
1760
|
+
await dispatchHoverAt(cdp, point, context.signal);
|
|
1761
|
+
const desc =
|
|
1762
|
+
resolved!.kind === "backend"
|
|
1763
|
+
? `eid=${resolved!.eid}`
|
|
1764
|
+
: resolved!.selector;
|
|
1765
|
+
return { content: `Hovered element: ${desc}`, isError: false };
|
|
876
1766
|
} catch (err) {
|
|
1767
|
+
const diagnosticMessage = formatCdpSendDiagnostics(
|
|
1768
|
+
err,
|
|
1769
|
+
acquired.browserMode,
|
|
1770
|
+
);
|
|
1771
|
+
if (diagnosticMessage) {
|
|
1772
|
+
return { content: diagnosticMessage, isError: true };
|
|
1773
|
+
}
|
|
877
1774
|
const msg = err instanceof Error ? err.message : String(err);
|
|
878
|
-
log.error({ err
|
|
1775
|
+
log.error({ err }, "Hover failed");
|
|
879
1776
|
return { content: `Error: Hover failed: ${msg}`, isError: true };
|
|
1777
|
+
} finally {
|
|
1778
|
+
cdp.dispose();
|
|
880
1779
|
}
|
|
881
1780
|
}
|
|
882
1781
|
|
|
@@ -917,39 +1816,59 @@ export async function executeBrowserWaitFor(
|
|
|
917
1816
|
? Math.min(input.timeout, MAX_WAIT_MS)
|
|
918
1817
|
: MAX_WAIT_MS;
|
|
919
1818
|
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
1819
|
+
// Validate browser_mode even on the duration path so invalid values
|
|
1820
|
+
// are rejected consistently regardless of which wait mode is used.
|
|
1821
|
+
const modeResult = parseBrowserMode(input);
|
|
1822
|
+
if (!modeResult.ok) {
|
|
1823
|
+
return { content: modeResult.error, isError: true };
|
|
1824
|
+
}
|
|
1825
|
+
|
|
1826
|
+
// Duration mode has no CDP interaction — handle without acquiring
|
|
1827
|
+
// a CdpClient so the common "sleep" path stays transport-agnostic.
|
|
1828
|
+
if (duration != null) {
|
|
1829
|
+
const waitMs = Math.min(duration, MAX_WAIT_MS);
|
|
1830
|
+
await new Promise((r) => setTimeout(r, waitMs));
|
|
1831
|
+
return { content: `Waited ${waitMs}ms.`, isError: false };
|
|
1832
|
+
}
|
|
924
1833
|
|
|
1834
|
+
const acquired = acquireCdpClientWithMode(input, context);
|
|
1835
|
+
if (acquired.errorResult) return acquired.errorResult;
|
|
1836
|
+
const cdp = acquired.cdp;
|
|
1837
|
+
try {
|
|
925
1838
|
if (selector) {
|
|
926
|
-
|
|
1839
|
+
// browser_wait_for selector mode is "did this node appear at
|
|
1840
|
+
// all" — preserve the existing semantics by polling for DOM
|
|
1841
|
+
// attachment, not full visibility. Tools that need
|
|
1842
|
+
// visible-state polling (click/hover) get it via the default
|
|
1843
|
+
// state in cdpWaitForSelector.
|
|
1844
|
+
await cdpWaitForSelector(cdp, selector, timeout, context.signal, {
|
|
1845
|
+
state: "attached",
|
|
1846
|
+
});
|
|
927
1847
|
return {
|
|
928
1848
|
content: `Element matching "${selector}" appeared.`,
|
|
929
1849
|
isError: false,
|
|
930
1850
|
};
|
|
931
1851
|
}
|
|
932
1852
|
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
return {
|
|
940
|
-
content: `Text "${truncate(text, 80)}" appeared on page.`,
|
|
941
|
-
isError: false,
|
|
942
|
-
};
|
|
943
|
-
}
|
|
944
|
-
|
|
945
|
-
// duration mode (milliseconds)
|
|
946
|
-
const waitMs = Math.min(duration!, MAX_WAIT_MS);
|
|
947
|
-
await new Promise((r) => setTimeout(r, waitMs));
|
|
948
|
-
return { content: `Waited ${waitMs}ms.`, isError: false };
|
|
1853
|
+
// text mode (validated above — modeCount === 1 means text is set)
|
|
1854
|
+
await cdpWaitForText(cdp, text!, timeout, context.signal);
|
|
1855
|
+
return {
|
|
1856
|
+
content: `Text "${truncate(text!, 80)}" appeared on page.`,
|
|
1857
|
+
isError: false,
|
|
1858
|
+
};
|
|
949
1859
|
} catch (err) {
|
|
1860
|
+
const diagnosticMessage = formatCdpSendDiagnostics(
|
|
1861
|
+
err,
|
|
1862
|
+
acquired.browserMode,
|
|
1863
|
+
);
|
|
1864
|
+
if (diagnosticMessage) {
|
|
1865
|
+
return { content: diagnosticMessage, isError: true };
|
|
1866
|
+
}
|
|
950
1867
|
const msg = err instanceof Error ? err.message : String(err);
|
|
951
1868
|
log.error({ err }, "Wait failed");
|
|
952
1869
|
return { content: `Error: Wait failed: ${msg}`, isError: true };
|
|
1870
|
+
} finally {
|
|
1871
|
+
cdp.dispose();
|
|
953
1872
|
}
|
|
954
1873
|
}
|
|
955
1874
|
|
|
@@ -961,20 +1880,24 @@ export async function executeBrowserExtract(
|
|
|
961
1880
|
): Promise<ToolExecutionResult> {
|
|
962
1881
|
const includeLinks = input.include_links === true;
|
|
963
1882
|
|
|
1883
|
+
const acquired = acquireCdpClientWithMode(input, context);
|
|
1884
|
+
if (acquired.errorResult) return acquired.errorResult;
|
|
1885
|
+
const cdp = acquired.cdp;
|
|
964
1886
|
try {
|
|
965
|
-
const
|
|
966
|
-
|
|
1887
|
+
const currentUrl = await getCurrentUrl(cdp, context.signal);
|
|
1888
|
+
const title = await getPageTitle(cdp, context.signal);
|
|
1889
|
+
|
|
1890
|
+
let textContent = await evaluateExpression<string>(
|
|
1891
|
+
cdp,
|
|
1892
|
+
"document.body?.innerText ?? ''",
|
|
1893
|
+
{},
|
|
1894
|
+
context.signal,
|
|
967
1895
|
);
|
|
968
|
-
const currentUrl = page.url();
|
|
969
|
-
const title = await page.title();
|
|
970
|
-
|
|
971
|
-
let textContent = (await page.evaluate(
|
|
972
|
-
`document.body?.innerText ?? ''`,
|
|
973
|
-
)) as string;
|
|
974
1896
|
|
|
975
1897
|
if (textContent.length > MAX_EXTRACT_LENGTH) {
|
|
976
1898
|
textContent =
|
|
977
|
-
textContent
|
|
1899
|
+
safeStringSlice(textContent, 0, MAX_EXTRACT_LENGTH) +
|
|
1900
|
+
"\n... (truncated)";
|
|
978
1901
|
}
|
|
979
1902
|
|
|
980
1903
|
const lines: string[] = [
|
|
@@ -985,15 +1908,9 @@ export async function executeBrowserExtract(
|
|
|
985
1908
|
];
|
|
986
1909
|
|
|
987
1910
|
if (includeLinks) {
|
|
988
|
-
const links =
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
return anchors.slice(0, 200).map(a => ({
|
|
992
|
-
text: (a.textContent || '').trim().slice(0, 80),
|
|
993
|
-
href: a.href,
|
|
994
|
-
}));
|
|
995
|
-
})()
|
|
996
|
-
`)) as Array<{ text: string; href: string }>;
|
|
1911
|
+
const links = await evaluateExpression<
|
|
1912
|
+
Array<{ text: string; href: string }>
|
|
1913
|
+
>(cdp, EXTRACT_LINKS_EXPRESSION, {}, context.signal);
|
|
997
1914
|
|
|
998
1915
|
if (links.length > 0) {
|
|
999
1916
|
lines.push("");
|
|
@@ -1006,9 +1923,18 @@ export async function executeBrowserExtract(
|
|
|
1006
1923
|
|
|
1007
1924
|
return { content: lines.join("\n"), isError: false };
|
|
1008
1925
|
} catch (err) {
|
|
1926
|
+
const diagnosticMessage = formatCdpSendDiagnostics(
|
|
1927
|
+
err,
|
|
1928
|
+
acquired.browserMode,
|
|
1929
|
+
);
|
|
1930
|
+
if (diagnosticMessage) {
|
|
1931
|
+
return { content: diagnosticMessage, isError: true };
|
|
1932
|
+
}
|
|
1009
1933
|
const msg = err instanceof Error ? err.message : String(err);
|
|
1010
1934
|
log.error({ err }, "Extract failed");
|
|
1011
1935
|
return { content: `Error: Extract failed: ${msg}`, isError: true };
|
|
1936
|
+
} finally {
|
|
1937
|
+
cdp.dispose();
|
|
1012
1938
|
}
|
|
1013
1939
|
}
|
|
1014
1940
|
|
|
@@ -1028,26 +1954,43 @@ export async function executeBrowserFillCredential(
|
|
|
1028
1954
|
return { content: "Error: field is required.", isError: true };
|
|
1029
1955
|
}
|
|
1030
1956
|
|
|
1031
|
-
const {
|
|
1957
|
+
const { resolved, error } = resolveElement(context.conversationId, input);
|
|
1032
1958
|
if (error) return { content: error, isError: true };
|
|
1033
1959
|
|
|
1034
1960
|
const pressEnter = input.press_enter === true;
|
|
1035
|
-
|
|
1961
|
+
const targetDescription =
|
|
1962
|
+
resolved!.kind === "backend"
|
|
1963
|
+
? `element_id "${resolved!.eid}"`
|
|
1964
|
+
: resolved!.selector;
|
|
1965
|
+
|
|
1966
|
+
const acquired = acquireCdpClientWithMode(input, context);
|
|
1967
|
+
if (acquired.errorResult) return acquired.errorResult;
|
|
1968
|
+
const cdp = acquired.cdp;
|
|
1036
1969
|
try {
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1970
|
+
let backendNodeId: number;
|
|
1971
|
+
if (resolved!.kind === "backend") {
|
|
1972
|
+
backendNodeId = resolved!.backendNodeId;
|
|
1973
|
+
} else {
|
|
1974
|
+
backendNodeId = await querySelectorBackendNodeId(
|
|
1975
|
+
cdp,
|
|
1976
|
+
resolved!.selector,
|
|
1977
|
+
context.signal,
|
|
1978
|
+
);
|
|
1979
|
+
}
|
|
1040
1980
|
|
|
1041
|
-
// Extract
|
|
1981
|
+
// Extract the current page's hostname for broker domain policy
|
|
1982
|
+
// enforcement. Failures here (pre-navigation, about:blank, malformed
|
|
1983
|
+
// URL) fall through with pageDomain undefined; if the credential
|
|
1984
|
+
// has a domain policy the broker will deny the fill.
|
|
1042
1985
|
let pageDomain: string | undefined;
|
|
1043
1986
|
try {
|
|
1044
|
-
const pageUrl =
|
|
1987
|
+
const pageUrl = await getCurrentUrl(cdp, context.signal);
|
|
1045
1988
|
if (pageUrl && pageUrl !== "about:blank") {
|
|
1046
1989
|
const parsed = new URL(pageUrl);
|
|
1047
1990
|
pageDomain = parsed.hostname;
|
|
1048
1991
|
}
|
|
1049
1992
|
} catch {
|
|
1050
|
-
//
|
|
1993
|
+
// pageDomain stays undefined
|
|
1051
1994
|
}
|
|
1052
1995
|
|
|
1053
1996
|
const result = await credentialBroker.browserFill({
|
|
@@ -1056,7 +1999,13 @@ export async function executeBrowserFillCredential(
|
|
|
1056
1999
|
toolName: "browser_fill_credential",
|
|
1057
2000
|
domain: pageDomain,
|
|
1058
2001
|
fill: async (value) => {
|
|
1059
|
-
|
|
2002
|
+
// Clear-then-focus-then-insert via the shared helper. We
|
|
2003
|
+
// MUST clear first: Input.insertText writes at the cursor,
|
|
2004
|
+
// so on autofilled / pre-populated fields a bare insert
|
|
2005
|
+
// would append the credential to the existing value,
|
|
2006
|
+
// producing a corrupted password and leaking partial state
|
|
2007
|
+
// back into the page.
|
|
2008
|
+
await clearAndInsertText(cdp, backendNodeId, value, context.signal);
|
|
1060
2009
|
},
|
|
1061
2010
|
});
|
|
1062
2011
|
|
|
@@ -1086,7 +2035,10 @@ export async function executeBrowserFillCredential(
|
|
|
1086
2035
|
isError: true,
|
|
1087
2036
|
};
|
|
1088
2037
|
}
|
|
1089
|
-
log.error(
|
|
2038
|
+
log.error(
|
|
2039
|
+
{ target: targetDescription, reason },
|
|
2040
|
+
"Fill credential failed",
|
|
2041
|
+
);
|
|
1090
2042
|
return {
|
|
1091
2043
|
content: `Error: Fill credential failed: ${reason}`,
|
|
1092
2044
|
isError: true,
|
|
@@ -1094,7 +2046,7 @@ export async function executeBrowserFillCredential(
|
|
|
1094
2046
|
}
|
|
1095
2047
|
|
|
1096
2048
|
if (pressEnter) {
|
|
1097
|
-
await
|
|
2049
|
+
await dispatchKeyPress(cdp, "Enter", context.signal);
|
|
1098
2050
|
}
|
|
1099
2051
|
|
|
1100
2052
|
return {
|
|
@@ -1102,8 +2054,501 @@ export async function executeBrowserFillCredential(
|
|
|
1102
2054
|
isError: false,
|
|
1103
2055
|
};
|
|
1104
2056
|
} catch (err) {
|
|
2057
|
+
const diagnosticMessage = formatCdpSendDiagnostics(
|
|
2058
|
+
err,
|
|
2059
|
+
acquired.browserMode,
|
|
2060
|
+
);
|
|
2061
|
+
if (diagnosticMessage) {
|
|
2062
|
+
return { content: diagnosticMessage, isError: true };
|
|
2063
|
+
}
|
|
1105
2064
|
const msg = err instanceof Error ? err.message : String(err);
|
|
1106
2065
|
log.error({ err }, "Fill credential failed");
|
|
1107
2066
|
return { content: `Error: Fill credential failed: ${msg}`, isError: true };
|
|
2067
|
+
} finally {
|
|
2068
|
+
cdp.dispose();
|
|
2069
|
+
}
|
|
2070
|
+
}
|
|
2071
|
+
|
|
2072
|
+
function dedupeStrings(values: string[]): string[] {
|
|
2073
|
+
const seen = new Set<string>();
|
|
2074
|
+
const out: string[] = [];
|
|
2075
|
+
for (const value of values) {
|
|
2076
|
+
if (!value) continue;
|
|
2077
|
+
if (seen.has(value)) continue;
|
|
2078
|
+
seen.add(value);
|
|
2079
|
+
out.push(value);
|
|
2080
|
+
}
|
|
2081
|
+
return out;
|
|
2082
|
+
}
|
|
2083
|
+
|
|
2084
|
+
function modeTradeoffs(mode: StatusCheckMode): string[] {
|
|
2085
|
+
return MODE_TRADEOFFS[mode];
|
|
2086
|
+
}
|
|
2087
|
+
|
|
2088
|
+
function extensionSetupActions(): string[] {
|
|
2089
|
+
return [
|
|
2090
|
+
"Install and enable the Vellum Relay Chrome extension.",
|
|
2091
|
+
"Open the extension popup and click Pair with local assistant.",
|
|
2092
|
+
"Keep the extension connected to the assistant relay.",
|
|
2093
|
+
];
|
|
2094
|
+
}
|
|
2095
|
+
|
|
2096
|
+
function cdpInspectSetupActions(): string[] {
|
|
2097
|
+
return [
|
|
2098
|
+
"Launch Chrome with --remote-debugging-port=9222 (or your configured port).",
|
|
2099
|
+
"Keep Chrome running while browser tools are in use.",
|
|
2100
|
+
"Ensure the configured host is loopback (localhost / 127.0.0.1 / ::1).",
|
|
2101
|
+
];
|
|
2102
|
+
}
|
|
2103
|
+
|
|
2104
|
+
function localSetupActions(): string[] {
|
|
2105
|
+
return [
|
|
2106
|
+
"Install assistant dependencies with bun install in assistant/.",
|
|
2107
|
+
"Install Chromium for Playwright: bunx playwright install chromium.",
|
|
2108
|
+
];
|
|
2109
|
+
}
|
|
2110
|
+
|
|
2111
|
+
function extractDiscoveryCodes(error: CdpError): string[] {
|
|
2112
|
+
const diagnostics = error.attemptDiagnostics ?? [];
|
|
2113
|
+
const codes: string[] = [];
|
|
2114
|
+
for (const diag of diagnostics) {
|
|
2115
|
+
if (diag.discoveryCode) codes.push(diag.discoveryCode);
|
|
2116
|
+
}
|
|
2117
|
+
return dedupeStrings(codes);
|
|
2118
|
+
}
|
|
2119
|
+
|
|
2120
|
+
function containsTokenCaseInsensitive(text: string, token: string): boolean {
|
|
2121
|
+
return text.toLowerCase().includes(token.toLowerCase());
|
|
2122
|
+
}
|
|
2123
|
+
|
|
2124
|
+
function probeFailureActions(mode: StatusCheckMode, error: CdpError): string[] {
|
|
2125
|
+
const actions: string[] = [];
|
|
2126
|
+
const message = error.message.toLowerCase();
|
|
2127
|
+
const discoveryCodes = extractDiscoveryCodes(error).map((c) =>
|
|
2128
|
+
c.toLowerCase(),
|
|
2129
|
+
);
|
|
2130
|
+
|
|
2131
|
+
if (mode === BROWSER_STATUS_MODE.EXTENSION) {
|
|
2132
|
+
actions.push(...extensionSetupActions());
|
|
2133
|
+
if (
|
|
2134
|
+
containsTokenCaseInsensitive(
|
|
2135
|
+
message,
|
|
2136
|
+
EXTENSION_STATUS_ERROR_MARKER.UNAUTHORIZED_ORIGIN,
|
|
2137
|
+
)
|
|
2138
|
+
) {
|
|
2139
|
+
actions.push(
|
|
2140
|
+
"Ensure this extension ID is present in meta/browser-extension/chrome-extension-allowlist.json and restart the assistant.",
|
|
2141
|
+
);
|
|
2142
|
+
}
|
|
2143
|
+
if (
|
|
2144
|
+
containsTokenCaseInsensitive(
|
|
2145
|
+
message,
|
|
2146
|
+
EXTENSION_STATUS_ERROR_MARKER.NATIVE_MESSAGING_HOST,
|
|
2147
|
+
)
|
|
2148
|
+
) {
|
|
2149
|
+
actions.push(
|
|
2150
|
+
"Reinstall the native messaging host manifest and confirm it allows this extension ID.",
|
|
2151
|
+
);
|
|
2152
|
+
}
|
|
2153
|
+
if (
|
|
2154
|
+
containsTokenCaseInsensitive(
|
|
2155
|
+
message,
|
|
2156
|
+
EXTENSION_STATUS_ERROR_MARKER.HTTP_401,
|
|
2157
|
+
)
|
|
2158
|
+
) {
|
|
2159
|
+
actions.push(
|
|
2160
|
+
"Re-pair the extension so it refreshes its local relay credential.",
|
|
2161
|
+
);
|
|
2162
|
+
}
|
|
2163
|
+
}
|
|
2164
|
+
|
|
2165
|
+
if (mode === BROWSER_STATUS_MODE.CDP_INSPECT) {
|
|
2166
|
+
actions.push(...cdpInspectSetupActions());
|
|
2167
|
+
if (discoveryCodes.includes(CDP_INSPECT_STATUS_DISCOVERY_CODE.NO_TARGETS)) {
|
|
2168
|
+
actions.push("Open at least one normal web page tab and retry.");
|
|
2169
|
+
}
|
|
2170
|
+
if (
|
|
2171
|
+
discoveryCodes.includes(
|
|
2172
|
+
CDP_INSPECT_STATUS_DISCOVERY_CODE.INVALID_RESPONSE,
|
|
2173
|
+
) ||
|
|
2174
|
+
discoveryCodes.includes(
|
|
2175
|
+
CDP_INSPECT_STATUS_DISCOVERY_CODE.WS_FALLBACK_FAILED,
|
|
2176
|
+
)
|
|
2177
|
+
) {
|
|
2178
|
+
actions.push(
|
|
2179
|
+
"Verify nothing else is bound to the configured CDP port and exposing non-DevTools responses.",
|
|
2180
|
+
);
|
|
2181
|
+
}
|
|
2182
|
+
}
|
|
2183
|
+
|
|
2184
|
+
if (mode === BROWSER_STATUS_MODE.LOCAL) {
|
|
2185
|
+
actions.push(...localSetupActions());
|
|
2186
|
+
}
|
|
2187
|
+
|
|
2188
|
+
return dedupeStrings(actions);
|
|
2189
|
+
}
|
|
2190
|
+
|
|
2191
|
+
async function probePinnedBrowserMode(
|
|
2192
|
+
mode: StatusCheckMode,
|
|
2193
|
+
context: ToolContext,
|
|
2194
|
+
): Promise<
|
|
2195
|
+
| {
|
|
2196
|
+
ok: true;
|
|
2197
|
+
backendKind: CdpClientKind;
|
|
2198
|
+
}
|
|
2199
|
+
| {
|
|
2200
|
+
ok: false;
|
|
2201
|
+
error: CdpError;
|
|
2202
|
+
diagnostic: string;
|
|
2203
|
+
}
|
|
2204
|
+
> {
|
|
2205
|
+
let cdp: ReturnType<typeof getCdpClient> | null = null;
|
|
2206
|
+
try {
|
|
2207
|
+
cdp = getCdpClient(context, { mode });
|
|
2208
|
+
await cdp.send(
|
|
2209
|
+
"Runtime.evaluate",
|
|
2210
|
+
{
|
|
2211
|
+
expression: "document.readyState",
|
|
2212
|
+
returnByValue: true,
|
|
2213
|
+
},
|
|
2214
|
+
context.signal,
|
|
2215
|
+
);
|
|
2216
|
+
return { ok: true, backendKind: cdp.kind };
|
|
2217
|
+
} catch (err) {
|
|
2218
|
+
if (err instanceof CdpError) {
|
|
2219
|
+
return {
|
|
2220
|
+
ok: false,
|
|
2221
|
+
error: err,
|
|
2222
|
+
diagnostic: formatModeSelectionFailure(mode, err),
|
|
2223
|
+
};
|
|
2224
|
+
}
|
|
2225
|
+
const wrapped = new CdpError(
|
|
2226
|
+
"transport_error",
|
|
2227
|
+
err instanceof Error ? err.message : String(err),
|
|
2228
|
+
{ underlying: err },
|
|
2229
|
+
);
|
|
2230
|
+
return {
|
|
2231
|
+
ok: false,
|
|
2232
|
+
error: wrapped,
|
|
2233
|
+
diagnostic: formatModeSelectionFailure(mode, wrapped),
|
|
2234
|
+
};
|
|
2235
|
+
} finally {
|
|
2236
|
+
cdp?.dispose();
|
|
2237
|
+
}
|
|
2238
|
+
}
|
|
2239
|
+
|
|
2240
|
+
async function checkExtensionModeStatus(
|
|
2241
|
+
context: ToolContext,
|
|
2242
|
+
autoCandidate: boolean,
|
|
2243
|
+
): Promise<BrowserStatusModeResult> {
|
|
2244
|
+
const proxyBound = Boolean(context.hostBrowserProxy);
|
|
2245
|
+
const proxyConnected = context.hostBrowserProxy?.isAvailable() ?? false;
|
|
2246
|
+
|
|
2247
|
+
if (!proxyBound) {
|
|
2248
|
+
return {
|
|
2249
|
+
mode: BROWSER_STATUS_MODE.EXTENSION,
|
|
2250
|
+
available: false,
|
|
2251
|
+
verified: "preflight",
|
|
2252
|
+
autoCandidate,
|
|
2253
|
+
summary:
|
|
2254
|
+
"Extension mode is unavailable: no host browser proxy is bound to this conversation.",
|
|
2255
|
+
userActions: extensionSetupActions(),
|
|
2256
|
+
tradeoffs: modeTradeoffs(BROWSER_STATUS_MODE.EXTENSION),
|
|
2257
|
+
details: {
|
|
2258
|
+
proxyBound,
|
|
2259
|
+
proxyConnected,
|
|
2260
|
+
},
|
|
2261
|
+
};
|
|
2262
|
+
}
|
|
2263
|
+
|
|
2264
|
+
if (!proxyConnected) {
|
|
2265
|
+
return {
|
|
2266
|
+
mode: BROWSER_STATUS_MODE.EXTENSION,
|
|
2267
|
+
available: false,
|
|
2268
|
+
verified: "preflight",
|
|
2269
|
+
autoCandidate,
|
|
2270
|
+
summary:
|
|
2271
|
+
"Extension mode is unavailable: the extension transport is currently disconnected.",
|
|
2272
|
+
userActions: extensionSetupActions(),
|
|
2273
|
+
tradeoffs: modeTradeoffs(BROWSER_STATUS_MODE.EXTENSION),
|
|
2274
|
+
details: {
|
|
2275
|
+
proxyBound,
|
|
2276
|
+
proxyConnected,
|
|
2277
|
+
},
|
|
2278
|
+
};
|
|
2279
|
+
}
|
|
2280
|
+
|
|
2281
|
+
const probe = await probePinnedBrowserMode(
|
|
2282
|
+
BROWSER_STATUS_MODE.EXTENSION,
|
|
2283
|
+
context,
|
|
2284
|
+
);
|
|
2285
|
+
if (probe.ok) {
|
|
2286
|
+
return {
|
|
2287
|
+
mode: BROWSER_STATUS_MODE.EXTENSION,
|
|
2288
|
+
available: true,
|
|
2289
|
+
verified: "active_probe",
|
|
2290
|
+
autoCandidate,
|
|
2291
|
+
summary: "Extension mode is ready and responded to an active CDP probe.",
|
|
2292
|
+
userActions: [],
|
|
2293
|
+
tradeoffs: modeTradeoffs(BROWSER_STATUS_MODE.EXTENSION),
|
|
2294
|
+
details: {
|
|
2295
|
+
proxyBound,
|
|
2296
|
+
proxyConnected,
|
|
2297
|
+
backendKind: probe.backendKind,
|
|
2298
|
+
},
|
|
2299
|
+
};
|
|
2300
|
+
}
|
|
2301
|
+
|
|
2302
|
+
return {
|
|
2303
|
+
mode: BROWSER_STATUS_MODE.EXTENSION,
|
|
2304
|
+
available: false,
|
|
2305
|
+
verified: "active_probe",
|
|
2306
|
+
autoCandidate,
|
|
2307
|
+
summary: `Extension mode probe failed: ${probe.error.message}`,
|
|
2308
|
+
userActions: probeFailureActions(
|
|
2309
|
+
BROWSER_STATUS_MODE.EXTENSION,
|
|
2310
|
+
probe.error,
|
|
2311
|
+
),
|
|
2312
|
+
tradeoffs: modeTradeoffs(BROWSER_STATUS_MODE.EXTENSION),
|
|
2313
|
+
details: {
|
|
2314
|
+
proxyBound,
|
|
2315
|
+
proxyConnected,
|
|
2316
|
+
errorCode: probe.error.code,
|
|
2317
|
+
diagnostic: probe.diagnostic,
|
|
2318
|
+
attemptDiagnostics: probe.error.attemptDiagnostics ?? [],
|
|
2319
|
+
},
|
|
2320
|
+
};
|
|
2321
|
+
}
|
|
2322
|
+
|
|
2323
|
+
async function checkCdpInspectModeStatus(
|
|
2324
|
+
context: ToolContext,
|
|
2325
|
+
autoCandidate: boolean,
|
|
2326
|
+
): Promise<BrowserStatusModeResult> {
|
|
2327
|
+
const cdpInspectConfig = getConfig().hostBrowser.cdpInspect;
|
|
2328
|
+
const desktopAutoEnabled =
|
|
2329
|
+
context.transportInterface === "macos" &&
|
|
2330
|
+
cdpInspectConfig.desktopAuto.enabled;
|
|
2331
|
+
const cooldownActive =
|
|
2332
|
+
desktopAutoEnabled &&
|
|
2333
|
+
isDesktopAutoCooldownActive(cdpInspectConfig.desktopAuto.cooldownMs);
|
|
2334
|
+
|
|
2335
|
+
const probe = await probePinnedBrowserMode(
|
|
2336
|
+
BROWSER_STATUS_MODE.CDP_INSPECT,
|
|
2337
|
+
context,
|
|
2338
|
+
);
|
|
2339
|
+
if (probe.ok) {
|
|
2340
|
+
return {
|
|
2341
|
+
mode: BROWSER_STATUS_MODE.CDP_INSPECT,
|
|
2342
|
+
available: true,
|
|
2343
|
+
verified: "active_probe",
|
|
2344
|
+
autoCandidate,
|
|
2345
|
+
summary:
|
|
2346
|
+
"CDP inspect mode is ready and responded to an active CDP probe.",
|
|
2347
|
+
userActions: [],
|
|
2348
|
+
tradeoffs: modeTradeoffs(BROWSER_STATUS_MODE.CDP_INSPECT),
|
|
2349
|
+
details: {
|
|
2350
|
+
backendKind: probe.backendKind,
|
|
2351
|
+
configEnabled: cdpInspectConfig.enabled,
|
|
2352
|
+
configHost: cdpInspectConfig.host,
|
|
2353
|
+
configPort: cdpInspectConfig.port,
|
|
2354
|
+
desktopAutoEnabled,
|
|
2355
|
+
desktopAutoCooldownActive: cooldownActive,
|
|
2356
|
+
},
|
|
2357
|
+
};
|
|
2358
|
+
}
|
|
2359
|
+
|
|
2360
|
+
return {
|
|
2361
|
+
mode: BROWSER_STATUS_MODE.CDP_INSPECT,
|
|
2362
|
+
available: false,
|
|
2363
|
+
verified: "active_probe",
|
|
2364
|
+
autoCandidate,
|
|
2365
|
+
summary: `CDP inspect probe failed: ${probe.error.message}`,
|
|
2366
|
+
userActions: probeFailureActions(
|
|
2367
|
+
BROWSER_STATUS_MODE.CDP_INSPECT,
|
|
2368
|
+
probe.error,
|
|
2369
|
+
),
|
|
2370
|
+
tradeoffs: modeTradeoffs(BROWSER_STATUS_MODE.CDP_INSPECT),
|
|
2371
|
+
details: {
|
|
2372
|
+
errorCode: probe.error.code,
|
|
2373
|
+
discoveryCodes: extractDiscoveryCodes(probe.error),
|
|
2374
|
+
diagnostic: probe.diagnostic,
|
|
2375
|
+
attemptDiagnostics: probe.error.attemptDiagnostics ?? [],
|
|
2376
|
+
configEnabled: cdpInspectConfig.enabled,
|
|
2377
|
+
configHost: cdpInspectConfig.host,
|
|
2378
|
+
configPort: cdpInspectConfig.port,
|
|
2379
|
+
desktopAutoEnabled,
|
|
2380
|
+
desktopAutoCooldownActive: cooldownActive,
|
|
2381
|
+
},
|
|
2382
|
+
};
|
|
2383
|
+
}
|
|
2384
|
+
|
|
2385
|
+
async function checkLocalModeStatus(
|
|
2386
|
+
context: ToolContext,
|
|
2387
|
+
autoCandidate: boolean,
|
|
2388
|
+
checkLocalLaunch: boolean,
|
|
2389
|
+
): Promise<BrowserStatusModeResult> {
|
|
2390
|
+
const runtime = await checkBrowserRuntime();
|
|
2391
|
+
if (!runtime.playwrightAvailable || !runtime.chromiumInstalled) {
|
|
2392
|
+
return {
|
|
2393
|
+
mode: BROWSER_STATUS_MODE.LOCAL,
|
|
2394
|
+
available: false,
|
|
2395
|
+
verified: "preflight",
|
|
2396
|
+
autoCandidate,
|
|
2397
|
+
summary:
|
|
2398
|
+
runtime.error ??
|
|
2399
|
+
"Local mode preflight failed: Playwright Chromium runtime is not ready.",
|
|
2400
|
+
userActions: localSetupActions(),
|
|
2401
|
+
tradeoffs: modeTradeoffs(BROWSER_STATUS_MODE.LOCAL),
|
|
2402
|
+
details: {
|
|
2403
|
+
runtime,
|
|
2404
|
+
launchProbeRequested: checkLocalLaunch,
|
|
2405
|
+
},
|
|
2406
|
+
};
|
|
2407
|
+
}
|
|
2408
|
+
|
|
2409
|
+
if (!checkLocalLaunch) {
|
|
2410
|
+
return {
|
|
2411
|
+
mode: BROWSER_STATUS_MODE.LOCAL,
|
|
2412
|
+
available: true,
|
|
2413
|
+
verified: "preflight",
|
|
2414
|
+
autoCandidate,
|
|
2415
|
+
summary:
|
|
2416
|
+
"Local mode preflight passed (Playwright + Chromium are present). Launch probe was skipped.",
|
|
2417
|
+
userActions: [],
|
|
2418
|
+
tradeoffs: modeTradeoffs(BROWSER_STATUS_MODE.LOCAL),
|
|
2419
|
+
details: {
|
|
2420
|
+
runtime,
|
|
2421
|
+
launchProbeRequested: checkLocalLaunch,
|
|
2422
|
+
},
|
|
2423
|
+
};
|
|
2424
|
+
}
|
|
2425
|
+
|
|
2426
|
+
const probe = await probePinnedBrowserMode(
|
|
2427
|
+
BROWSER_STATUS_MODE.LOCAL,
|
|
2428
|
+
context,
|
|
2429
|
+
);
|
|
2430
|
+
if (probe.ok) {
|
|
2431
|
+
return {
|
|
2432
|
+
mode: BROWSER_STATUS_MODE.LOCAL,
|
|
2433
|
+
available: true,
|
|
2434
|
+
verified: "active_probe",
|
|
2435
|
+
autoCandidate,
|
|
2436
|
+
summary: "Local mode is ready and responded to an active CDP probe.",
|
|
2437
|
+
userActions: [],
|
|
2438
|
+
tradeoffs: modeTradeoffs(BROWSER_STATUS_MODE.LOCAL),
|
|
2439
|
+
details: {
|
|
2440
|
+
runtime,
|
|
2441
|
+
launchProbeRequested: checkLocalLaunch,
|
|
2442
|
+
backendKind: probe.backendKind,
|
|
2443
|
+
},
|
|
2444
|
+
};
|
|
2445
|
+
}
|
|
2446
|
+
|
|
2447
|
+
return {
|
|
2448
|
+
mode: BROWSER_STATUS_MODE.LOCAL,
|
|
2449
|
+
available: false,
|
|
2450
|
+
verified: "active_probe",
|
|
2451
|
+
autoCandidate,
|
|
2452
|
+
summary: `Local mode probe failed: ${probe.error.message}`,
|
|
2453
|
+
userActions: probeFailureActions(BROWSER_STATUS_MODE.LOCAL, probe.error),
|
|
2454
|
+
tradeoffs: modeTradeoffs(BROWSER_STATUS_MODE.LOCAL),
|
|
2455
|
+
details: {
|
|
2456
|
+
runtime,
|
|
2457
|
+
launchProbeRequested: checkLocalLaunch,
|
|
2458
|
+
errorCode: probe.error.code,
|
|
2459
|
+
diagnostic: probe.diagnostic,
|
|
2460
|
+
attemptDiagnostics: probe.error.attemptDiagnostics ?? [],
|
|
2461
|
+
},
|
|
2462
|
+
};
|
|
2463
|
+
}
|
|
2464
|
+
|
|
2465
|
+
// ── browser_status ────────────────────────────────────────────────────
|
|
2466
|
+
|
|
2467
|
+
export async function executeBrowserStatus(
|
|
2468
|
+
input: Record<string, unknown>,
|
|
2469
|
+
context: ToolContext,
|
|
2470
|
+
): Promise<ToolExecutionResult> {
|
|
2471
|
+
const parsedMode = parseBrowserMode(input);
|
|
2472
|
+
if (!parsedMode.ok) {
|
|
2473
|
+
return { content: parsedMode.error, isError: true };
|
|
2474
|
+
}
|
|
2475
|
+
|
|
2476
|
+
if (
|
|
2477
|
+
input[BROWSER_STATUS_INPUT_FIELD.CHECK_LOCAL_LAUNCH] !== undefined &&
|
|
2478
|
+
typeof input[BROWSER_STATUS_INPUT_FIELD.CHECK_LOCAL_LAUNCH] !== "boolean"
|
|
2479
|
+
) {
|
|
2480
|
+
return {
|
|
2481
|
+
content: `Error: ${BROWSER_STATUS_INPUT_FIELD.CHECK_LOCAL_LAUNCH} must be a boolean when provided.`,
|
|
2482
|
+
isError: true,
|
|
2483
|
+
};
|
|
2484
|
+
}
|
|
2485
|
+
|
|
2486
|
+
const checkLocalLaunch =
|
|
2487
|
+
input[BROWSER_STATUS_INPUT_FIELD.CHECK_LOCAL_LAUNCH] === true;
|
|
2488
|
+
const requestedMode = parsedMode.mode;
|
|
2489
|
+
const modesToCheck: readonly StatusCheckMode[] =
|
|
2490
|
+
requestedMode === BROWSER_MODE.AUTO
|
|
2491
|
+
? BROWSER_STATUS_MODES
|
|
2492
|
+
: [requestedMode];
|
|
2493
|
+
|
|
2494
|
+
const autoCandidateKinds = buildCandidateList(context).map((c) => c.kind);
|
|
2495
|
+
const autoCandidateSet = new Set<CdpClientKind>(autoCandidateKinds);
|
|
2496
|
+
|
|
2497
|
+
try {
|
|
2498
|
+
const modeResults: BrowserStatusModeResult[] = [];
|
|
2499
|
+
for (const mode of modesToCheck) {
|
|
2500
|
+
const autoCandidate = autoCandidateSet.has(mode);
|
|
2501
|
+
if (mode === BROWSER_STATUS_MODE.EXTENSION) {
|
|
2502
|
+
modeResults.push(
|
|
2503
|
+
await checkExtensionModeStatus(context, autoCandidate),
|
|
2504
|
+
);
|
|
2505
|
+
} else if (mode === BROWSER_STATUS_MODE.CDP_INSPECT) {
|
|
2506
|
+
modeResults.push(
|
|
2507
|
+
await checkCdpInspectModeStatus(context, autoCandidate),
|
|
2508
|
+
);
|
|
2509
|
+
} else {
|
|
2510
|
+
modeResults.push(
|
|
2511
|
+
await checkLocalModeStatus(context, autoCandidate, checkLocalLaunch),
|
|
2512
|
+
);
|
|
2513
|
+
}
|
|
2514
|
+
}
|
|
2515
|
+
|
|
2516
|
+
const stickyMode = browserManager.getPreferredBackendKind(
|
|
2517
|
+
context.conversationId,
|
|
2518
|
+
);
|
|
2519
|
+
const availableModes = modeResults
|
|
2520
|
+
.filter((r) => r.available)
|
|
2521
|
+
.map((r) => r.mode);
|
|
2522
|
+
const recommendedMode =
|
|
2523
|
+
autoCandidateKinds.find((candidate) =>
|
|
2524
|
+
modeResults.some(
|
|
2525
|
+
(result) => result.mode === candidate && result.available,
|
|
2526
|
+
),
|
|
2527
|
+
) ??
|
|
2528
|
+
availableModes[0] ??
|
|
2529
|
+
null;
|
|
2530
|
+
|
|
2531
|
+
return {
|
|
2532
|
+
content: JSON.stringify(
|
|
2533
|
+
{
|
|
2534
|
+
requestedMode,
|
|
2535
|
+
checkedModes: modesToCheck,
|
|
2536
|
+
autoCandidateOrder: autoCandidateKinds,
|
|
2537
|
+
stickyConversationMode: stickyMode,
|
|
2538
|
+
recommendedMode,
|
|
2539
|
+
checkLocalLaunch,
|
|
2540
|
+
modes: modeResults,
|
|
2541
|
+
},
|
|
2542
|
+
null,
|
|
2543
|
+
2,
|
|
2544
|
+
),
|
|
2545
|
+
isError: false,
|
|
2546
|
+
};
|
|
2547
|
+
} catch (err) {
|
|
2548
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
2549
|
+
return {
|
|
2550
|
+
content: `Error: browser_status failed: ${msg}`,
|
|
2551
|
+
isError: true,
|
|
2552
|
+
};
|
|
1108
2553
|
}
|
|
1109
2554
|
}
|