@vellumai/assistant 0.6.3 → 0.6.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.prettierignore +5 -0
- package/ARCHITECTURE.md +298 -39
- package/Dockerfile +14 -3
- package/README.md +3 -4
- package/bun.lock +13 -16
- package/docs/architecture/integrations.md +1 -20
- package/docs/architecture/security.md +16 -16
- package/docs/backup-troubleshooting.md +52 -0
- package/docs/browser-use-architecture-phase2.md +174 -0
- package/docs/error-handling.md +111 -0
- package/docs/skills.md +10 -10
- package/docs/stt-provider-onboarding.md +121 -0
- package/knip.json +20 -3
- package/node_modules/@vellumai/ces-contracts/bun.lock +8 -6
- package/node_modules/@vellumai/ces-contracts/package.json +5 -4
- package/node_modules/@vellumai/ces-contracts/src/__tests__/trust-rules.test.ts +471 -0
- package/node_modules/@vellumai/ces-contracts/src/trust-rules.ts +398 -4
- package/node_modules/@vellumai/credential-storage/bun.lock +2 -2
- package/node_modules/@vellumai/credential-storage/package.json +2 -2
- package/node_modules/@vellumai/credential-storage/src/oauth-runtime.ts +20 -2
- package/node_modules/@vellumai/egress-proxy/bun.lock +2 -2
- package/node_modules/@vellumai/egress-proxy/package.json +2 -2
- package/openapi.yaml +1094 -72
- package/package.json +9 -8
- package/scripts/generate-openapi.ts +50 -12
- package/scripts/test.sh +73 -18
- package/src/__tests__/agent-image-optimize.test.ts +28 -0
- package/src/__tests__/agent-loop-callsite-precedence.test.ts +318 -0
- package/src/__tests__/agent-loop-sentry-hygiene.test.ts +137 -0
- package/src/__tests__/agent-loop.test.ts +235 -1
- package/src/__tests__/anthropic-error-formatting.test.ts +98 -0
- package/src/__tests__/anthropic-provider.test.ts +434 -12
- package/src/__tests__/approval-cascade.test.ts +31 -10
- package/src/__tests__/approval-routes-http.test.ts +134 -10
- package/src/__tests__/assistant-attachments.test.ts +44 -0
- package/src/__tests__/assistant-feature-flags-integration.test.ts +29 -0
- package/src/__tests__/auto-analysis-end-to-end.test.ts +550 -0
- package/src/__tests__/auto-analysis-prompt.test.ts +50 -0
- package/src/__tests__/browser-fill-credential.test.ts +12 -1
- package/src/__tests__/browser-identifier-parity-guard.test.ts +53 -0
- package/src/__tests__/browser-skill-baseline-tool-payload.test.ts +23 -33
- package/src/__tests__/browser-skill-endstate.test.ts +52 -159
- package/src/__tests__/btw-routes.test.ts +54 -1
- package/src/__tests__/call-controller.test.ts +582 -22
- package/src/__tests__/call-site-routing-provider.test.ts +214 -0
- package/src/__tests__/catalog-cache.test.ts +27 -4
- package/src/__tests__/catalog-files.test.ts +138 -0
- package/src/__tests__/channel-approval-routes.test.ts +4 -4
- package/src/__tests__/channel-invite-transport.test.ts +2 -2
- package/src/__tests__/channel-readiness-routes.test.ts +16 -20
- package/src/__tests__/channel-readiness-service.test.ts +12 -7
- package/src/__tests__/channel-reply-delivery.test.ts +300 -2
- package/src/__tests__/checker.test.ts +576 -502
- package/src/__tests__/clawhub-files.test.ts +347 -0
- package/src/__tests__/cli-command-risk-guard.test.ts +30 -33
- package/src/__tests__/commit-message-enrichment-service.test.ts +36 -19
- package/src/__tests__/compaction-circuit-breaker.test.ts +336 -0
- package/src/__tests__/compaction.benchmark.test.ts +1 -1
- package/src/__tests__/config-analysis.test.ts +83 -0
- package/src/__tests__/config-loader-backfill.test.ts +174 -0
- package/src/__tests__/config-loader-corrupt.test.ts +183 -0
- package/src/__tests__/config-loader-quarantine-bulletin.test.ts +202 -0
- package/src/__tests__/config-schema-cmd.test.ts +11 -5
- package/src/__tests__/config-schema.test.ts +1458 -198
- package/src/__tests__/config-watcher-cleanup-throttle.test.ts +339 -0
- package/src/__tests__/config-watcher.test.ts +45 -10
- package/src/__tests__/contact-store-user-file.test.ts +511 -0
- package/src/__tests__/contacts-write.test.ts +197 -0
- package/src/__tests__/context-token-estimator.test.ts +191 -1
- package/src/__tests__/context-window-manager.test.ts +618 -2
- package/src/__tests__/conversation-abort-tool-results.test.ts +32 -16
- package/src/__tests__/conversation-agent-loop-overflow.test.ts +62 -17
- package/src/__tests__/conversation-agent-loop.test.ts +510 -84
- package/src/__tests__/conversation-attachments.test.ts +1 -1
- package/src/__tests__/conversation-confirmation-signals.test.ts +165 -9
- package/src/__tests__/conversation-error.test.ts +102 -1
- package/src/__tests__/conversation-history-web-search.test.ts +17 -4
- package/src/__tests__/conversation-init.benchmark.test.ts +42 -1
- package/src/__tests__/conversation-launcher-skill-regression.test.ts +51 -0
- package/src/__tests__/conversation-lifecycle.test.ts +336 -0
- package/src/__tests__/conversation-list-source.test.ts +145 -0
- package/src/__tests__/conversation-load-history-repair.test.ts +27 -10
- package/src/__tests__/conversation-pre-run-repair.test.ts +32 -16
- package/src/__tests__/conversation-process-callsite.test.ts +306 -0
- package/src/__tests__/conversation-provider-retry-repair.test.ts +32 -16
- package/src/__tests__/conversation-queue.test.ts +932 -76
- package/src/__tests__/conversation-routes-disk-view.test.ts +299 -1
- package/src/__tests__/conversation-routes-slash-commands.test.ts +31 -3
- package/src/__tests__/conversation-runtime-assembly.test.ts +2790 -55
- package/src/__tests__/conversation-runtime-workspace.test.ts +12 -12
- package/src/__tests__/conversation-skill-tools.test.ts +12 -143
- package/src/__tests__/conversation-slash-commands.test.ts +33 -0
- package/src/__tests__/conversation-slash-queue.test.ts +120 -34
- package/src/__tests__/conversation-slash-unknown.test.ts +32 -16
- package/src/__tests__/conversation-speed-override.test.ts +30 -11
- package/src/__tests__/conversation-surfaces-standalone-payloads.test.ts +1035 -0
- package/src/__tests__/conversation-surfaces-standalone.test.ts +630 -0
- package/src/__tests__/conversation-title-service.test.ts +2 -2
- package/src/__tests__/conversation-tool-setup-batch-authorized.test.ts +226 -0
- package/src/__tests__/conversation-unread-route.test.ts +2 -2
- package/src/__tests__/conversation-usage.test.ts +3 -1
- package/src/__tests__/conversation-workspace-cache-state.test.ts +31 -10
- package/src/__tests__/conversation-workspace-injection.test.ts +45 -15
- package/src/__tests__/conversation-workspace-tool-tracking.test.ts +46 -16
- package/src/__tests__/credential-broker-browser-fill.test.ts +110 -0
- package/src/__tests__/credential-health-service.test.ts +352 -0
- package/src/__tests__/credential-security-invariants.test.ts +8 -3
- package/src/__tests__/credential-storage-oauth-compat.test.ts +18 -0
- package/src/__tests__/credential-storage-static-compat.test.ts +28 -0
- package/src/__tests__/credential-vault-unit.test.ts +495 -3
- package/src/__tests__/credentials-cli.test.ts +32 -16
- package/src/__tests__/cross-provider-web-search.test.ts +230 -35
- package/src/__tests__/daemon-server-persist-and-process-callsite.test.ts +92 -0
- package/src/__tests__/delete-propagation.test.ts +437 -0
- package/src/__tests__/deterministic-verification-control-plane.test.ts +10 -1
- package/src/__tests__/device-id.test.ts +112 -0
- package/src/__tests__/dm-backfill.test.ts +417 -0
- package/src/__tests__/dm-persistence.test.ts +227 -0
- package/src/__tests__/docker-signing-key-bootstrap.test.ts +167 -4
- package/src/__tests__/dynamic-skill-workflow-prompt.test.ts +1 -3
- package/src/__tests__/edit-propagation.test.ts +280 -0
- package/src/__tests__/email-html-renderer.test.ts +71 -0
- package/src/__tests__/email-invite-adapter.test.ts +36 -32
- package/src/__tests__/emit-event-signal.test.ts +71 -0
- package/src/__tests__/ephemeral-permissions.test.ts +93 -3
- package/src/__tests__/estimator-calibration-integration.test.ts +208 -0
- package/src/__tests__/estimator-calibration.test.ts +213 -0
- package/src/__tests__/extension-id-sync-guard.test.ts +101 -15
- package/src/__tests__/file-write-tool.test.ts +151 -1
- package/src/__tests__/filing-service.test.ts +255 -0
- package/src/__tests__/fixtures/mock-chrome-extension.ts +11 -0
- package/src/__tests__/gateway-only-enforcement.test.ts +206 -1
- package/src/__tests__/gateway-only-guard.test.ts +0 -1
- package/src/__tests__/gemini-provider.test.ts +64 -3
- package/src/__tests__/get-skill-detail-audit.test.ts +325 -0
- package/src/__tests__/guardian-grant-minting.test.ts +8 -0
- package/src/__tests__/headless-browser-interactions.test.ts +44 -1
- package/src/__tests__/headless-browser-mode.test.ts +614 -0
- package/src/__tests__/headless-browser-navigate.test.ts +142 -5
- package/src/__tests__/headless-browser-read-tools.test.ts +11 -0
- package/src/__tests__/headless-browser-snapshot.test.ts +10 -0
- package/src/__tests__/heartbeat-service.test.ts +166 -32
- package/src/__tests__/home-state-routes.test.ts +162 -0
- package/src/__tests__/host-bash-proxy.test.ts +0 -5
- package/src/__tests__/host-browser-e2e-cloud.test.ts +138 -4
- package/src/__tests__/host-browser-e2e-self-hosted.test.ts +4 -4
- package/src/__tests__/host-browser-ws-events-e2e.test.ts +103 -0
- package/src/__tests__/host-cu-proxy.test.ts +0 -5
- package/src/__tests__/host-shell-tool.test.ts +124 -18
- package/src/__tests__/http-user-message-parity.test.ts +29 -1
- package/src/__tests__/identity-intro-cache.test.ts +40 -10
- package/src/__tests__/inbound-slack-persistence.test.ts +340 -0
- package/src/__tests__/init-feature-flag-overrides.test.ts +38 -112
- package/src/__tests__/intent-routing.test.ts +1 -40
- package/src/__tests__/jobs-store-upsert-debounced.test.ts +141 -0
- package/src/__tests__/llm-catalog-parity.test.ts +174 -0
- package/src/__tests__/llm-context-normalization.test.ts +609 -0
- package/src/__tests__/llm-context-route-provider.test.ts +86 -5
- package/src/__tests__/llm-resolver.test.ts +214 -0
- package/src/__tests__/llm-schema.test.ts +223 -0
- package/src/__tests__/llm-usage-store.test.ts +363 -0
- package/src/__tests__/managed-proxy-context.test.ts +6 -2
- package/src/__tests__/media-stream-output.test.ts +555 -0
- package/src/__tests__/media-stream-parser.test.ts +374 -0
- package/src/__tests__/media-stream-server-integration.test.ts +1234 -0
- package/src/__tests__/media-stream-stt-session.test.ts +588 -0
- package/src/__tests__/media-turn-detector.test.ts +440 -0
- package/src/__tests__/message-queue.test.ts +125 -0
- package/src/__tests__/messaging-skill-split.test.ts +3 -34
- package/src/__tests__/migration-export-http.test.ts +6 -6
- package/src/__tests__/migration-import-commit-http.test.ts +8 -6
- package/src/__tests__/migration-import-from-url.test.ts +684 -0
- package/src/__tests__/migration-import-preflight-http.test.ts +6 -5
- package/src/__tests__/migration-validate-http.test.ts +3 -3
- package/src/__tests__/mock-gateway-ipc.ts +151 -0
- package/src/__tests__/model-intents.test.ts +10 -84
- package/src/__tests__/notification-decision-fallback.test.ts +0 -10
- package/src/__tests__/notification-decision-identity.test.ts +0 -9
- package/src/__tests__/notification-decision-recipient-context.test.ts +0 -9
- package/src/__tests__/oauth-apps-routes.test.ts +1 -0
- package/src/__tests__/oauth-cli.test.ts +2 -0
- package/src/__tests__/oauth-connect-orchestrator.test.ts +2 -0
- package/src/__tests__/oauth-provider-serializer.test.ts +1 -0
- package/src/__tests__/oauth-providers-routes.test.ts +2 -0
- package/src/__tests__/oauth-store.test.ts +95 -7
- package/src/__tests__/oauth2-gateway-transport.test.ts +257 -9
- package/src/__tests__/oauth2-refresh-retry.test.ts +279 -0
- package/src/__tests__/onboarding-template-contract.test.ts +6 -13
- package/src/__tests__/openai-provider.test.ts +183 -0
- package/src/__tests__/openai-responses-cutover-guard.test.ts +184 -0
- package/src/__tests__/openai-responses-provider.test.ts +1501 -0
- package/src/__tests__/openrouter-provider-only.test.ts +135 -0
- package/src/__tests__/openrouter-token-estimation.test.ts +100 -0
- package/src/__tests__/outbound-slack-persistence.test.ts +293 -0
- package/src/__tests__/permission-checker-host-gate.test.ts +1 -1
- package/src/__tests__/permission-mode.test.ts +16 -0
- package/src/__tests__/permission-types.test.ts +0 -1
- package/src/__tests__/persona-resolver.test.ts +251 -0
- package/src/__tests__/pkb-autoinject.test.ts +37 -1
- package/src/__tests__/platform-bash-auto-approve.test.ts +5 -1
- package/src/__tests__/platform.test.ts +92 -1
- package/src/__tests__/post-turn-tool-result-truncation.test.ts +47 -0
- package/src/__tests__/prechat-onboarding-contract.test.ts +267 -0
- package/src/__tests__/pricing.test.ts +224 -3
- package/src/__tests__/profiler-routes.test.ts +1 -1
- package/src/__tests__/provider-commit-message-generator.test.ts +14 -84
- package/src/__tests__/provider-env-vars-scope.test.ts +52 -0
- package/src/__tests__/provider-error-scenarios.test.ts +135 -6
- package/src/__tests__/provider-managed-proxy-integration.test.ts +42 -11
- package/src/__tests__/provider-registry-ollama.test.ts +1 -2
- package/src/__tests__/proxy-approval-callback.test.ts +0 -1
- package/src/__tests__/qdrant-manager.test.ts +29 -8
- package/src/__tests__/reaction-persistence.test.ts +560 -0
- package/src/__tests__/regenerate-fire-and-forget-trace.test.ts +194 -0
- package/src/__tests__/relationship-state-contract.test.ts +175 -0
- package/src/__tests__/relay-server.test.ts +424 -6
- package/src/__tests__/require-fresh-approval.test.ts +1 -1
- package/src/__tests__/retry-openrouter-only-normalization.test.ts +136 -0
- package/src/__tests__/retry-thinking-tool-choice.test.ts +226 -0
- package/src/__tests__/risk-classifier-parity.test.ts +230 -0
- package/src/__tests__/sanitize-config-for-transfer.test.ts +78 -1
- package/src/__tests__/search-skills-unified.test.ts +118 -0
- package/src/__tests__/secret-ingress-http.test.ts +28 -0
- package/src/__tests__/secret-prompter-channel-fallback.test.ts +125 -0
- package/src/__tests__/secret-routes-managed-proxy.test.ts +2 -3
- package/src/__tests__/secret-scanner-executor.test.ts +5 -1
- package/src/__tests__/secure-keys.test.ts +107 -0
- package/src/__tests__/send-endpoint-busy.test.ts +34 -2
- package/src/__tests__/sequence-store.test.ts +1 -1
- package/src/__tests__/server-history-render.test.ts +80 -0
- package/src/__tests__/settings-routes.test.ts +201 -0
- package/src/__tests__/shell-parser-property.test.ts +13 -13
- package/src/__tests__/skill-cache-store.test.ts +182 -0
- package/src/__tests__/skill-load-feature-flag.test.ts +1 -0
- package/src/__tests__/skills-file-content-endpoint.test.ts +276 -145
- package/src/__tests__/skills-files-catalog-fallback.test.ts +381 -93
- package/src/__tests__/skills.test.ts +19 -30
- package/src/__tests__/skillssh-files.test.ts +446 -0
- package/src/__tests__/slack-app-setup-skill-regression.test.ts +3 -1
- package/src/__tests__/slack-block-formatting.test.ts +110 -0
- package/src/__tests__/slack-channel-config.test.ts +564 -1
- package/src/__tests__/slack-skill.test.ts +3 -8
- package/src/__tests__/starter-bundle.test.ts +35 -0
- package/src/__tests__/stt-catalog-parity.test.ts +282 -0
- package/src/__tests__/stt-stream-session.test.ts +535 -0
- package/src/__tests__/subagent-call-site-routing.test.ts +280 -0
- package/src/__tests__/suggestion-routes.test.ts +160 -3
- package/src/__tests__/system-prompt.test.ts +126 -53
- package/src/__tests__/task-runner.test.ts +3 -1
- package/src/__tests__/tcc-sandbox-deny.test.ts +198 -0
- package/src/__tests__/telephony-stt-routing.test.ts +329 -0
- package/src/__tests__/terminal-tools.test.ts +26 -7
- package/src/__tests__/test-preload.ts +18 -0
- package/src/__tests__/test-support/browser-skill-harness.ts +2 -49
- package/src/__tests__/thread-backfill.test.ts +941 -0
- package/src/__tests__/tool-execution-pipeline.benchmark.test.ts +2 -2
- package/src/__tests__/tool-executor-lifecycle-events.test.ts +10 -6
- package/src/__tests__/tool-executor-shell-integration.test.ts +4 -0
- package/src/__tests__/tool-executor.test.ts +88 -113
- package/src/__tests__/tool-result-truncation.test.ts +36 -0
- package/src/__tests__/trust-store.test.ts +442 -103
- package/src/__tests__/trusted-contact-approval-notifier.test.ts +1 -1
- package/src/__tests__/tts-catalog-parity.test.ts +345 -0
- package/src/__tests__/twilio-routes-twiml.test.ts +512 -114
- package/src/__tests__/twilio-routes.test.ts +376 -0
- package/src/__tests__/unicode.test.ts +293 -0
- package/src/__tests__/update-bulletin-job.test.ts +389 -0
- package/src/__tests__/usage-cache-backfill-migration.test.ts +3 -1
- package/src/__tests__/usage-routes.test.ts +25 -4
- package/src/__tests__/user-reference.test.ts +46 -61
- package/src/__tests__/verification-control-plane-policy.test.ts +5 -22
- package/src/__tests__/voice-config-update.test.ts +403 -0
- package/src/__tests__/voice-quality.test.ts +434 -19
- package/src/__tests__/voice-session-bridge.test.ts +39 -0
- package/src/__tests__/volume-security-guard.test.ts +3 -2
- package/src/__tests__/web-search-history.test.ts +337 -0
- package/src/__tests__/workspace-heartbeat-service.test.ts +7 -0
- package/src/__tests__/workspace-migration-033-stt-service-explicit-config.test.ts +547 -0
- package/src/__tests__/workspace-migration-034-remove-calls-voice-transcription-provider.test.ts +596 -0
- package/src/__tests__/workspace-migration-039-drop-legacy-llm-keys.test.ts +343 -0
- package/src/__tests__/workspace-migration-043-release-notes-latex-rendering.test.ts +202 -0
- package/src/__tests__/workspace-migration-045-release-notes-meet-avatar.test.ts +210 -0
- package/src/__tests__/workspace-migration-drop-user-md.test.ts +368 -0
- package/src/__tests__/workspace-migration-meets.test.ts +244 -0
- package/src/__tests__/workspace-migration-seed-device-id.test.ts +14 -20
- package/src/__tests__/workspace-migration-unify-llm-callsite-configs.test.ts +841 -0
- package/src/__tests__/workspace-policy.test.ts +1 -11
- package/src/acp/client-handler.ts +1 -2
- package/src/agent/image-optimize.ts +24 -12
- package/src/agent/loop.ts +251 -19
- package/src/avatar/resvg-lazy.test.ts +136 -0
- package/src/avatar/resvg-lazy.ts +82 -9
- package/src/avatar/traits-png-sync.ts +21 -1
- package/src/backup/__tests__/backup-key.test.ts +152 -0
- package/src/backup/__tests__/backup-worker.test.ts +767 -0
- package/src/backup/__tests__/list-snapshots.test.ts +87 -0
- package/src/backup/__tests__/local-writer.test.ts +218 -0
- package/src/backup/__tests__/offsite-writer.test.ts +641 -0
- package/src/backup/__tests__/paths.test.ts +300 -0
- package/src/backup/__tests__/restore.test.ts +498 -0
- package/src/backup/__tests__/snapshot-lock.test.ts +352 -0
- package/src/backup/__tests__/stream-crypt.test.ts +228 -0
- package/src/backup/backup-key.ts +137 -0
- package/src/backup/backup-worker.ts +459 -0
- package/src/backup/list-snapshots.ts +147 -0
- package/src/backup/local-writer.ts +133 -0
- package/src/backup/offsite-writer.ts +222 -0
- package/src/backup/paths.ts +226 -0
- package/src/backup/restore.ts +322 -0
- package/src/backup/snapshot-lock.ts +431 -0
- package/src/backup/stream-crypt.ts +263 -0
- package/src/browser/__tests__/operations.test.ts +163 -0
- package/src/browser/identifiers.ts +51 -0
- package/src/browser/operations.ts +660 -0
- package/src/browser/types.ts +81 -0
- package/src/bundler/package-resolver.ts +4 -0
- package/src/calls/audio-store.ts +11 -5
- package/src/calls/call-controller.ts +226 -71
- package/src/calls/call-domain.ts +9 -0
- package/src/calls/call-speech-output.ts +190 -0
- package/src/calls/call-transport.ts +77 -0
- package/src/calls/guardian-question-copy.ts +2 -2
- package/src/calls/media-stream-audio-transcode.ts +173 -0
- package/src/calls/media-stream-output.ts +660 -0
- package/src/calls/media-stream-parser.ts +300 -0
- package/src/calls/media-stream-protocol.ts +166 -0
- package/src/calls/media-stream-server.ts +592 -0
- package/src/calls/media-stream-stt-session.ts +460 -0
- package/src/calls/media-turn-detector.ts +230 -0
- package/src/calls/relay-server.ts +90 -75
- package/src/calls/resolve-call-tts-provider.ts +136 -0
- package/src/calls/telephony-stt-routing.ts +145 -0
- package/src/calls/tts-call-strategy.ts +161 -0
- package/src/calls/tts-text-sanitizer.ts +32 -16
- package/src/calls/twilio-routes.ts +281 -17
- package/src/calls/voice-quality.ts +78 -35
- package/src/calls/voice-session-bridge.ts +9 -1
- package/src/channels/types.ts +16 -0
- package/src/cli/AGENTS.md +1 -1
- package/src/cli/__tests__/run-assistant-command.ts +11 -1
- package/src/cli/commands/__tests__/attachment.test.ts +438 -0
- package/src/cli/commands/__tests__/backup.test.ts +1165 -0
- package/src/cli/commands/__tests__/browser.test.ts +554 -0
- package/src/cli/commands/__tests__/cache.test.ts +623 -0
- package/src/cli/commands/__tests__/domain-register.test.ts +234 -0
- package/src/cli/commands/__tests__/domain-status.test.ts +132 -0
- package/src/cli/commands/__tests__/email-attachment.test.ts +422 -0
- package/src/cli/commands/__tests__/email-download.test.ts +16 -1
- package/src/cli/commands/__tests__/email-list.test.ts +28 -4
- package/src/cli/commands/__tests__/email-register.test.ts +4 -4
- package/src/cli/commands/__tests__/email-send.test.ts +130 -5
- package/src/cli/commands/__tests__/email-status.test.ts +5 -1
- package/src/cli/commands/__tests__/email-unregister.test.ts +34 -5
- package/src/cli/commands/__tests__/image-generation.test.ts +666 -0
- package/src/cli/commands/__tests__/inference-send.test.ts +451 -0
- package/src/cli/commands/__tests__/stt-transcribe.test.ts +454 -0
- package/src/cli/commands/__tests__/task.test.ts +913 -0
- package/src/cli/commands/__tests__/tts-synthesize.test.ts +594 -0
- package/src/cli/commands/__tests__/ui-confirm.test.ts +650 -0
- package/src/cli/commands/__tests__/ui.test.ts +1215 -0
- package/src/cli/commands/__tests__/watchers.test.ts +716 -0
- package/src/cli/commands/attachment.ts +182 -0
- package/src/cli/commands/backup.ts +993 -0
- package/src/cli/commands/browser.ts +350 -0
- package/src/cli/commands/cache.ts +341 -0
- package/src/cli/commands/completions.ts +0 -3
- package/src/cli/commands/config.ts +6 -6
- package/src/cli/commands/conversations-import.ts +347 -0
- package/src/cli/commands/conversations.ts +90 -0
- package/src/cli/commands/credentials.ts +0 -1
- package/src/cli/commands/domain.ts +210 -0
- package/src/cli/commands/email.ts +308 -16
- package/src/cli/commands/image-generation.ts +300 -0
- package/src/cli/commands/inference.ts +200 -0
- package/src/cli/commands/memory.ts +127 -17
- package/src/cli/commands/oauth/__tests__/connect.test.ts +12 -0
- package/src/cli/commands/oauth/__tests__/providers-delete.test.ts +1 -0
- package/src/cli/commands/oauth/__tests__/providers-register.test.ts +1 -0
- package/src/cli/commands/oauth/__tests__/providers-update.test.ts +1 -0
- package/src/cli/commands/oauth/mode.ts +12 -3
- package/src/cli/commands/oauth/providers.ts +15 -0
- package/src/cli/commands/oauth/shared.ts +2 -1
- package/src/cli/commands/platform/__tests__/callback-routes-list.test.ts +4 -10
- package/src/cli/commands/platform/__tests__/connect.test.ts +6 -1
- package/src/cli/commands/platform/__tests__/disconnect.test.ts +7 -2
- package/src/cli/commands/platform/__tests__/status.test.ts +6 -1
- package/src/cli/commands/stt.ts +339 -0
- package/src/cli/commands/task.ts +795 -0
- package/src/cli/commands/trust.ts +50 -19
- package/src/cli/commands/tts.ts +273 -0
- package/src/cli/commands/ui.ts +670 -0
- package/src/cli/commands/watchers.ts +509 -0
- package/src/cli/lib/daemon-credential-client.ts +0 -19
- package/src/cli/program.ts +53 -8
- package/src/cli.ts +0 -37
- package/src/config/__tests__/backup-schema.test.ts +134 -0
- package/src/config/assistant-feature-flags.ts +61 -62
- package/src/config/bundled-skills/app-builder/references/CUSTOM_ROUTES.md +37 -1
- package/src/config/bundled-skills/contacts/SKILL.md +2 -2
- package/src/config/bundled-skills/conversations/tools/rename-conversation.ts +23 -1
- package/src/config/bundled-skills/media-processing/SKILL.md +3 -9
- package/src/config/bundled-skills/media-processing/TOOLS.json +1 -6
- package/src/config/bundled-skills/media-processing/__tests__/audio-transcribe.test.ts +125 -0
- package/src/config/bundled-skills/media-processing/__tests__/extract-keyframes.test.ts +181 -0
- package/src/config/bundled-skills/media-processing/__tests__/preprocess-audio.test.ts +141 -0
- package/src/config/bundled-skills/media-processing/services/audio-transcribe.ts +32 -87
- package/src/config/bundled-skills/media-processing/services/preprocess.ts +8 -4
- package/src/config/bundled-skills/media-processing/services/reduce.ts +1 -1
- package/src/config/bundled-skills/media-processing/tools/extract-keyframes.ts +0 -10
- package/src/config/bundled-skills/messaging/SKILL.md +5 -5
- package/src/config/bundled-skills/messaging/TOOLS.json +4 -0
- package/src/config/bundled-skills/messaging/tools/messaging-archive-by-sender.ts +9 -2
- package/src/config/bundled-skills/messaging/tools/messaging-read.ts +15 -1
- package/src/config/bundled-skills/messaging/tools/messaging-search.ts +21 -1
- package/src/config/bundled-skills/messaging/tools/messaging-send.ts +11 -12
- package/src/config/bundled-skills/phone-calls/SKILL.md +2 -2
- package/src/config/bundled-skills/phone-calls/references/CONFIG.md +28 -18
- package/src/config/bundled-skills/phone-calls/references/TROUBLESHOOTING.md +3 -3
- package/src/config/bundled-skills/settings/TOOLS.json +3 -3
- package/src/config/bundled-skills/settings/tools/voice-config-update.ts +26 -22
- package/src/config/bundled-skills/transcribe/SKILL.md +9 -14
- package/src/config/bundled-skills/transcribe/TOOLS.json +2 -7
- package/src/config/bundled-skills/transcribe/tools/transcribe-media.test.ts +256 -0
- package/src/config/bundled-skills/transcribe/tools/transcribe-media.ts +38 -188
- package/src/config/bundled-tool-registry.ts +0 -167
- package/src/config/env-registry.ts +24 -0
- package/src/config/env.ts +39 -10
- package/src/config/feature-flag-registry.json +63 -15
- package/src/config/llm-resolver.ts +128 -0
- package/src/config/loader.ts +220 -22
- package/src/config/raw-config-utils.ts +30 -2
- package/src/config/sanitize-for-transfer.ts +35 -0
- package/src/config/schema.ts +65 -51
- package/src/config/schemas/__tests__/stt.test.ts +43 -0
- package/src/config/schemas/analysis.ts +32 -0
- package/src/config/schemas/backup.ts +72 -0
- package/src/config/schemas/calls.ts +1 -30
- package/src/config/schemas/elevenlabs.ts +0 -59
- package/src/config/schemas/filing.ts +49 -14
- package/src/config/schemas/heartbeat.ts +27 -10
- package/src/config/schemas/host-browser.ts +47 -1
- package/src/config/schemas/inference.ts +3 -23
- package/src/config/schemas/llm.ts +318 -0
- package/src/config/schemas/memory-lifecycle.ts +14 -2
- package/src/config/schemas/memory-processing.ts +1 -9
- package/src/config/schemas/notifications.ts +4 -11
- package/src/config/schemas/platform.ts +3 -9
- package/src/config/schemas/security.ts +33 -0
- package/src/config/schemas/services.ts +53 -4
- package/src/config/schemas/stt.ts +60 -0
- package/src/config/schemas/tts.ts +283 -0
- package/src/config/schemas/updates.ts +14 -0
- package/src/config/schemas/workspace-git.ts +3 -40
- package/src/config/skills.ts +6 -2
- package/src/config/types.ts +4 -0
- package/src/contacts/contact-store.ts +56 -11
- package/src/contacts/contacts-write.ts +38 -1
- package/src/context/__tests__/compact-prompt.test.ts +45 -0
- package/src/context/__tests__/microcompact.test.ts +805 -0
- package/src/context/estimator-calibration.ts +136 -0
- package/src/context/microcompact.ts +443 -0
- package/src/context/post-turn-tool-result-truncation.ts +3 -2
- package/src/context/prompts/compact.md +12 -0
- package/src/context/token-estimator.ts +61 -3
- package/src/context/tool-result-truncation.ts +2 -1
- package/src/context/window-manager.ts +272 -35
- package/src/credential-execution/approval-bridge.ts +0 -1
- package/src/credential-execution/executable-discovery.ts +23 -2
- package/src/credential-execution/process-manager.test.ts +109 -0
- package/src/credential-execution/process-manager.ts +96 -2
- package/src/credential-health/credential-health-service.ts +366 -0
- package/src/daemon/__tests__/conversation-lifecycle-auto-analyze.test.ts +324 -0
- package/src/daemon/__tests__/conversation-surfaces-launch.test.ts +497 -0
- package/src/daemon/__tests__/conversation-tool-setup.test.ts +17 -8
- package/src/daemon/__tests__/lifecycle-startup-ordering.test.ts +127 -0
- package/src/daemon/approval-generators.ts +29 -4
- package/src/daemon/assistant-attachments.ts +24 -13
- package/src/daemon/classifier.ts +2 -2
- package/src/daemon/config-watcher.ts +99 -6
- package/src/daemon/context-overflow-reducer.ts +4 -1
- package/src/daemon/conversation-agent-loop-handlers.ts +85 -12
- package/src/daemon/conversation-agent-loop.ts +563 -104
- package/src/daemon/conversation-attachments.ts +2 -6
- package/src/daemon/conversation-error.ts +46 -0
- package/src/daemon/conversation-history.ts +40 -6
- package/src/daemon/conversation-launch.ts +220 -0
- package/src/daemon/conversation-lifecycle.ts +85 -11
- package/src/daemon/conversation-messaging.ts +110 -7
- package/src/daemon/conversation-notifiers.ts +5 -0
- package/src/daemon/conversation-process.ts +591 -23
- package/src/daemon/conversation-queue-manager.ts +27 -0
- package/src/daemon/conversation-runtime-assembly.ts +769 -28
- package/src/daemon/conversation-slash.ts +38 -2
- package/src/daemon/conversation-surfaces.ts +483 -5
- package/src/daemon/conversation-tool-setup.ts +35 -5
- package/src/daemon/conversation-usage.ts +8 -5
- package/src/daemon/conversation.ts +193 -47
- package/src/daemon/external-skills-bootstrap.ts +41 -0
- package/src/daemon/guardian-action-generators.ts +34 -14
- package/src/daemon/handlers/config-model.test.ts +86 -0
- package/src/daemon/handlers/config-model.ts +54 -12
- package/src/daemon/handlers/config-slack-channel.ts +269 -94
- package/src/daemon/handlers/conversations.ts +13 -3
- package/src/daemon/handlers/shared.ts +51 -1
- package/src/daemon/handlers/skills.ts +323 -79
- package/src/daemon/handlers/slack-channel-oauth-install.ts +197 -0
- package/src/daemon/host-browser-proxy.ts +2 -1
- package/src/daemon/lifecycle.ts +185 -26
- package/src/daemon/message-protocol.ts +6 -0
- package/src/daemon/message-types/conversations.ts +48 -1
- package/src/daemon/message-types/home.ts +40 -0
- package/src/daemon/message-types/meet.ts +143 -0
- package/src/daemon/message-types/messages.ts +23 -1
- package/src/daemon/message-types/schedules.ts +34 -2
- package/src/daemon/message-types/skills.ts +16 -0
- package/src/daemon/message-types/surfaces.ts +2 -0
- package/src/daemon/message-types/trust.ts +0 -2
- package/src/daemon/parse-actual-tokens-from-error.test.ts +57 -1
- package/src/daemon/parse-actual-tokens-from-error.ts +66 -0
- package/src/daemon/pkb-context-tracker.test.ts +169 -0
- package/src/daemon/pkb-context-tracker.ts +125 -0
- package/src/daemon/pkb-reminder-builder.test.ts +70 -0
- package/src/daemon/pkb-reminder-builder.ts +31 -0
- package/src/daemon/providers-setup.ts +6 -0
- package/src/daemon/server.ts +463 -10
- package/src/daemon/shutdown-handlers.ts +32 -4
- package/src/daemon/shutdown-registry.ts +40 -0
- package/src/daemon/tool-side-effects.ts +9 -9
- package/src/daemon/watch-handler.ts +4 -4
- package/src/daemon/web-search-history.ts +126 -0
- package/src/email/html-renderer.ts +76 -0
- package/src/events/domain-events.ts +0 -1
- package/src/filing/filing-service.ts +9 -10
- package/src/heartbeat/heartbeat-service.ts +156 -22
- package/src/home/__tests__/assistant-feed-authoring.test.ts +156 -0
- package/src/home/__tests__/emit-feed-event.test.ts +169 -0
- package/src/home/__tests__/feed-scheduler.test.ts +222 -0
- package/src/home/__tests__/feed-types.test.ts +275 -0
- package/src/home/__tests__/feed-writer.test.ts +688 -0
- package/src/home/__tests__/phase5-exit-criteria.test.ts +212 -0
- package/src/home/__tests__/platform-gmail-digest.test.ts +222 -0
- package/src/home/__tests__/progress-formula.test.ts +213 -0
- package/src/home/__tests__/relationship-state-writer.test.ts +740 -0
- package/src/home/__tests__/rollup-producer.test.ts +442 -0
- package/src/home/assistant-feed-authoring.ts +128 -0
- package/src/home/emit-feed-event.ts +162 -0
- package/src/home/feed-scheduler.ts +263 -0
- package/src/home/feed-types.ts +235 -0
- package/src/home/feed-writer.ts +469 -0
- package/src/home/platform-gmail-digest.ts +163 -0
- package/src/home/progress-formula.ts +86 -0
- package/src/home/relationship-state-writer.ts +824 -0
- package/src/home/relationship-state.ts +143 -0
- package/src/home/rollup-producer.ts +413 -0
- package/src/home/suggested-prompts.ts +101 -0
- package/src/hooks/runner.ts +7 -0
- package/src/inbound/platform-callback-registration.ts +12 -3
- package/src/inbound/public-ingress-urls.ts +12 -0
- package/src/instrument.ts +1 -1
- package/src/ipc/__tests__/attachment-ipc.test.ts +213 -0
- package/src/ipc/__tests__/browser-ipc.test.ts +339 -0
- package/src/ipc/__tests__/cache-ipc.test.ts +266 -0
- package/src/ipc/__tests__/cli-ipc.test.ts +200 -0
- package/src/ipc/__tests__/socket-path.test.ts +73 -0
- package/src/ipc/__tests__/task-ipc.test.ts +577 -0
- package/src/ipc/__tests__/ui-request-route.test.ts +495 -0
- package/src/ipc/__tests__/watcher-ipc.test.ts +295 -0
- package/src/ipc/cli-client.ts +152 -0
- package/src/ipc/cli-server.ts +252 -0
- package/src/ipc/gateway-client.ts +180 -0
- package/src/ipc/routes/attachment.ts +114 -0
- package/src/ipc/routes/browser-context.ts +61 -0
- package/src/ipc/routes/browser.ts +96 -0
- package/src/ipc/routes/cache.ts +96 -0
- package/src/ipc/routes/index.ts +21 -0
- package/src/ipc/routes/task-queue.ts +226 -0
- package/src/ipc/routes/task.ts +173 -0
- package/src/ipc/routes/ui-request.ts +50 -0
- package/src/ipc/routes/wake-conversation.ts +19 -0
- package/src/ipc/routes/watcher.ts +203 -0
- package/src/ipc/socket-path.ts +100 -0
- package/src/memory/__tests__/auto-analysis-enqueue.test.ts +356 -0
- package/src/memory/__tests__/auto-analysis-guard.test.ts +57 -0
- package/src/memory/__tests__/conversation-analyze-job.test.ts +233 -0
- package/src/memory/__tests__/conversation-group-migration.test.ts +99 -0
- package/src/memory/__tests__/find-analysis-conversation.test.ts +196 -0
- package/src/memory/admin.ts +18 -0
- package/src/memory/app-store.ts +1 -1
- package/src/memory/attachments-store.ts +70 -0
- package/src/memory/auto-analysis-enqueue.ts +127 -0
- package/src/memory/auto-analysis-guard.ts +27 -0
- package/src/memory/cleanup-schedule-state.ts +37 -0
- package/src/memory/conversation-analyze-job.ts +74 -0
- package/src/memory/conversation-attention-store.ts +13 -6
- package/src/memory/conversation-crud.ts +199 -0
- package/src/memory/conversation-disk-view.ts +7 -0
- package/src/memory/conversation-group-migration.ts +65 -1
- package/src/memory/conversation-queries.ts +6 -5
- package/src/memory/conversation-title-service.ts +7 -4
- package/src/memory/db-init.ts +8 -0
- package/src/memory/db-maintenance.ts +108 -0
- package/src/memory/db.ts +1 -0
- package/src/memory/embedding-backend.ts +1 -1
- package/src/memory/graph/compaction.ts +299 -0
- package/src/memory/graph/consolidation.ts +4 -4
- package/src/memory/graph/conversation-graph-memory.ts +104 -29
- package/src/memory/graph/extraction.test.ts +295 -2
- package/src/memory/graph/extraction.ts +181 -51
- package/src/memory/graph/graph-search.test.ts +92 -0
- package/src/memory/graph/graph-search.ts +4 -1
- package/src/memory/graph/narrative.ts +2 -2
- package/src/memory/graph/pattern-scan.ts +2 -2
- package/src/memory/graph/retriever.test.ts +459 -0
- package/src/memory/graph/retriever.ts +257 -66
- package/src/memory/graph/scoring.test.ts +186 -0
- package/src/memory/graph/scoring.ts +31 -1
- package/src/memory/graph/store.ts +41 -0
- package/src/memory/graph/tool-handlers.ts +27 -0
- package/src/memory/graph/tools.ts +6 -1
- package/src/memory/group-crud.ts +6 -1
- package/src/memory/indexer.ts +95 -16
- package/src/memory/job-handlers/cleanup.ts +11 -8
- package/src/memory/job-handlers/conversation-starters.ts +39 -30
- package/src/memory/job-handlers/summarization.ts +2 -2
- package/src/memory/job-utils.ts +7 -1
- package/src/memory/jobs/embed-pkb-file.test.ts +168 -0
- package/src/memory/jobs/embed-pkb-file.ts +54 -0
- package/src/memory/jobs-store.ts +106 -5
- package/src/memory/jobs-worker.ts +26 -9
- package/src/memory/llm-usage-store.ts +92 -56
- package/src/memory/migrations/140-backfill-usage-cache-accounting.ts +1 -1
- package/src/memory/migrations/219-oauth-providers-token-exchange-body-format.ts +15 -0
- package/src/memory/migrations/220-normalize-user-file-by-principal.ts +190 -0
- package/src/memory/migrations/221-conversations-archived-at.ts +16 -0
- package/src/memory/migrations/222-strip-placeholder-sentinels-from-messages.ts +82 -0
- package/src/memory/migrations/index.ts +7 -0
- package/src/memory/migrations/registry.ts +8 -0
- package/src/memory/pkb/pkb-index.test.ts +368 -0
- package/src/memory/pkb/pkb-index.ts +255 -0
- package/src/memory/pkb/pkb-reconcile.test.ts +251 -0
- package/src/memory/pkb/pkb-reconcile.ts +148 -0
- package/src/memory/pkb/pkb-search.test.ts +438 -0
- package/src/memory/pkb/pkb-search.ts +137 -0
- package/src/memory/pkb/types.ts +53 -0
- package/src/memory/qdrant-client.ts +122 -1
- package/src/memory/qdrant-manager.ts +43 -16
- package/src/memory/schema/conversations.ts +2 -0
- package/src/memory/schema/oauth.ts +3 -0
- package/src/memory/slack-thread-store.ts +37 -0
- package/src/memory/usage-buckets.ts +396 -0
- package/src/messaging/providers/gmail/adapter.ts +6 -16
- package/src/messaging/providers/gmail/client.ts +79 -6
- package/src/messaging/providers/gmail/types.ts +7 -0
- package/src/messaging/providers/slack/__tests__/adapter-token-routing.test.ts +282 -0
- package/src/messaging/providers/slack/adapter.ts +155 -38
- package/src/messaging/providers/slack/backfill.test.ts +257 -0
- package/src/messaging/providers/slack/backfill.ts +101 -0
- package/src/messaging/providers/slack/client.ts +16 -0
- package/src/messaging/providers/slack/message-metadata.test.ts +316 -0
- package/src/messaging/providers/slack/message-metadata.ts +123 -0
- package/src/messaging/providers/slack/render-transcript.test.ts +1373 -0
- package/src/messaging/providers/slack/render-transcript.ts +443 -0
- package/src/messaging/providers/slack/types.ts +4 -0
- package/src/messaging/style-analyzer.ts +5 -2
- package/src/notifications/README.md +9 -5
- package/src/notifications/decision-engine.ts +6 -12
- package/src/notifications/preference-extractor.ts +2 -6
- package/src/notifications/signal.ts +5 -0
- package/src/oauth/__tests__/identity-verifier.test.ts +1 -0
- package/src/oauth/byo-connection.test.ts +18 -1
- package/src/oauth/byo-connection.ts +3 -1
- package/src/oauth/connect-orchestrator.ts +2 -0
- package/src/oauth/connection-resolver.ts +6 -2
- package/src/oauth/connection.ts +2 -0
- package/src/oauth/oauth-store.ts +10 -0
- package/src/oauth/platform-connection.test.ts +145 -0
- package/src/oauth/platform-connection.ts +62 -31
- package/src/oauth/seed-providers.ts +10 -1
- package/src/permissions/approval-policy.test.ts +948 -0
- package/src/permissions/approval-policy.ts +257 -0
- package/src/permissions/bash-risk-classifier.test.ts +1208 -0
- package/src/permissions/bash-risk-classifier.ts +707 -0
- package/src/permissions/checker.ts +218 -699
- package/src/permissions/command-registry.test.ts +535 -0
- package/src/permissions/command-registry.ts +825 -0
- package/src/permissions/defaults.ts +71 -75
- package/src/permissions/file-risk-classifier.test.ts +535 -0
- package/src/permissions/file-risk-classifier.ts +274 -0
- package/src/permissions/risk-types.ts +205 -0
- package/src/permissions/secret-prompter.ts +53 -2
- package/src/permissions/skill-risk-classifier.test.ts +311 -0
- package/src/permissions/skill-risk-classifier.ts +214 -0
- package/src/permissions/trust-client.ts +52 -25
- package/src/permissions/trust-store-interface.ts +1 -6
- package/src/permissions/trust-store.ts +164 -65
- package/src/permissions/types.ts +23 -14
- package/src/permissions/web-risk-classifier.test.ts +170 -0
- package/src/permissions/web-risk-classifier.ts +89 -0
- package/src/permissions/workspace-policy.ts +1 -13
- package/src/platform/client.test.ts +10 -0
- package/src/platform/client.ts +19 -1
- package/src/platform/sync-identity.ts +129 -0
- package/src/prompts/persona-resolver.ts +127 -3
- package/src/prompts/system-prompt.ts +78 -38
- package/src/prompts/templates/BOOTSTRAP.md +5 -5
- package/src/prompts/templates/SOUL.md +5 -3
- package/src/prompts/templates/channels/slack.md +20 -0
- package/src/prompts/update-bulletin-job.ts +190 -0
- package/src/prompts/user-reference.ts +20 -17
- package/src/providers/__tests__/context-overflow-error.test.ts +328 -0
- package/src/providers/__tests__/provider-env-vars.test.ts +102 -0
- package/src/providers/__tests__/provider-secret-catalog.test.ts +42 -0
- package/src/providers/__tests__/retry-callsite.test.ts +424 -0
- package/src/providers/anthropic/client.ts +335 -70
- package/src/providers/call-site-routing.ts +71 -0
- package/src/providers/fireworks/client.ts +2 -2
- package/src/providers/gemini/client.ts +74 -3
- package/src/providers/managed-proxy/constants.ts +2 -1
- package/src/providers/model-catalog.ts +502 -28
- package/src/providers/model-intents.ts +8 -8
- package/src/providers/ollama/client.ts +2 -2
- package/src/providers/openai/chat-completions-provider.ts +530 -0
- package/src/providers/openai/client.ts +25 -440
- package/src/providers/openai/responses-provider.ts +579 -0
- package/src/providers/openrouter/client.ts +168 -4
- package/src/providers/provider-env-vars.ts +56 -0
- package/src/providers/provider-secret-catalog.ts +139 -0
- package/src/providers/provider-send-message.ts +22 -5
- package/src/providers/ratelimit.ts +4 -0
- package/src/providers/registry.ts +21 -10
- package/src/providers/retry.ts +185 -39
- package/src/providers/speech-to-text/__tests__/provider-catalog.test.ts +251 -0
- package/src/providers/speech-to-text/__tests__/resolve.test.ts +883 -0
- package/src/providers/speech-to-text/deepgram-realtime.test.ts +980 -0
- package/src/providers/speech-to-text/deepgram-realtime.ts +767 -0
- package/src/providers/speech-to-text/deepgram.test.ts +332 -0
- package/src/providers/speech-to-text/deepgram.ts +115 -0
- package/src/providers/speech-to-text/google-gemini-live-stream.test.ts +743 -0
- package/src/providers/speech-to-text/google-gemini-live-stream.ts +625 -0
- package/src/providers/speech-to-text/google-gemini.test.ts +226 -0
- package/src/providers/speech-to-text/google-gemini.ts +101 -0
- package/src/providers/speech-to-text/openai-whisper-stream.test.ts +564 -0
- package/src/providers/speech-to-text/openai-whisper-stream.ts +381 -0
- package/src/providers/speech-to-text/openai-whisper.test.ts +1 -37
- package/src/providers/speech-to-text/openai-whisper.ts +63 -33
- package/src/providers/speech-to-text/provider-catalog.ts +323 -0
- package/src/providers/speech-to-text/resolve.ts +393 -6
- package/src/providers/speech-to-text/xai-realtime.test.ts +578 -0
- package/src/providers/speech-to-text/xai-realtime.ts +796 -0
- package/src/providers/speech-to-text/xai.test.ts +155 -0
- package/src/providers/speech-to-text/xai.ts +97 -0
- package/src/providers/types.ts +102 -3
- package/src/runtime/AGENTS.md +45 -3
- package/src/runtime/__tests__/agent-wake.test.ts +872 -0
- package/src/runtime/__tests__/interactive-ui.test.ts +673 -0
- package/src/runtime/__tests__/runtime-mode.test.ts +62 -0
- package/src/runtime/__tests__/slack-block-formatting.test.ts +481 -0
- package/src/runtime/agent-wake.ts +553 -0
- package/src/runtime/auth/__tests__/route-policy.test.ts +40 -0
- package/src/runtime/auth/route-policy.ts +34 -5
- package/src/runtime/auth/token-service.ts +56 -1
- package/src/runtime/btw-sidechain.ts +15 -3
- package/src/runtime/capability-tokens.ts +10 -10
- package/src/runtime/channel-invite-transport.ts +1 -1
- package/src/runtime/channel-invite-transports/email.ts +14 -6
- package/src/runtime/channel-readiness-service.ts +12 -22
- package/src/runtime/channel-reply-delivery.ts +106 -2
- package/src/runtime/chrome-extension-registry.ts +38 -2
- package/src/runtime/decision-token.ts +116 -0
- package/src/runtime/gateway-client.ts +2 -2
- package/src/runtime/http-router.ts +32 -0
- package/src/runtime/http-server.ts +447 -11
- package/src/runtime/http-types.ts +29 -3
- package/src/runtime/interactive-ui.ts +362 -0
- package/src/runtime/invite-instruction-generator.ts +2 -2
- package/src/runtime/migrations/__tests__/gcs-signed-url.test.ts +176 -0
- package/src/runtime/migrations/__tests__/vbundle-import-credentials.test.ts +36 -0
- package/src/runtime/migrations/__tests__/vbundle-legacy-user-md.test.ts +360 -0
- package/src/runtime/migrations/__tests__/vbundle-metadata-merge-integration.test.ts +390 -0
- package/src/runtime/migrations/__tests__/vbundle-metadata-merge.test.ts +221 -0
- package/src/runtime/migrations/__tests__/vbundle-streaming-importer.test.ts +1540 -0
- package/src/runtime/migrations/__tests__/vbundle-streaming-validator.test.ts +453 -0
- package/src/runtime/migrations/__tests__/vbundle-tar-stream.test.ts +222 -0
- package/src/runtime/migrations/gcs-signed-url.ts +162 -0
- package/src/runtime/migrations/migration-transport.ts +1 -0
- package/src/runtime/migrations/migration-wizard.ts +1 -0
- package/src/runtime/migrations/vbundle-import-analyzer.ts +77 -1
- package/src/runtime/migrations/vbundle-importer.ts +187 -8
- package/src/runtime/migrations/vbundle-metadata-merge.ts +124 -0
- package/src/runtime/migrations/vbundle-streaming-importer.ts +2522 -0
- package/src/runtime/migrations/vbundle-streaming-validator.ts +244 -0
- package/src/runtime/migrations/vbundle-tar-stream.ts +217 -0
- package/src/runtime/migrations/vbundle-validator.ts +15 -6
- package/src/runtime/pending-interactions.ts +0 -11
- package/src/runtime/routes/__tests__/backup-routes.test.ts +967 -0
- package/src/runtime/routes/__tests__/home-feed-routes.test.ts +618 -0
- package/src/runtime/routes/__tests__/migration-import-credential-filter.test.ts +247 -0
- package/src/runtime/routes/__tests__/migration-vellum-metadata-reconcile.test.ts +246 -0
- package/src/runtime/routes/__tests__/stt-routes.test.ts +406 -0
- package/src/runtime/routes/__tests__/tts-routes.test.ts +474 -0
- package/src/runtime/routes/__tests__/user-route-dispatcher.test.ts +148 -17
- package/src/runtime/routes/app-management-routes.ts +12 -18
- package/src/runtime/routes/approval-prompt-ts-tracker.ts +58 -0
- package/src/runtime/routes/approval-routes.ts +12 -17
- package/src/runtime/routes/approval-strategies/guardian-callback-strategy.ts +9 -0
- package/src/runtime/routes/attachment-routes.test.ts +9 -3
- package/src/runtime/routes/attachment-routes.ts +216 -17
- package/src/runtime/routes/avatar-routes.ts +20 -4
- package/src/runtime/routes/backup-routes.ts +519 -0
- package/src/runtime/routes/browser-extension-pair-routes.ts +82 -23
- package/src/runtime/routes/btw-routes.ts +9 -10
- package/src/runtime/routes/contact-routes.test.ts +298 -0
- package/src/runtime/routes/contact-routes.ts +132 -5
- package/src/runtime/routes/conversation-analysis-routes.ts +22 -142
- package/src/runtime/routes/conversation-management-routes.ts +133 -0
- package/src/runtime/routes/conversation-routes.ts +487 -160
- package/src/runtime/routes/debug-routes.ts +1 -1
- package/src/runtime/routes/diagnostics-routes.ts +6 -4
- package/src/runtime/routes/events-routes.ts +16 -0
- package/src/runtime/routes/filing-routes.ts +93 -0
- package/src/runtime/routes/guardian-approval-interception.ts +33 -3
- package/src/runtime/routes/guardian-approval-prompt.ts +13 -3
- package/src/runtime/routes/home-feed-routes.ts +452 -0
- package/src/runtime/routes/home-state-routes.ts +138 -0
- package/src/runtime/routes/host-browser-routes.ts +3 -14
- package/src/runtime/routes/identity-intro-cache.ts +7 -3
- package/src/runtime/routes/identity-routes.ts +3 -17
- package/src/runtime/routes/inbound-message-handler.ts +912 -2
- package/src/runtime/routes/inbound-stages/background-dispatch.test.ts +113 -2
- package/src/runtime/routes/inbound-stages/background-dispatch.ts +61 -3
- package/src/runtime/routes/inbound-stages/edit-intercept.ts +129 -6
- package/src/runtime/routes/inbound-stages/transcribe-audio.test.ts +46 -39
- package/src/runtime/routes/inbound-stages/transcribe-audio.ts +15 -15
- package/src/runtime/routes/integrations/slack/__tests__/channel.test.ts +137 -0
- package/src/runtime/routes/integrations/slack/__tests__/share.test.ts +179 -0
- package/src/runtime/routes/integrations/slack/channel.ts +36 -6
- package/src/runtime/routes/integrations/slack/share.ts +45 -7
- package/src/runtime/routes/llm-context-normalization.ts +325 -0
- package/src/runtime/routes/memory-item-routes.test.ts +3 -2
- package/src/runtime/routes/migration-routes.ts +722 -91
- package/src/runtime/routes/settings-routes.ts +26 -7
- package/src/runtime/routes/skills-routes.ts +76 -7
- package/src/runtime/routes/stt-routes.ts +233 -0
- package/src/runtime/routes/surface-action-routes.ts +41 -2
- package/src/runtime/routes/trust-rules-routes.ts +30 -14
- package/src/runtime/routes/tts-routes.ts +108 -24
- package/src/runtime/routes/usage-routes.ts +30 -2
- package/src/runtime/routes/user-route-dispatcher.ts +50 -5
- package/src/runtime/routes/user-routes.ts +13 -1
- package/src/runtime/routes/work-items-routes.test.ts +1 -1
- package/src/runtime/routes/work-items-routes.ts +11 -3
- package/src/runtime/runtime-mode.ts +33 -0
- package/src/runtime/services/__tests__/analyze-conversation.test.ts +426 -0
- package/src/runtime/services/__tests__/analyze-deps-singleton.test.ts +67 -0
- package/src/runtime/services/__tests__/auto-analysis-prompt.test.ts +53 -0
- package/src/runtime/services/__tests__/manual-analysis-prompt.test.ts +41 -0
- package/src/runtime/services/analyze-conversation.ts +340 -0
- package/src/runtime/services/analyze-deps-singleton.ts +32 -0
- package/src/runtime/services/auto-analysis-prompt.ts +55 -0
- package/src/runtime/skill-route-registry.ts +71 -0
- package/src/runtime/slack-block-formatting.ts +437 -10
- package/src/schedule/scheduler.ts +58 -0
- package/src/security/__tests__/provider-key-env-fallback.test.ts +119 -0
- package/src/security/__tests__/untrusted-content.test.ts +109 -0
- package/src/security/oauth2.ts +122 -37
- package/src/security/secure-keys.ts +32 -10
- package/src/security/token-manager.ts +35 -13
- package/src/security/untrusted-content.ts +102 -0
- package/src/sequence/engine.ts +23 -0
- package/src/sequence/types.ts +1 -1
- package/src/skills/catalog-cache.ts +26 -7
- package/src/skills/catalog-files.ts +64 -2
- package/src/skills/catalog-install.ts +31 -3
- package/src/skills/category-inference.ts +122 -0
- package/src/skills/clawhub-files.ts +213 -0
- package/src/skills/clawhub.ts +84 -23
- package/src/skills/skill-cache-store.ts +97 -0
- package/src/skills/skill-file-provider.ts +40 -0
- package/src/skills/skillssh-files.ts +395 -0
- package/src/skills/skillssh-registry.ts +4 -4
- package/src/stt/__tests__/daemon-batch-transcriber.test.ts +468 -0
- package/src/stt/__tests__/types.test.ts +89 -0
- package/src/stt/daemon-batch-transcriber.ts +228 -0
- package/src/stt/stt-stream-session.ts +506 -0
- package/src/stt/types.ts +334 -0
- package/src/stt/wav-encoder.test.ts +373 -0
- package/src/stt/wav-encoder.ts +175 -0
- package/src/subagent/manager.ts +79 -27
- package/src/tasks/ephemeral-permissions.ts +9 -4
- package/src/telemetry/usage-telemetry-reporter.ts +27 -5
- package/src/tools/browser/__tests__/browser-mode.test.ts +119 -0
- package/src/tools/browser/__tests__/browser-status.test.ts +166 -0
- package/src/tools/browser/browser-execution.ts +1208 -41
- package/src/tools/browser/browser-manager.ts +45 -0
- package/src/tools/browser/browser-mode-constants.ts +12 -0
- package/src/tools/browser/browser-mode.ts +92 -0
- package/src/tools/browser/browser-status-constants.ts +33 -0
- package/src/tools/browser/cdp-client/__tests__/cdp-inspect-client.test.ts +393 -0
- package/src/tools/browser/cdp-client/__tests__/extension-cdp-client.test.ts +29 -0
- package/src/tools/browser/cdp-client/__tests__/factory.test.ts +1648 -32
- package/src/tools/browser/cdp-client/cdp-inspect/__tests__/discovery.test.ts +264 -0
- package/src/tools/browser/cdp-client/cdp-inspect/discovery.ts +205 -17
- package/src/tools/browser/cdp-client/cdp-inspect-client.ts +254 -21
- package/src/tools/browser/cdp-client/errors.ts +15 -0
- package/src/tools/browser/cdp-client/extension-cdp-client.ts +39 -16
- package/src/tools/browser/cdp-client/factory.ts +797 -87
- package/src/tools/browser/cdp-client/index.ts +16 -2
- package/src/tools/browser/cdp-client/types.ts +68 -0
- package/src/tools/credentials/tool-policy.ts +39 -5
- package/src/tools/credentials/vault.ts +41 -7
- package/src/tools/executor.ts +4 -0
- package/src/tools/filesystem/write.ts +52 -0
- package/src/tools/host-terminal/host-shell.ts +45 -5
- package/src/tools/memory/register.test.ts +185 -0
- package/src/tools/memory/register.ts +3 -1
- package/src/tools/network/web-fetch.ts +25 -12
- package/src/tools/network/web-search.ts +20 -2
- package/src/tools/permission-checker.ts +36 -15
- package/src/tools/policy-context.ts +25 -8
- package/src/tools/registry.ts +55 -3
- package/src/tools/shared/shell-output.ts +3 -1
- package/src/tools/side-effects.ts +0 -9
- package/src/tools/skills/execute.ts +2 -2
- package/src/tools/skills/sandbox-runner.ts +6 -2
- package/src/tools/terminal/backends/native.ts +51 -2
- package/src/tools/terminal/safe-env.ts +11 -2
- package/src/tools/terminal/shell.ts +16 -4
- package/src/tools/tool-manifest.ts +6 -0
- package/src/tools/types.ts +29 -3
- package/src/tools/ui-surface/definitions.ts +6 -1
- package/src/tools/verification-control-plane-policy.ts +1 -1
- package/src/tts/__tests__/provider-adapters.test.ts +1061 -0
- package/src/tts/__tests__/provider-catalog-consistency.test.ts +196 -0
- package/src/tts/__tests__/provider-catalog.test.ts +183 -0
- package/src/tts/__tests__/provider-registry.test.ts +90 -0
- package/src/tts/provider-catalog.ts +219 -0
- package/src/tts/provider-registry.ts +73 -0
- package/src/tts/providers/deepgram-provider.ts +219 -0
- package/src/tts/providers/elevenlabs-provider.ts +211 -0
- package/src/tts/providers/fish-audio-provider.ts +183 -0
- package/src/tts/providers/index.ts +44 -0
- package/src/tts/providers/register-builtins.ts +130 -0
- package/src/tts/providers/xai-provider.ts +224 -0
- package/src/tts/synthesize-text.ts +110 -0
- package/src/tts/tts-config-resolver.ts +78 -0
- package/src/tts/types.ts +199 -0
- package/src/types/onboarding-context.ts +7 -0
- package/src/types/tar-stream.d.ts +66 -0
- package/src/util/abort-reasons.ts +58 -0
- package/src/util/device-id.ts +32 -16
- package/src/util/errors.ts +9 -1
- package/src/util/json.ts +17 -0
- package/src/util/platform.ts +56 -12
- package/src/util/pricing.ts +78 -5
- package/src/util/spawn.ts +1 -1
- package/src/util/truncate.ts +4 -2
- package/src/util/unicode.ts +201 -0
- package/src/version.ts +19 -24
- package/src/watcher/engine.ts +24 -1
- package/src/watcher/providers/google-calendar.ts +134 -8
- package/src/watcher/providers/outlook-calendar.ts +42 -2
- package/src/watcher/watcher-store.ts +31 -0
- package/src/workspace/git-service.ts +23 -4
- package/src/workspace/migrations/003-seed-device-id.ts +9 -3
- package/src/workspace/migrations/017-seed-persona-dirs.ts +68 -4
- package/src/workspace/migrations/029-seed-pkb.ts +1 -1
- package/src/workspace/migrations/031-drop-user-md.ts +317 -0
- package/src/workspace/migrations/031-llm-log-retention-zero-to-null.ts +73 -0
- package/src/workspace/migrations/032-tts-provider-unification.ts +227 -0
- package/src/workspace/migrations/033-stt-service-explicit-config.ts +122 -0
- package/src/workspace/migrations/034-remove-calls-voice-transcription-provider.ts +215 -0
- package/src/workspace/migrations/035-seed-slack-channel-persona.ts +50 -0
- package/src/workspace/migrations/036-update-pkb-index-bar.ts +37 -0
- package/src/workspace/migrations/037-create-meets-dir.ts +61 -0
- package/src/workspace/migrations/038-unify-llm-callsite-configs.ts +516 -0
- package/src/workspace/migrations/039-drop-legacy-llm-keys.ts +171 -0
- package/src/workspace/migrations/040-seed-latency-callsite-defaults.ts +154 -0
- package/src/workspace/migrations/041-backfill-google-gmail-settings-scope.ts +57 -0
- package/src/workspace/migrations/042-fix-backfill-google-gmail-settings-scope.ts +70 -0
- package/src/workspace/migrations/043-release-notes-latex-rendering.ts +75 -0
- package/src/workspace/migrations/044-bump-stale-provider-stream-timeout.ts +51 -0
- package/src/workspace/migrations/045-release-notes-meet-avatar.ts +130 -0
- package/src/workspace/migrations/AGENTS.md +1 -1
- package/src/workspace/migrations/registry.ts +32 -0
- package/src/workspace/provider-commit-message-generator.ts +19 -38
- package/src/workspace/top-level-renderer.ts +13 -1
- package/src/workspace/turn-commit.ts +31 -0
- package/src/__tests__/email-cli.test.ts +0 -297
- package/src/__tests__/email-service-config-fallback.test.ts +0 -102
- package/src/__tests__/outlook-attachments.test.ts +0 -301
- package/src/__tests__/outlook-automation-tools.test.ts +0 -425
- package/src/__tests__/outlook-categories.test.ts +0 -212
- package/src/__tests__/outlook-compose-tools.test.ts +0 -325
- package/src/__tests__/outlook-declutter-tools.test.ts +0 -585
- package/src/__tests__/outlook-follow-up.test.ts +0 -196
- package/src/__tests__/outlook-trash.test.ts +0 -77
- package/src/__tests__/outlook-unsubscribe.test.ts +0 -250
- package/src/__tests__/update-bulletin-format.test.ts +0 -122
- package/src/__tests__/update-bulletin-state.test.ts +0 -135
- package/src/__tests__/update-bulletin.test.ts +0 -277
- package/src/__tests__/update-template-contract.test.ts +0 -29
- package/src/cli/commands/browser-relay.ts +0 -466
- package/src/cli/commands/doctor.ts +0 -341
- package/src/config/bundled-skills/browser/SKILL.md +0 -63
- package/src/config/bundled-skills/browser/TOOLS.json +0 -393
- package/src/config/bundled-skills/browser/tools/browser-click.ts +0 -12
- package/src/config/bundled-skills/browser/tools/browser-close.ts +0 -12
- package/src/config/bundled-skills/browser/tools/browser-extract.ts +0 -12
- package/src/config/bundled-skills/browser/tools/browser-fill-credential.ts +0 -12
- package/src/config/bundled-skills/browser/tools/browser-hover.ts +0 -12
- package/src/config/bundled-skills/browser/tools/browser-navigate.ts +0 -12
- package/src/config/bundled-skills/browser/tools/browser-press-key.ts +0 -12
- package/src/config/bundled-skills/browser/tools/browser-screenshot.ts +0 -12
- package/src/config/bundled-skills/browser/tools/browser-scroll.ts +0 -12
- package/src/config/bundled-skills/browser/tools/browser-select-option.ts +0 -12
- package/src/config/bundled-skills/browser/tools/browser-snapshot.ts +0 -12
- package/src/config/bundled-skills/browser/tools/browser-type.ts +0 -12
- package/src/config/bundled-skills/browser/tools/browser-wait-for-download.ts +0 -32
- package/src/config/bundled-skills/browser/tools/browser-wait-for.ts +0 -12
- package/src/config/bundled-skills/chatgpt-import/SKILL.md +0 -27
- package/src/config/bundled-skills/chatgpt-import/TOOLS.json +0 -27
- package/src/config/bundled-skills/chatgpt-import/tools/chatgpt-import.ts +0 -378
- package/src/config/bundled-skills/gmail/SKILL.md +0 -175
- package/src/config/bundled-skills/gmail/TOOLS.json +0 -558
- package/src/config/bundled-skills/gmail/tools/gmail-archive.ts +0 -149
- package/src/config/bundled-skills/gmail/tools/gmail-attachments.ts +0 -112
- package/src/config/bundled-skills/gmail/tools/gmail-draft.ts +0 -44
- package/src/config/bundled-skills/gmail/tools/gmail-filters.ts +0 -81
- package/src/config/bundled-skills/gmail/tools/gmail-follow-up.ts +0 -108
- package/src/config/bundled-skills/gmail/tools/gmail-forward.ts +0 -146
- package/src/config/bundled-skills/gmail/tools/gmail-label.ts +0 -53
- package/src/config/bundled-skills/gmail/tools/gmail-outreach-scan.ts +0 -220
- package/src/config/bundled-skills/gmail/tools/gmail-send-draft.ts +0 -26
- package/src/config/bundled-skills/gmail/tools/gmail-sender-digest.ts +0 -251
- package/src/config/bundled-skills/gmail/tools/gmail-trash.ts +0 -29
- package/src/config/bundled-skills/gmail/tools/gmail-unsubscribe.ts +0 -122
- package/src/config/bundled-skills/gmail/tools/gmail-vacation.ts +0 -67
- package/src/config/bundled-skills/gmail/tools/scan-result-store.ts +0 -100
- package/src/config/bundled-skills/gmail/tools/shared.ts +0 -47
- package/src/config/bundled-skills/google-calendar/SKILL.md +0 -51
- package/src/config/bundled-skills/google-calendar/TOOLS.json +0 -226
- package/src/config/bundled-skills/google-calendar/calendar-client.ts +0 -223
- package/src/config/bundled-skills/google-calendar/tools/calendar-check-availability.ts +0 -27
- package/src/config/bundled-skills/google-calendar/tools/calendar-create-event.ts +0 -48
- package/src/config/bundled-skills/google-calendar/tools/calendar-get-event.ts +0 -19
- package/src/config/bundled-skills/google-calendar/tools/calendar-list-events.ts +0 -36
- package/src/config/bundled-skills/google-calendar/tools/calendar-rsvp.ts +0 -58
- package/src/config/bundled-skills/google-calendar/tools/shared.ts +0 -17
- package/src/config/bundled-skills/google-calendar/types.ts +0 -97
- package/src/config/bundled-skills/outlook/SKILL.md +0 -196
- package/src/config/bundled-skills/outlook/TOOLS.json +0 -530
- package/src/config/bundled-skills/outlook/tools/outlook-attachments.ts +0 -85
- package/src/config/bundled-skills/outlook/tools/outlook-categories.ts +0 -77
- package/src/config/bundled-skills/outlook/tools/outlook-draft.ts +0 -84
- package/src/config/bundled-skills/outlook/tools/outlook-follow-up.ts +0 -94
- package/src/config/bundled-skills/outlook/tools/outlook-forward.ts +0 -49
- package/src/config/bundled-skills/outlook/tools/outlook-outreach-scan.ts +0 -237
- package/src/config/bundled-skills/outlook/tools/outlook-rules.ts +0 -161
- package/src/config/bundled-skills/outlook/tools/outlook-send-draft.ts +0 -32
- package/src/config/bundled-skills/outlook/tools/outlook-sender-digest.ts +0 -272
- package/src/config/bundled-skills/outlook/tools/outlook-trash.ts +0 -29
- package/src/config/bundled-skills/outlook/tools/outlook-unsubscribe.ts +0 -129
- package/src/config/bundled-skills/outlook/tools/outlook-vacation.ts +0 -87
- package/src/config/bundled-skills/outlook/tools/shared.ts +0 -20
- package/src/config/bundled-skills/outlook-calendar/SKILL.md +0 -51
- package/src/config/bundled-skills/outlook-calendar/TOOLS.json +0 -221
- package/src/config/bundled-skills/outlook-calendar/calendar-client.ts +0 -252
- package/src/config/bundled-skills/outlook-calendar/tools/outlook-calendar-check-availability.ts +0 -53
- package/src/config/bundled-skills/outlook-calendar/tools/outlook-calendar-create-event.ts +0 -74
- package/src/config/bundled-skills/outlook-calendar/tools/outlook-calendar-get-event.ts +0 -18
- package/src/config/bundled-skills/outlook-calendar/tools/outlook-calendar-list-events.ts +0 -46
- package/src/config/bundled-skills/outlook-calendar/tools/outlook-calendar-rsvp.ts +0 -36
- package/src/config/bundled-skills/outlook-calendar/tools/shared.ts +0 -17
- package/src/config/bundled-skills/outlook-calendar/types.ts +0 -120
- package/src/config/bundled-skills/slack/SKILL.md +0 -107
- package/src/config/bundled-skills/tasks/SKILL.md +0 -37
- package/src/config/bundled-skills/tasks/TOOLS.json +0 -353
- package/src/config/bundled-skills/tasks/icon.svg +0 -34
- package/src/config/bundled-skills/tasks/tools/task-delete.ts +0 -12
- package/src/config/bundled-skills/tasks/tools/task-list-add.ts +0 -12
- package/src/config/bundled-skills/tasks/tools/task-list-remove.ts +0 -12
- package/src/config/bundled-skills/tasks/tools/task-list-show.ts +0 -12
- package/src/config/bundled-skills/tasks/tools/task-list-update.ts +0 -12
- package/src/config/bundled-skills/tasks/tools/task-list.ts +0 -12
- package/src/config/bundled-skills/tasks/tools/task-queue-run.ts +0 -12
- package/src/config/bundled-skills/tasks/tools/task-run.ts +0 -12
- package/src/config/bundled-skills/tasks/tools/task-save.ts +0 -12
- package/src/config/bundled-skills/watcher/SKILL.md +0 -31
- package/src/config/bundled-skills/watcher/TOOLS.json +0 -167
- package/src/config/bundled-skills/watcher/tools/watcher-create.ts +0 -12
- package/src/config/bundled-skills/watcher/tools/watcher-delete.ts +0 -12
- package/src/config/bundled-skills/watcher/tools/watcher-digest.ts +0 -12
- package/src/config/bundled-skills/watcher/tools/watcher-list.ts +0 -12
- package/src/config/bundled-skills/watcher/tools/watcher-update.ts +0 -12
- package/src/email/guardrails.ts +0 -221
- package/src/email/provider.ts +0 -117
- package/src/email/providers/agentmail.ts +0 -361
- package/src/email/providers/index.ts +0 -65
- package/src/email/service.ts +0 -384
- package/src/email/types.ts +0 -126
- package/src/prompts/templates/UPDATES.md +0 -38
- package/src/prompts/templates/USER.md +0 -13
- package/src/prompts/update-bulletin-format.ts +0 -68
- package/src/prompts/update-bulletin-state.ts +0 -58
- package/src/prompts/update-bulletin-template-path.ts +0 -13
- package/src/prompts/update-bulletin.ts +0 -128
- package/src/providers/speech-to-text/types.ts +0 -17
- package/src/runtime/routes/browser-cdp-routes.ts +0 -229
- package/src/shared/provider-env-vars.ts +0 -19
- package/src/tools/watcher/create.ts +0 -86
- package/src/tools/watcher/delete.ts +0 -36
- package/src/tools/watcher/digest.ts +0 -54
- package/src/tools/watcher/list.ts +0 -83
- package/src/tools/watcher/update.ts +0 -71
|
@@ -1,7 +1,10 @@
|
|
|
1
|
+
import { getConfig } from "../../config/loader.js";
|
|
1
2
|
import type { ImageContent } from "../../providers/types.js";
|
|
2
3
|
import { getLogger } from "../../util/logger.js";
|
|
3
4
|
import { truncate } from "../../util/truncate.js";
|
|
5
|
+
import { safeStringSlice } from "../../util/unicode.js";
|
|
4
6
|
import { credentialBroker } from "../credentials/broker.js";
|
|
7
|
+
import { BROWSER_FILL_CAPABILITY } from "../credentials/tool-policy.js";
|
|
5
8
|
import {
|
|
6
9
|
isPrivateOrLocalHost,
|
|
7
10
|
parseUrl,
|
|
@@ -17,12 +20,22 @@ import {
|
|
|
17
20
|
} from "./auth-detector.js";
|
|
18
21
|
import type { RouteHandler } from "./browser-manager.js";
|
|
19
22
|
import { browserManager } from "./browser-manager.js";
|
|
23
|
+
import { type BrowserMode, normalizeBrowserMode } from "./browser-mode.js";
|
|
24
|
+
import { BROWSER_MODE } from "./browser-mode-constants.js";
|
|
20
25
|
import {
|
|
21
26
|
ensureScreencast,
|
|
22
27
|
getSender,
|
|
23
28
|
stopAllScreencasts,
|
|
24
29
|
stopBrowserScreencast,
|
|
25
30
|
} from "./browser-screencast.js";
|
|
31
|
+
import {
|
|
32
|
+
BROWSER_STATUS_INPUT_FIELD,
|
|
33
|
+
BROWSER_STATUS_MODE,
|
|
34
|
+
BROWSER_STATUS_MODES,
|
|
35
|
+
type BrowserStatusMode,
|
|
36
|
+
CDP_INSPECT_STATUS_DISCOVERY_CODE,
|
|
37
|
+
EXTENSION_STATUS_ERROR_MARKER,
|
|
38
|
+
} from "./browser-status-constants.js";
|
|
26
39
|
import {
|
|
27
40
|
formatAxSnapshot,
|
|
28
41
|
transformAxTree,
|
|
@@ -45,8 +58,18 @@ import {
|
|
|
45
58
|
waitForSelector as cdpWaitForSelector,
|
|
46
59
|
waitForText as cdpWaitForText,
|
|
47
60
|
} from "./cdp-client/cdp-dom-helpers.js";
|
|
48
|
-
import {
|
|
49
|
-
import
|
|
61
|
+
import { CdpError } from "./cdp-client/errors.js";
|
|
62
|
+
import {
|
|
63
|
+
buildCandidateList,
|
|
64
|
+
getCdpClient,
|
|
65
|
+
isDesktopAutoCooldownActive,
|
|
66
|
+
} from "./cdp-client/factory.js";
|
|
67
|
+
import type {
|
|
68
|
+
AttemptDiagnostic,
|
|
69
|
+
CdpClient,
|
|
70
|
+
CdpClientKind,
|
|
71
|
+
} from "./cdp-client/types.js";
|
|
72
|
+
import { checkBrowserRuntime } from "./runtime-check.js";
|
|
50
73
|
|
|
51
74
|
const log = getLogger("headless-browser");
|
|
52
75
|
|
|
@@ -60,6 +83,40 @@ export const MAX_WAIT_MS = 30_000;
|
|
|
60
83
|
|
|
61
84
|
export const MAX_EXTRACT_LENGTH = 50_000;
|
|
62
85
|
|
|
86
|
+
type StatusCheckMode = BrowserStatusMode;
|
|
87
|
+
|
|
88
|
+
const MODE_TRADEOFFS: Record<StatusCheckMode, string[]> = {
|
|
89
|
+
[BROWSER_STATUS_MODE.EXTENSION]: [
|
|
90
|
+
"This is the preferred approach for all things browser-use.",
|
|
91
|
+
"It requires a one-time install of the Vellum Assistant Chrome Extension.",
|
|
92
|
+
"More secure than relying on Chrome's native remote debugging functionality.",
|
|
93
|
+
"Requires the Vellum extension to be paired and actively connected.",
|
|
94
|
+
],
|
|
95
|
+
[BROWSER_STATUS_MODE.CDP_INSPECT]: [
|
|
96
|
+
"This is the second-best approach for all things browser-use, after the native Vellum Assistant Chrome Extension.",
|
|
97
|
+
"It requires Chrome version 146 or greater",
|
|
98
|
+
"It requires toggling on remote debugging in Chrome Settings",
|
|
99
|
+
"It's prone to phishing attacks from other local processes that may try to do their own remote debugging.",
|
|
100
|
+
],
|
|
101
|
+
[BROWSER_STATUS_MODE.LOCAL]: [
|
|
102
|
+
"The least-preferred approach for all things browser-use.",
|
|
103
|
+
"Considered a last-resort fallback if the user has not installed the Chrome Extension or enabled remote debugging in Chrome, and has indicated that they do not want to.",
|
|
104
|
+
"Does not use the user's existing browser profile, so sessions/cookies may differ.",
|
|
105
|
+
"Requires that Playwright and Chromium are installed on the host machine,",
|
|
106
|
+
],
|
|
107
|
+
};
|
|
108
|
+
|
|
109
|
+
interface BrowserStatusModeResult {
|
|
110
|
+
mode: StatusCheckMode;
|
|
111
|
+
available: boolean;
|
|
112
|
+
verified: "active_probe" | "preflight";
|
|
113
|
+
autoCandidate: boolean;
|
|
114
|
+
summary: string;
|
|
115
|
+
userActions: string[];
|
|
116
|
+
tradeoffs: string[];
|
|
117
|
+
details: Record<string, unknown>;
|
|
118
|
+
}
|
|
119
|
+
|
|
63
120
|
/**
|
|
64
121
|
* IIFE evaluated inside the page via `Runtime.evaluate` to auto-dismiss
|
|
65
122
|
* common blocker modals (regulatory notices, cookie banners) that
|
|
@@ -99,6 +156,337 @@ export const EXTRACT_LINKS_EXPRESSION = `
|
|
|
99
156
|
})()
|
|
100
157
|
`;
|
|
101
158
|
|
|
159
|
+
// ── browser_mode parsing ─────────────────────────────────────────────
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* Parse the `browser_mode` field from a tool input map. Returns either
|
|
163
|
+
* a normalized {@link BrowserMode} or a pre-formatted error string
|
|
164
|
+
* suitable for returning directly in a tool response.
|
|
165
|
+
*
|
|
166
|
+
* When the value is absent, undefined, or empty the default `"auto"`
|
|
167
|
+
* is returned. Invalid values produce a descriptive error listing
|
|
168
|
+
* accepted values and aliases.
|
|
169
|
+
*/
|
|
170
|
+
export function parseBrowserMode(
|
|
171
|
+
input: Record<string, unknown>,
|
|
172
|
+
): { ok: true; mode: BrowserMode } | { ok: false; error: string } {
|
|
173
|
+
const raw = input.browser_mode;
|
|
174
|
+
const result = normalizeBrowserMode(raw);
|
|
175
|
+
if ("error" in result) {
|
|
176
|
+
return { ok: false, error: `Error: ${result.error}` };
|
|
177
|
+
}
|
|
178
|
+
return { ok: true, mode: result.mode };
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
// ── Mode-selection failure formatter ─────────────────────────────────
|
|
182
|
+
|
|
183
|
+
/**
|
|
184
|
+
* Remediation hints keyed by (candidateKind, discoveryCode | errorCode).
|
|
185
|
+
* Discovery codes come from DevToolsDiscoveryError; error codes come
|
|
186
|
+
* from CdpError. The formatter walks these in priority order: exact
|
|
187
|
+
* (kind, discoveryCode) first, then (kind, errorCode), then a generic
|
|
188
|
+
* per-kind fallback.
|
|
189
|
+
*/
|
|
190
|
+
const REMEDIATION_HINTS: Record<string, string[]> = {
|
|
191
|
+
// Extension backend
|
|
192
|
+
"extension:transport_error": [
|
|
193
|
+
"Ensure the Vellum browser extension is installed and enabled.",
|
|
194
|
+
"Check that the extension WebSocket connection is active (extension popup → status).",
|
|
195
|
+
"Try reconnecting the extension or reloading the extension service worker.",
|
|
196
|
+
],
|
|
197
|
+
// cdp-inspect backend — discovery-level failures
|
|
198
|
+
"cdp-inspect:unreachable": [
|
|
199
|
+
"Ensure that Chrome is on version 146 or higher by going to chrome://settings/help.",
|
|
200
|
+
'Ensure that you have toggled on "Allow remote debugging for this browser instance" by going to chrome://inspect/#remote-debugging',
|
|
201
|
+
"Verify no firewall or antivirus is blocking localhost:9222.",
|
|
202
|
+
],
|
|
203
|
+
"cdp-inspect:non_chrome": [
|
|
204
|
+
"The process listening on the configured port is not Chrome/Chromium.",
|
|
205
|
+
"Check if another application (dev server, proxy) is using port 9222.",
|
|
206
|
+
"Ensure Chrome is launched with --remote-debugging-port=9222.",
|
|
207
|
+
],
|
|
208
|
+
"cdp-inspect:timeout": [
|
|
209
|
+
"Chrome DevTools endpoint did not respond within the probe timeout.",
|
|
210
|
+
"Ensure Chrome is running and listening on the configured port.",
|
|
211
|
+
"Try increasing hostBrowser.cdpInspect.probeTimeoutMs in config.",
|
|
212
|
+
],
|
|
213
|
+
"cdp-inspect:no_targets": [
|
|
214
|
+
"Chrome is reachable but has no open page targets.",
|
|
215
|
+
"Open at least one browser tab, then retry.",
|
|
216
|
+
],
|
|
217
|
+
"cdp-inspect:non_loopback": [
|
|
218
|
+
"CDP inspect only allows loopback hosts (localhost, 127.0.0.1, ::1).",
|
|
219
|
+
"Update hostBrowser.cdpInspect.host in config to a loopback address.",
|
|
220
|
+
],
|
|
221
|
+
"cdp-inspect:transport_error": [
|
|
222
|
+
"CDP endpoint unreachable. Ensure Chrome is running with --remote-debugging-port.",
|
|
223
|
+
"Verify the configured host:port matches Chrome's DevTools listener.",
|
|
224
|
+
"Consider using browser_mode: 'extension' or 'local' as an alternative.",
|
|
225
|
+
],
|
|
226
|
+
// Local/Playwright backend
|
|
227
|
+
"local:transport_error": [
|
|
228
|
+
"The local Playwright-managed browser failed to start or connect.",
|
|
229
|
+
"Check that the Playwright browser binary is downloaded (bun run install).",
|
|
230
|
+
"Try closing any stale Chromium processes and retrying.",
|
|
231
|
+
],
|
|
232
|
+
};
|
|
233
|
+
|
|
234
|
+
/**
|
|
235
|
+
* Build a human-readable, tool-response-ready error string from a
|
|
236
|
+
* pinned-mode failure. Includes:
|
|
237
|
+
* - the requested mode
|
|
238
|
+
* - ordered attempted modes with exact failure reasons
|
|
239
|
+
* - a remediation checklist tailored by backend and failure code
|
|
240
|
+
*
|
|
241
|
+
* Exported for testing.
|
|
242
|
+
*/
|
|
243
|
+
export function formatModeSelectionFailure(
|
|
244
|
+
requestedMode: BrowserMode,
|
|
245
|
+
error: CdpError,
|
|
246
|
+
): string {
|
|
247
|
+
const lines: string[] = [];
|
|
248
|
+
lines.push(`Error: Browser mode "${requestedMode}" failed.`);
|
|
249
|
+
lines.push("");
|
|
250
|
+
|
|
251
|
+
const diagnostics: readonly AttemptDiagnostic[] =
|
|
252
|
+
error.attemptDiagnostics ?? [];
|
|
253
|
+
|
|
254
|
+
if (diagnostics.length > 0) {
|
|
255
|
+
lines.push("Attempted backends:");
|
|
256
|
+
for (const diag of diagnostics) {
|
|
257
|
+
const status =
|
|
258
|
+
diag.stage === "success" ? "OK" : `FAILED at ${diag.stage}`;
|
|
259
|
+
lines.push(` - ${diag.candidateKind}: ${status}`);
|
|
260
|
+
if (diag.errorMessage) {
|
|
261
|
+
lines.push(` Reason: ${diag.errorMessage}`);
|
|
262
|
+
}
|
|
263
|
+
if (diag.discoveryCode) {
|
|
264
|
+
lines.push(` Discovery code: ${diag.discoveryCode}`);
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
lines.push("");
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
// Collect remediation hints
|
|
271
|
+
const hints = collectRemediationHints(diagnostics, error);
|
|
272
|
+
if (hints.length > 0) {
|
|
273
|
+
lines.push("Remediation:");
|
|
274
|
+
for (const hint of hints) {
|
|
275
|
+
lines.push(` - ${hint}`);
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
return lines.join("\n");
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
/**
|
|
283
|
+
* Gather remediation hints based on attempt diagnostics and the error.
|
|
284
|
+
* Walks each diagnostic and looks up hints by (kind, discoveryCode),
|
|
285
|
+
* then (kind, errorCode), then generic kind-level fallback.
|
|
286
|
+
*/
|
|
287
|
+
function collectRemediationHints(
|
|
288
|
+
diagnostics: readonly AttemptDiagnostic[],
|
|
289
|
+
error: CdpError,
|
|
290
|
+
): string[] {
|
|
291
|
+
const seen = new Set<string>();
|
|
292
|
+
const hints: string[] = [];
|
|
293
|
+
|
|
294
|
+
const addHints = (key: string) => {
|
|
295
|
+
const list = REMEDIATION_HINTS[key];
|
|
296
|
+
if (!list) return;
|
|
297
|
+
for (const hint of list) {
|
|
298
|
+
if (!seen.has(hint)) {
|
|
299
|
+
seen.add(hint);
|
|
300
|
+
hints.push(hint);
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
};
|
|
304
|
+
|
|
305
|
+
for (const diag of diagnostics) {
|
|
306
|
+
if (diag.stage === "success") continue;
|
|
307
|
+
if (diag.discoveryCode) {
|
|
308
|
+
addHints(`${diag.candidateKind}:${diag.discoveryCode}`);
|
|
309
|
+
}
|
|
310
|
+
if (diag.errorCode) {
|
|
311
|
+
addHints(`${diag.candidateKind}:${diag.errorCode}`);
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
// Fallback: if no diagnostics but we have a top-level error, use
|
|
316
|
+
// the error code with a generic candidate kind derived from the mode.
|
|
317
|
+
if (diagnostics.length === 0 && error.code) {
|
|
318
|
+
// Try to infer the candidate kind from the error message
|
|
319
|
+
for (const kind of BROWSER_STATUS_MODES) {
|
|
320
|
+
if (error.message.toLowerCase().includes(kind)) {
|
|
321
|
+
addHints(`${kind}:${error.code}`);
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
return hints;
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
/**
|
|
330
|
+
* Detect the common extension CDP failure where the active tab is a
|
|
331
|
+
* restricted Chrome internal page (e.g. `chrome://newtab`).
|
|
332
|
+
*/
|
|
333
|
+
function isRestrictedChromePageProbeError(error: CdpError): boolean {
|
|
334
|
+
return error.message.toLowerCase().includes("chrome://");
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
/**
|
|
338
|
+
* Parse browser_mode from input and acquire a CdpClient. Returns
|
|
339
|
+
* either a `{ cdp, browserMode }` pair on success or a pre-formatted
|
|
340
|
+
* `{ errorResult }` on failure (invalid mode or pinned-mode
|
|
341
|
+
* precondition not met).
|
|
342
|
+
*
|
|
343
|
+
* This is the single integration point for all CDP-backed tool
|
|
344
|
+
* functions. Using it ensures every tool:
|
|
345
|
+
* - normalizes aliases (`cdp-debugger` -> `cdp-inspect`, etc.)
|
|
346
|
+
* - passes the mode preference to the factory
|
|
347
|
+
* - surfaces a remediation-rich error on pinned-mode failures
|
|
348
|
+
*
|
|
349
|
+
* Per-conversation stickiness: when the incoming `browser_mode` is
|
|
350
|
+
* `"auto"` and the conversation has already resolved to a backend
|
|
351
|
+
* kind on a prior call, the factory is pinned to that kind instead
|
|
352
|
+
* of re-running the auto priority list. This prevents
|
|
353
|
+
* `browser_navigate` (e.g. pinned to `local`) and `browser_screenshot`
|
|
354
|
+
* (default auto) in the same conversation from landing on different
|
|
355
|
+
* Chrome instances. Explicit non-auto modes override and update the
|
|
356
|
+
* memo; teardown via browser_close / browser_detach clears it.
|
|
357
|
+
*
|
|
358
|
+
* The returned client is wrapped so its first successful `send()`
|
|
359
|
+
* writes the resolved kind back to the conversation memo.
|
|
360
|
+
*/
|
|
361
|
+
export function acquireCdpClientWithMode(
|
|
362
|
+
input: Record<string, unknown>,
|
|
363
|
+
context: ToolContext,
|
|
364
|
+
):
|
|
365
|
+
| {
|
|
366
|
+
cdp: ReturnType<typeof getCdpClient>;
|
|
367
|
+
browserMode: BrowserMode;
|
|
368
|
+
errorResult?: never;
|
|
369
|
+
}
|
|
370
|
+
| { cdp?: never; browserMode?: never; errorResult: ToolExecutionResult } {
|
|
371
|
+
const modeResult = parseBrowserMode(input);
|
|
372
|
+
if (!modeResult.ok) {
|
|
373
|
+
return {
|
|
374
|
+
errorResult: { content: modeResult.error, isError: true },
|
|
375
|
+
};
|
|
376
|
+
}
|
|
377
|
+
const browserMode = modeResult.mode;
|
|
378
|
+
|
|
379
|
+
const rememberedKind = browserManager.getPreferredBackendKind(
|
|
380
|
+
context.conversationId,
|
|
381
|
+
);
|
|
382
|
+
const effectiveMode: BrowserMode =
|
|
383
|
+
browserMode === "auto" && rememberedKind !== null
|
|
384
|
+
? rememberedKind
|
|
385
|
+
: browserMode;
|
|
386
|
+
|
|
387
|
+
try {
|
|
388
|
+
const raw = getCdpClient(context, { mode: effectiveMode });
|
|
389
|
+
const cdp = wrapWithKindMemo(raw, context.conversationId);
|
|
390
|
+
return { cdp, browserMode };
|
|
391
|
+
} catch (err) {
|
|
392
|
+
// Sticky-mode fallback: the caller requested "auto" but we pinned to
|
|
393
|
+
// a remembered backend kind that has since become unavailable. Drop
|
|
394
|
+
// the stale memo and retry with fresh auto selection so a dead
|
|
395
|
+
// sticky preference doesn't surface as a hard failure.
|
|
396
|
+
if (browserMode === "auto" && effectiveMode !== "auto") {
|
|
397
|
+
browserManager.clearPreferredBackendKind(context.conversationId);
|
|
398
|
+
try {
|
|
399
|
+
const raw = getCdpClient(context, { mode: "auto" });
|
|
400
|
+
const cdp = wrapWithKindMemo(raw, context.conversationId);
|
|
401
|
+
return { cdp, browserMode };
|
|
402
|
+
} catch (retryErr) {
|
|
403
|
+
if (retryErr instanceof CdpError) {
|
|
404
|
+
return {
|
|
405
|
+
errorResult: {
|
|
406
|
+
content: formatModeSelectionFailure("auto", retryErr),
|
|
407
|
+
isError: true,
|
|
408
|
+
},
|
|
409
|
+
};
|
|
410
|
+
}
|
|
411
|
+
throw retryErr;
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
if (err instanceof CdpError && browserMode !== "auto") {
|
|
415
|
+
return {
|
|
416
|
+
errorResult: {
|
|
417
|
+
content: formatModeSelectionFailure(browserMode, err),
|
|
418
|
+
isError: true,
|
|
419
|
+
},
|
|
420
|
+
};
|
|
421
|
+
}
|
|
422
|
+
throw err;
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
/**
|
|
427
|
+
* Wrap a {@link ScopedCdpClient} so the first successful `send()`
|
|
428
|
+
* records the resolved backend kind in the conversation's
|
|
429
|
+
* `preferredBackendKinds` memo. Subsequent sends are no-ops for the
|
|
430
|
+
* memo; dispose() delegates to the underlying client.
|
|
431
|
+
*/
|
|
432
|
+
function wrapWithKindMemo(
|
|
433
|
+
inner: ReturnType<typeof getCdpClient>,
|
|
434
|
+
conversationId: string,
|
|
435
|
+
): ReturnType<typeof getCdpClient> {
|
|
436
|
+
let recorded = false;
|
|
437
|
+
return {
|
|
438
|
+
get kind() {
|
|
439
|
+
return inner.kind;
|
|
440
|
+
},
|
|
441
|
+
conversationId: inner.conversationId,
|
|
442
|
+
async send<T = unknown>(
|
|
443
|
+
method: string,
|
|
444
|
+
params?: Record<string, unknown>,
|
|
445
|
+
signal?: AbortSignal,
|
|
446
|
+
): Promise<T> {
|
|
447
|
+
const result = await inner.send<T>(method, params, signal);
|
|
448
|
+
if (!recorded) {
|
|
449
|
+
browserManager.setPreferredBackendKind(conversationId, inner.kind);
|
|
450
|
+
recorded = true;
|
|
451
|
+
}
|
|
452
|
+
return result;
|
|
453
|
+
},
|
|
454
|
+
dispose(): void {
|
|
455
|
+
inner.dispose();
|
|
456
|
+
},
|
|
457
|
+
};
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
// ── CDP error diagnostics helper ─────────────────────────────────────
|
|
461
|
+
|
|
462
|
+
/**
|
|
463
|
+
* Check whether a caught error is a {@link CdpError} carrying
|
|
464
|
+
* {@link AttemptDiagnostic attempt diagnostics} from the factory's
|
|
465
|
+
* failover walk. When the browser_mode is pinned (not "auto") and
|
|
466
|
+
* diagnostics are present, format the error with the full remediation
|
|
467
|
+
* checklist via {@link formatModeSelectionFailure}. Otherwise return
|
|
468
|
+
* `null` so the caller falls through to its generic error message.
|
|
469
|
+
*
|
|
470
|
+
* This handles the case where pinned-mode unavailability is surfaced
|
|
471
|
+
* on the first `cdp.send()` (via `sendWithFailover`) rather than
|
|
472
|
+
* during client construction (which `acquireCdpClientWithMode` already
|
|
473
|
+
* covers).
|
|
474
|
+
*/
|
|
475
|
+
function formatCdpSendDiagnostics(
|
|
476
|
+
err: unknown,
|
|
477
|
+
browserMode: BrowserMode,
|
|
478
|
+
): string | null {
|
|
479
|
+
if (
|
|
480
|
+
err instanceof CdpError &&
|
|
481
|
+
browserMode !== "auto" &&
|
|
482
|
+
err.code === "transport_error" &&
|
|
483
|
+
err.attemptDiagnostics
|
|
484
|
+
) {
|
|
485
|
+
return formatModeSelectionFailure(browserMode, err);
|
|
486
|
+
}
|
|
487
|
+
return null;
|
|
488
|
+
}
|
|
489
|
+
|
|
102
490
|
// ── Shared element resolution ────────────────────────────────────────
|
|
103
491
|
|
|
104
492
|
/**
|
|
@@ -154,7 +542,7 @@ export function resolveElement(
|
|
|
154
542
|
}
|
|
155
543
|
return {
|
|
156
544
|
resolved: null,
|
|
157
|
-
error: `Error: element_id "${elementId}" not found. Run
|
|
545
|
+
error: `Error: element_id "${elementId}" not found. Run a snapshot first to get current element IDs.`,
|
|
158
546
|
};
|
|
159
547
|
}
|
|
160
548
|
|
|
@@ -174,6 +562,8 @@ export async function executeBrowserNavigate(
|
|
|
174
562
|
return { content: "Error: operation was cancelled", isError: true };
|
|
175
563
|
}
|
|
176
564
|
|
|
565
|
+
// Pre-flight URL validation runs before CDP acquisition so we fail
|
|
566
|
+
// fast on obviously invalid URLs without opening a browser session.
|
|
177
567
|
const parsedUrl = parseUrl(input.url);
|
|
178
568
|
if (!parsedUrl) {
|
|
179
569
|
return {
|
|
@@ -212,7 +602,10 @@ export async function executeBrowserNavigate(
|
|
|
212
602
|
}
|
|
213
603
|
}
|
|
214
604
|
|
|
215
|
-
|
|
605
|
+
// URL validation passed — acquire the CDP client.
|
|
606
|
+
const acquired = acquireCdpClientWithMode(input, context);
|
|
607
|
+
if (acquired.errorResult) return acquired.errorResult;
|
|
608
|
+
const { cdp, browserMode } = acquired;
|
|
216
609
|
|
|
217
610
|
// Screencast + handoff are Playwright-backed and only meaningful
|
|
218
611
|
// for the local sacrificial-profile path. On the extension path the
|
|
@@ -223,10 +616,12 @@ export async function executeBrowserNavigate(
|
|
|
223
616
|
await ensureScreencast(context.conversationId);
|
|
224
617
|
}
|
|
225
618
|
|
|
226
|
-
// SSRF route interception
|
|
227
|
-
//
|
|
228
|
-
//
|
|
229
|
-
//
|
|
619
|
+
// SSRF route interception uses the Playwright page.route() API to
|
|
620
|
+
// block redirect-time requests to private networks. This only works
|
|
621
|
+
// on the local path where Playwright manages the browser; on the
|
|
622
|
+
// extension/cdp-inspect paths, CDP navigates a different browser so
|
|
623
|
+
// the Playwright route handler would be a no-op. The post-navigation
|
|
624
|
+
// final URL check below provides defense-in-depth for all paths.
|
|
230
625
|
let routeHandler: RouteHandler | null = null;
|
|
231
626
|
let blockedUrl: string | null = null;
|
|
232
627
|
|
|
@@ -338,6 +733,53 @@ export async function executeBrowserNavigate(
|
|
|
338
733
|
{ timeoutMs: NAVIGATE_TIMEOUT_MS },
|
|
339
734
|
context.signal,
|
|
340
735
|
);
|
|
736
|
+
|
|
737
|
+
// Defense-in-depth: check the final URL after navigation completes.
|
|
738
|
+
// This catches redirect-based SSRF even when Playwright route
|
|
739
|
+
// interception is unavailable (e.g. extension-backed sessions where
|
|
740
|
+
// the CDP transport is separate from the Playwright page).
|
|
741
|
+
if (!allowPrivateNetwork) {
|
|
742
|
+
const finalParsed = parseUrl(finalUrl);
|
|
743
|
+
if (
|
|
744
|
+
finalParsed &&
|
|
745
|
+
(isPrivateOrLocalHost(finalParsed.hostname) ||
|
|
746
|
+
(
|
|
747
|
+
await resolveRequestAddress(
|
|
748
|
+
finalParsed.hostname,
|
|
749
|
+
resolveHostAddresses,
|
|
750
|
+
false,
|
|
751
|
+
)
|
|
752
|
+
).blockedAddress)
|
|
753
|
+
) {
|
|
754
|
+
// Navigate the page away from the private target to prevent
|
|
755
|
+
// follow-up operations (e.g. snapshot) from reading the
|
|
756
|
+
// already-loaded private content.
|
|
757
|
+
try {
|
|
758
|
+
await navigateAndWait(
|
|
759
|
+
cdp,
|
|
760
|
+
"about:blank",
|
|
761
|
+
{ timeoutMs: 3_000 },
|
|
762
|
+
context.signal,
|
|
763
|
+
);
|
|
764
|
+
} catch {
|
|
765
|
+
// Best-effort — if the reset fails, the CDP session will be
|
|
766
|
+
// disposed in the finally block anyway.
|
|
767
|
+
}
|
|
768
|
+
// Clean up the route handler before returning to avoid leaking
|
|
769
|
+
// a stale interception handler on the session page.
|
|
770
|
+
if (routeHandler) {
|
|
771
|
+
const page = await browserManager.getOrCreateSessionPage(
|
|
772
|
+
context.conversationId,
|
|
773
|
+
);
|
|
774
|
+
await page.unroute("**/*", routeHandler);
|
|
775
|
+
routeHandler = null;
|
|
776
|
+
}
|
|
777
|
+
return {
|
|
778
|
+
content: `Error: Navigation blocked. Final URL resolved to a local/private network target (${sanitizeUrlForOutput(finalParsed)}). Set allow_private_network=true if you explicitly need it.`,
|
|
779
|
+
isError: true,
|
|
780
|
+
};
|
|
781
|
+
}
|
|
782
|
+
}
|
|
341
783
|
if (navigationTimedOut) {
|
|
342
784
|
// If the page URL never changed from before navigation, the page
|
|
343
785
|
// never actually loaded - re-throw instead of reporting success.
|
|
@@ -353,7 +795,7 @@ export async function executeBrowserNavigate(
|
|
|
353
795
|
}
|
|
354
796
|
|
|
355
797
|
// Remove the Playwright route handler now that navigation is
|
|
356
|
-
// complete (local path only).
|
|
798
|
+
// complete (local path only — route interception is gated above).
|
|
357
799
|
if (routeHandler) {
|
|
358
800
|
const page = await browserManager.getOrCreateSessionPage(
|
|
359
801
|
context.conversationId,
|
|
@@ -417,7 +859,7 @@ export async function executeBrowserNavigate(
|
|
|
417
859
|
|
|
418
860
|
if (navigationTimedOut) {
|
|
419
861
|
lines.push(
|
|
420
|
-
`Note: Page is still loading (document.readyState timed out). The page should still be interactive -
|
|
862
|
+
`Note: Page is still loading (document.readyState timed out). The page should still be interactive - take a snapshot to check.`,
|
|
421
863
|
);
|
|
422
864
|
}
|
|
423
865
|
|
|
@@ -487,14 +929,12 @@ export async function executeBrowserNavigate(
|
|
|
487
929
|
lines.push("");
|
|
488
930
|
lines.push(formatAuthChallenge(postCaptchaAuth));
|
|
489
931
|
lines.push("");
|
|
932
|
+
lines.push("Handle this by interacting with the login form:");
|
|
490
933
|
lines.push(
|
|
491
|
-
"
|
|
934
|
+
"1. Take a snapshot to find the sign-in form elements",
|
|
492
935
|
);
|
|
493
936
|
lines.push(
|
|
494
|
-
"
|
|
495
|
-
);
|
|
496
|
-
lines.push(
|
|
497
|
-
"2. Use browser_fill_credential to fill email/password from credential_store",
|
|
937
|
+
"2. Use credential fill to enter email/password from credential_store",
|
|
498
938
|
);
|
|
499
939
|
lines.push(
|
|
500
940
|
"3. For email verification codes, use ui_show with a form to ask the user for the code mid-turn",
|
|
@@ -514,18 +954,14 @@ export async function executeBrowserNavigate(
|
|
|
514
954
|
}
|
|
515
955
|
} else {
|
|
516
956
|
// Login / 2FA / OAuth - the agent should handle these itself
|
|
517
|
-
// using browser
|
|
957
|
+
// using browser operations + credential_store. Don't hand off.
|
|
518
958
|
lines.push("");
|
|
519
959
|
lines.push(formatAuthChallenge(challenge));
|
|
520
960
|
lines.push("");
|
|
961
|
+
lines.push("Handle this by interacting with the login form:");
|
|
962
|
+
lines.push("1. Take a snapshot to find the sign-in form elements");
|
|
521
963
|
lines.push(
|
|
522
|
-
"
|
|
523
|
-
);
|
|
524
|
-
lines.push(
|
|
525
|
-
"1. Use browser_snapshot to find the sign-in form elements",
|
|
526
|
-
);
|
|
527
|
-
lines.push(
|
|
528
|
-
"2. Use browser_fill_credential to fill email/password from credential_store",
|
|
964
|
+
"2. Use credential fill to enter email/password from credential_store",
|
|
529
965
|
);
|
|
530
966
|
lines.push(
|
|
531
967
|
"3. For email verification codes, use ui_show with a form to ask the user for the code mid-turn",
|
|
@@ -563,6 +999,11 @@ export async function executeBrowserNavigate(
|
|
|
563
999
|
};
|
|
564
1000
|
}
|
|
565
1001
|
|
|
1002
|
+
const diagnosticMessage = formatCdpSendDiagnostics(err, browserMode);
|
|
1003
|
+
if (diagnosticMessage) {
|
|
1004
|
+
return { content: diagnosticMessage, isError: true };
|
|
1005
|
+
}
|
|
1006
|
+
|
|
566
1007
|
const msg = err instanceof Error ? err.message : String(err);
|
|
567
1008
|
log.error({ err, url: safeRequestedUrl }, "Navigation failed");
|
|
568
1009
|
return { content: `Error: Navigation failed: ${msg}`, isError: true };
|
|
@@ -571,13 +1012,16 @@ export async function executeBrowserNavigate(
|
|
|
571
1012
|
}
|
|
572
1013
|
}
|
|
573
1014
|
|
|
574
|
-
// ──
|
|
1015
|
+
// ── snapshot ─────────────────────────────────────────────────────────
|
|
575
1016
|
|
|
576
1017
|
export async function executeBrowserSnapshot(
|
|
577
1018
|
_input: Record<string, unknown>,
|
|
578
1019
|
context: ToolContext,
|
|
579
1020
|
): Promise<ToolExecutionResult> {
|
|
580
|
-
const
|
|
1021
|
+
const acquired = acquireCdpClientWithMode(_input, context);
|
|
1022
|
+
if (acquired.errorResult) return acquired.errorResult;
|
|
1023
|
+
const { cdp, browserMode } = acquired;
|
|
1024
|
+
|
|
581
1025
|
try {
|
|
582
1026
|
const currentUrl = await getCurrentUrl(cdp, context.signal);
|
|
583
1027
|
const title = await getPageTitle(cdp, context.signal);
|
|
@@ -608,6 +1052,10 @@ export async function executeBrowserSnapshot(
|
|
|
608
1052
|
isError: false,
|
|
609
1053
|
};
|
|
610
1054
|
} catch (err) {
|
|
1055
|
+
const diagnosticMessage = formatCdpSendDiagnostics(err, browserMode);
|
|
1056
|
+
if (diagnosticMessage) {
|
|
1057
|
+
return { content: diagnosticMessage, isError: true };
|
|
1058
|
+
}
|
|
611
1059
|
const msg = err instanceof Error ? err.message : String(err);
|
|
612
1060
|
log.error({ err }, "Snapshot failed");
|
|
613
1061
|
return { content: `Error: Snapshot failed: ${msg}`, isError: true };
|
|
@@ -622,9 +1070,11 @@ export async function executeBrowserScreenshot(
|
|
|
622
1070
|
input: Record<string, unknown>,
|
|
623
1071
|
context: ToolContext,
|
|
624
1072
|
): Promise<ToolExecutionResult> {
|
|
1073
|
+
const acquired = acquireCdpClientWithMode(input, context);
|
|
1074
|
+
if (acquired.errorResult) return acquired.errorResult;
|
|
1075
|
+
const { cdp, browserMode } = acquired;
|
|
625
1076
|
const fullPage = input.full_page === true;
|
|
626
1077
|
|
|
627
|
-
const cdp = getCdpClient(context);
|
|
628
1078
|
try {
|
|
629
1079
|
const buffer = await captureScreenshotJpeg(
|
|
630
1080
|
cdp,
|
|
@@ -650,6 +1100,10 @@ export async function executeBrowserScreenshot(
|
|
|
650
1100
|
contentBlocks: [imageBlock],
|
|
651
1101
|
};
|
|
652
1102
|
} catch (err) {
|
|
1103
|
+
const diagnosticMessage = formatCdpSendDiagnostics(err, browserMode);
|
|
1104
|
+
if (diagnosticMessage) {
|
|
1105
|
+
return { content: diagnosticMessage, isError: true };
|
|
1106
|
+
}
|
|
653
1107
|
const msg = err instanceof Error ? err.message : String(err);
|
|
654
1108
|
log.error({ err }, "Screenshot failed");
|
|
655
1109
|
return { content: `Error: Screenshot failed: ${msg}`, isError: true };
|
|
@@ -658,13 +1112,118 @@ export async function executeBrowserScreenshot(
|
|
|
658
1112
|
}
|
|
659
1113
|
}
|
|
660
1114
|
|
|
1115
|
+
// ── browser_attach ───────────────────────────────────────────────────
|
|
1116
|
+
|
|
1117
|
+
export async function executeBrowserAttach(
|
|
1118
|
+
_input: Record<string, unknown>,
|
|
1119
|
+
context: ToolContext,
|
|
1120
|
+
): Promise<ToolExecutionResult> {
|
|
1121
|
+
const acquired = acquireCdpClientWithMode(_input, context);
|
|
1122
|
+
if (acquired.errorResult) return acquired.errorResult;
|
|
1123
|
+
const cdp = acquired.cdp;
|
|
1124
|
+
try {
|
|
1125
|
+
if (cdp.kind === "extension") {
|
|
1126
|
+
// Extension path: explicitly attach the debugger via a synthetic
|
|
1127
|
+
// Vellum.attach command so the debugging session is established
|
|
1128
|
+
// before any navigation or interaction.
|
|
1129
|
+
const result = await cdp.send<{ attached?: boolean; target?: unknown }>(
|
|
1130
|
+
"Vellum.attach",
|
|
1131
|
+
{},
|
|
1132
|
+
context.signal,
|
|
1133
|
+
);
|
|
1134
|
+
log.debug(
|
|
1135
|
+
{ conversationId: context.conversationId, result },
|
|
1136
|
+
"Browser debugger attached (extension)",
|
|
1137
|
+
);
|
|
1138
|
+
return {
|
|
1139
|
+
content: "Browser debugger attached.",
|
|
1140
|
+
isError: false,
|
|
1141
|
+
};
|
|
1142
|
+
}
|
|
1143
|
+
|
|
1144
|
+
// Non-extension backends (local / cdp-inspect): explicit attach is
|
|
1145
|
+
// not required — the backend manages its own connection lifecycle.
|
|
1146
|
+
// Return a deterministic no-op success.
|
|
1147
|
+
return {
|
|
1148
|
+
content:
|
|
1149
|
+
"Browser session ready. (Explicit attach is not required on this backend.)",
|
|
1150
|
+
isError: false,
|
|
1151
|
+
};
|
|
1152
|
+
} catch (err) {
|
|
1153
|
+
const diagnosticMessage = formatCdpSendDiagnostics(
|
|
1154
|
+
err,
|
|
1155
|
+
acquired.browserMode,
|
|
1156
|
+
);
|
|
1157
|
+
if (diagnosticMessage) {
|
|
1158
|
+
return { content: diagnosticMessage, isError: true };
|
|
1159
|
+
}
|
|
1160
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1161
|
+
log.error({ err }, "Attach failed");
|
|
1162
|
+
return { content: `Error: Attach failed: ${msg}`, isError: true };
|
|
1163
|
+
} finally {
|
|
1164
|
+
cdp.dispose();
|
|
1165
|
+
}
|
|
1166
|
+
}
|
|
1167
|
+
|
|
1168
|
+
// ── browser_detach ──────────────────────────────────────────────────
|
|
1169
|
+
|
|
1170
|
+
export async function executeBrowserDetach(
|
|
1171
|
+
_input: Record<string, unknown>,
|
|
1172
|
+
context: ToolContext,
|
|
1173
|
+
): Promise<ToolExecutionResult> {
|
|
1174
|
+
const acquired = acquireCdpClientWithMode(_input, context);
|
|
1175
|
+
if (acquired.errorResult) return acquired.errorResult;
|
|
1176
|
+
const cdp = acquired.cdp;
|
|
1177
|
+
try {
|
|
1178
|
+
if (cdp.kind === "extension") {
|
|
1179
|
+
// Extension path: explicitly detach the debugger via a synthetic
|
|
1180
|
+
// Vellum.detach command so the Chrome debugging banner clears.
|
|
1181
|
+
const result = await cdp.send<{ detached?: boolean; target?: unknown }>(
|
|
1182
|
+
"Vellum.detach",
|
|
1183
|
+
{},
|
|
1184
|
+
context.signal,
|
|
1185
|
+
);
|
|
1186
|
+
log.debug(
|
|
1187
|
+
{ conversationId: context.conversationId, result },
|
|
1188
|
+
"Browser debugger detached (extension)",
|
|
1189
|
+
);
|
|
1190
|
+
}
|
|
1191
|
+
|
|
1192
|
+
return {
|
|
1193
|
+
content: "Browser debugger detached and snapshot state cleared.",
|
|
1194
|
+
isError: false,
|
|
1195
|
+
};
|
|
1196
|
+
} catch (err) {
|
|
1197
|
+
const diagnosticMessage = formatCdpSendDiagnostics(
|
|
1198
|
+
err,
|
|
1199
|
+
acquired.browserMode,
|
|
1200
|
+
);
|
|
1201
|
+
if (diagnosticMessage) {
|
|
1202
|
+
return { content: diagnosticMessage, isError: true };
|
|
1203
|
+
}
|
|
1204
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1205
|
+
log.error({ err }, "Detach failed");
|
|
1206
|
+
return { content: `Error: Detach failed: ${msg}`, isError: true };
|
|
1207
|
+
} finally {
|
|
1208
|
+
// Always reset conversation-scoped browser state, even if the
|
|
1209
|
+
// Vellum.detach round-trip failed (target gone, transport dropped).
|
|
1210
|
+
// browser_detach is the user's recovery path — leaving a stale
|
|
1211
|
+
// sticky backend or snapshot map behind would defeat its purpose.
|
|
1212
|
+
browserManager.clearSnapshotBackendNodeMap(context.conversationId);
|
|
1213
|
+
browserManager.clearPreferredBackendKind(context.conversationId);
|
|
1214
|
+
cdp.dispose();
|
|
1215
|
+
}
|
|
1216
|
+
}
|
|
1217
|
+
|
|
661
1218
|
// ── browser_close ────────────────────────────────────────────────────
|
|
662
1219
|
|
|
663
1220
|
export async function executeBrowserClose(
|
|
664
1221
|
input: Record<string, unknown>,
|
|
665
1222
|
context: ToolContext,
|
|
666
1223
|
): Promise<ToolExecutionResult> {
|
|
667
|
-
const
|
|
1224
|
+
const acquired = acquireCdpClientWithMode(input, context);
|
|
1225
|
+
if (acquired.errorResult) return acquired.errorResult;
|
|
1226
|
+
const cdp = acquired.cdp;
|
|
668
1227
|
try {
|
|
669
1228
|
if (cdp.kind === "local") {
|
|
670
1229
|
// Local/sacrificial-profile path: tear down the Playwright page,
|
|
@@ -690,15 +1249,29 @@ export async function executeBrowserClose(
|
|
|
690
1249
|
}
|
|
691
1250
|
|
|
692
1251
|
// Extension path: the user owns their Chrome tab — we must not
|
|
693
|
-
// close it.
|
|
694
|
-
//
|
|
1252
|
+
// close it. Detach the debugger (so the Chrome debugging banner
|
|
1253
|
+
// clears promptly) and drop the cached snapshot state so stale
|
|
1254
|
+
// eids from prior snapshots cannot be resolved by later tool calls.
|
|
1255
|
+
try {
|
|
1256
|
+
await cdp.send("Vellum.detach", {}, context.signal);
|
|
1257
|
+
} catch {
|
|
1258
|
+
// Tolerate detach failures (already detached, tab closed, etc.)
|
|
1259
|
+
}
|
|
695
1260
|
browserManager.clearSnapshotBackendNodeMap(context.conversationId);
|
|
1261
|
+
browserManager.clearPreferredBackendKind(context.conversationId);
|
|
696
1262
|
return {
|
|
697
1263
|
content:
|
|
698
1264
|
"Browser session cleared. (Your Chrome tab was not closed — close it yourself if desired.)",
|
|
699
1265
|
isError: false,
|
|
700
1266
|
};
|
|
701
1267
|
} catch (err) {
|
|
1268
|
+
const diagnosticMessage = formatCdpSendDiagnostics(
|
|
1269
|
+
err,
|
|
1270
|
+
acquired.browserMode,
|
|
1271
|
+
);
|
|
1272
|
+
if (diagnosticMessage) {
|
|
1273
|
+
return { content: diagnosticMessage, isError: true };
|
|
1274
|
+
}
|
|
702
1275
|
const msg = err instanceof Error ? err.message : String(err);
|
|
703
1276
|
log.error({ err }, "Close failed");
|
|
704
1277
|
return { content: `Error: Close failed: ${msg}`, isError: true };
|
|
@@ -716,7 +1289,9 @@ export async function executeBrowserClick(
|
|
|
716
1289
|
const { resolved, error } = resolveElement(context.conversationId, input);
|
|
717
1290
|
if (error) return { content: error, isError: true };
|
|
718
1291
|
|
|
719
|
-
const
|
|
1292
|
+
const acquired = acquireCdpClientWithMode(input, context);
|
|
1293
|
+
if (acquired.errorResult) return acquired.errorResult;
|
|
1294
|
+
const cdp = acquired.cdp;
|
|
720
1295
|
try {
|
|
721
1296
|
let backendNodeId: number;
|
|
722
1297
|
if (resolved!.kind === "backend") {
|
|
@@ -744,6 +1319,13 @@ export async function executeBrowserClick(
|
|
|
744
1319
|
: resolved!.selector;
|
|
745
1320
|
return { content: `Clicked element: ${desc}`, isError: false };
|
|
746
1321
|
} catch (err) {
|
|
1322
|
+
const diagnosticMessage = formatCdpSendDiagnostics(
|
|
1323
|
+
err,
|
|
1324
|
+
acquired.browserMode,
|
|
1325
|
+
);
|
|
1326
|
+
if (diagnosticMessage) {
|
|
1327
|
+
return { content: diagnosticMessage, isError: true };
|
|
1328
|
+
}
|
|
747
1329
|
const msg = err instanceof Error ? err.message : String(err);
|
|
748
1330
|
log.error({ err }, "Click failed");
|
|
749
1331
|
return { content: `Error: Click failed: ${msg}`, isError: true };
|
|
@@ -828,7 +1410,9 @@ export async function executeBrowserType(
|
|
|
828
1410
|
? `element_id "${resolved!.eid}"`
|
|
829
1411
|
: resolved!.selector;
|
|
830
1412
|
|
|
831
|
-
const
|
|
1413
|
+
const acquired = acquireCdpClientWithMode(input, context);
|
|
1414
|
+
if (acquired.errorResult) return acquired.errorResult;
|
|
1415
|
+
const cdp = acquired.cdp;
|
|
832
1416
|
try {
|
|
833
1417
|
let backendNodeId: number;
|
|
834
1418
|
if (resolved!.kind === "backend") {
|
|
@@ -857,6 +1441,13 @@ export async function executeBrowserType(
|
|
|
857
1441
|
if (pressEnter) lines.push("(pressed Enter after typing)");
|
|
858
1442
|
return { content: lines.join("\n"), isError: false };
|
|
859
1443
|
} catch (err) {
|
|
1444
|
+
const diagnosticMessage = formatCdpSendDiagnostics(
|
|
1445
|
+
err,
|
|
1446
|
+
acquired.browserMode,
|
|
1447
|
+
);
|
|
1448
|
+
if (diagnosticMessage) {
|
|
1449
|
+
return { content: diagnosticMessage, isError: true };
|
|
1450
|
+
}
|
|
860
1451
|
const msg = err instanceof Error ? err.message : String(err);
|
|
861
1452
|
log.error({ err, target: targetDescription }, "Type failed");
|
|
862
1453
|
return { content: `Error: Type failed: ${msg}`, isError: true };
|
|
@@ -896,7 +1487,9 @@ export async function executeBrowserPressKey(
|
|
|
896
1487
|
: resolved!.selector;
|
|
897
1488
|
}
|
|
898
1489
|
|
|
899
|
-
const
|
|
1490
|
+
const acquired = acquireCdpClientWithMode(input, context);
|
|
1491
|
+
if (acquired.errorResult) return acquired.errorResult;
|
|
1492
|
+
const cdp = acquired.cdp;
|
|
900
1493
|
try {
|
|
901
1494
|
if (resolved) {
|
|
902
1495
|
let backendNodeId: number;
|
|
@@ -921,6 +1514,13 @@ export async function executeBrowserPressKey(
|
|
|
921
1514
|
await dispatchKeyPress(cdp, key, context.signal);
|
|
922
1515
|
return { content: `Pressed "${key}"`, isError: false };
|
|
923
1516
|
} catch (err) {
|
|
1517
|
+
const diagnosticMessage = formatCdpSendDiagnostics(
|
|
1518
|
+
err,
|
|
1519
|
+
acquired.browserMode,
|
|
1520
|
+
);
|
|
1521
|
+
if (diagnosticMessage) {
|
|
1522
|
+
return { content: diagnosticMessage, isError: true };
|
|
1523
|
+
}
|
|
924
1524
|
const msg = err instanceof Error ? err.message : String(err);
|
|
925
1525
|
log.error({ err, key }, "Press key failed");
|
|
926
1526
|
return { content: `Error: Press key failed: ${msg}`, isError: true };
|
|
@@ -964,7 +1564,9 @@ export async function executeBrowserScroll(
|
|
|
964
1564
|
break;
|
|
965
1565
|
}
|
|
966
1566
|
|
|
967
|
-
const
|
|
1567
|
+
const acquired = acquireCdpClientWithMode(input, context);
|
|
1568
|
+
if (acquired.errorResult) return acquired.errorResult;
|
|
1569
|
+
const cdp = acquired.cdp;
|
|
968
1570
|
try {
|
|
969
1571
|
// Fetch viewport dimensions so we can dispatch the wheel event at
|
|
970
1572
|
// the viewport center — scrolling from (0, 0) misses sticky
|
|
@@ -985,6 +1587,13 @@ export async function executeBrowserScroll(
|
|
|
985
1587
|
|
|
986
1588
|
return { content: `Scrolled ${direction} by ${amount}px`, isError: false };
|
|
987
1589
|
} catch (err) {
|
|
1590
|
+
const diagnosticMessage = formatCdpSendDiagnostics(
|
|
1591
|
+
err,
|
|
1592
|
+
acquired.browserMode,
|
|
1593
|
+
);
|
|
1594
|
+
if (diagnosticMessage) {
|
|
1595
|
+
return { content: diagnosticMessage, isError: true };
|
|
1596
|
+
}
|
|
988
1597
|
const msg = err instanceof Error ? err.message : String(err);
|
|
989
1598
|
log.error({ err, direction }, "Scroll failed");
|
|
990
1599
|
return { content: `Error: Scroll failed: ${msg}`, isError: true };
|
|
@@ -1018,7 +1627,9 @@ export async function executeBrowserSelectOption(
|
|
|
1018
1627
|
? `element_id "${resolved!.eid}"`
|
|
1019
1628
|
: resolved!.selector;
|
|
1020
1629
|
|
|
1021
|
-
const
|
|
1630
|
+
const acquired = acquireCdpClientWithMode(input, context);
|
|
1631
|
+
if (acquired.errorResult) return acquired.errorResult;
|
|
1632
|
+
const cdp = acquired.cdp;
|
|
1022
1633
|
try {
|
|
1023
1634
|
let backendNodeId: number;
|
|
1024
1635
|
if (resolved!.kind === "backend") {
|
|
@@ -1108,6 +1719,13 @@ export async function executeBrowserSelectOption(
|
|
|
1108
1719
|
isError: false,
|
|
1109
1720
|
};
|
|
1110
1721
|
} catch (err) {
|
|
1722
|
+
const diagnosticMessage = formatCdpSendDiagnostics(
|
|
1723
|
+
err,
|
|
1724
|
+
acquired.browserMode,
|
|
1725
|
+
);
|
|
1726
|
+
if (diagnosticMessage) {
|
|
1727
|
+
return { content: diagnosticMessage, isError: true };
|
|
1728
|
+
}
|
|
1111
1729
|
const msg = err instanceof Error ? err.message : String(err);
|
|
1112
1730
|
log.error({ err, target: targetDescription }, "Select option failed");
|
|
1113
1731
|
return { content: `Error: Select option failed: ${msg}`, isError: true };
|
|
@@ -1125,7 +1743,9 @@ export async function executeBrowserHover(
|
|
|
1125
1743
|
const { resolved, error } = resolveElement(context.conversationId, input);
|
|
1126
1744
|
if (error) return { content: error, isError: true };
|
|
1127
1745
|
|
|
1128
|
-
const
|
|
1746
|
+
const acquired = acquireCdpClientWithMode(input, context);
|
|
1747
|
+
if (acquired.errorResult) return acquired.errorResult;
|
|
1748
|
+
const cdp = acquired.cdp;
|
|
1129
1749
|
try {
|
|
1130
1750
|
let backendNodeId: number;
|
|
1131
1751
|
if (resolved!.kind === "backend") {
|
|
@@ -1150,6 +1770,13 @@ export async function executeBrowserHover(
|
|
|
1150
1770
|
: resolved!.selector;
|
|
1151
1771
|
return { content: `Hovered element: ${desc}`, isError: false };
|
|
1152
1772
|
} catch (err) {
|
|
1773
|
+
const diagnosticMessage = formatCdpSendDiagnostics(
|
|
1774
|
+
err,
|
|
1775
|
+
acquired.browserMode,
|
|
1776
|
+
);
|
|
1777
|
+
if (diagnosticMessage) {
|
|
1778
|
+
return { content: diagnosticMessage, isError: true };
|
|
1779
|
+
}
|
|
1153
1780
|
const msg = err instanceof Error ? err.message : String(err);
|
|
1154
1781
|
log.error({ err }, "Hover failed");
|
|
1155
1782
|
return { content: `Error: Hover failed: ${msg}`, isError: true };
|
|
@@ -1195,6 +1822,13 @@ export async function executeBrowserWaitFor(
|
|
|
1195
1822
|
? Math.min(input.timeout, MAX_WAIT_MS)
|
|
1196
1823
|
: MAX_WAIT_MS;
|
|
1197
1824
|
|
|
1825
|
+
// Validate browser_mode even on the duration path so invalid values
|
|
1826
|
+
// are rejected consistently regardless of which wait mode is used.
|
|
1827
|
+
const modeResult = parseBrowserMode(input);
|
|
1828
|
+
if (!modeResult.ok) {
|
|
1829
|
+
return { content: modeResult.error, isError: true };
|
|
1830
|
+
}
|
|
1831
|
+
|
|
1198
1832
|
// Duration mode has no CDP interaction — handle without acquiring
|
|
1199
1833
|
// a CdpClient so the common "sleep" path stays transport-agnostic.
|
|
1200
1834
|
if (duration != null) {
|
|
@@ -1203,7 +1837,9 @@ export async function executeBrowserWaitFor(
|
|
|
1203
1837
|
return { content: `Waited ${waitMs}ms.`, isError: false };
|
|
1204
1838
|
}
|
|
1205
1839
|
|
|
1206
|
-
const
|
|
1840
|
+
const acquired = acquireCdpClientWithMode(input, context);
|
|
1841
|
+
if (acquired.errorResult) return acquired.errorResult;
|
|
1842
|
+
const cdp = acquired.cdp;
|
|
1207
1843
|
try {
|
|
1208
1844
|
if (selector) {
|
|
1209
1845
|
// browser_wait_for selector mode is "did this node appear at
|
|
@@ -1227,6 +1863,13 @@ export async function executeBrowserWaitFor(
|
|
|
1227
1863
|
isError: false,
|
|
1228
1864
|
};
|
|
1229
1865
|
} catch (err) {
|
|
1866
|
+
const diagnosticMessage = formatCdpSendDiagnostics(
|
|
1867
|
+
err,
|
|
1868
|
+
acquired.browserMode,
|
|
1869
|
+
);
|
|
1870
|
+
if (diagnosticMessage) {
|
|
1871
|
+
return { content: diagnosticMessage, isError: true };
|
|
1872
|
+
}
|
|
1230
1873
|
const msg = err instanceof Error ? err.message : String(err);
|
|
1231
1874
|
log.error({ err }, "Wait failed");
|
|
1232
1875
|
return { content: `Error: Wait failed: ${msg}`, isError: true };
|
|
@@ -1243,7 +1886,9 @@ export async function executeBrowserExtract(
|
|
|
1243
1886
|
): Promise<ToolExecutionResult> {
|
|
1244
1887
|
const includeLinks = input.include_links === true;
|
|
1245
1888
|
|
|
1246
|
-
const
|
|
1889
|
+
const acquired = acquireCdpClientWithMode(input, context);
|
|
1890
|
+
if (acquired.errorResult) return acquired.errorResult;
|
|
1891
|
+
const cdp = acquired.cdp;
|
|
1247
1892
|
try {
|
|
1248
1893
|
const currentUrl = await getCurrentUrl(cdp, context.signal);
|
|
1249
1894
|
const title = await getPageTitle(cdp, context.signal);
|
|
@@ -1257,7 +1902,8 @@ export async function executeBrowserExtract(
|
|
|
1257
1902
|
|
|
1258
1903
|
if (textContent.length > MAX_EXTRACT_LENGTH) {
|
|
1259
1904
|
textContent =
|
|
1260
|
-
textContent
|
|
1905
|
+
safeStringSlice(textContent, 0, MAX_EXTRACT_LENGTH) +
|
|
1906
|
+
"\n... (truncated)";
|
|
1261
1907
|
}
|
|
1262
1908
|
|
|
1263
1909
|
const lines: string[] = [
|
|
@@ -1283,6 +1929,13 @@ export async function executeBrowserExtract(
|
|
|
1283
1929
|
|
|
1284
1930
|
return { content: lines.join("\n"), isError: false };
|
|
1285
1931
|
} catch (err) {
|
|
1932
|
+
const diagnosticMessage = formatCdpSendDiagnostics(
|
|
1933
|
+
err,
|
|
1934
|
+
acquired.browserMode,
|
|
1935
|
+
);
|
|
1936
|
+
if (diagnosticMessage) {
|
|
1937
|
+
return { content: diagnosticMessage, isError: true };
|
|
1938
|
+
}
|
|
1286
1939
|
const msg = err instanceof Error ? err.message : String(err);
|
|
1287
1940
|
log.error({ err }, "Extract failed");
|
|
1288
1941
|
return { content: `Error: Extract failed: ${msg}`, isError: true };
|
|
@@ -1291,7 +1944,7 @@ export async function executeBrowserExtract(
|
|
|
1291
1944
|
}
|
|
1292
1945
|
}
|
|
1293
1946
|
|
|
1294
|
-
// ──
|
|
1947
|
+
// ── browser credential fill ──────────────────────────────────────────
|
|
1295
1948
|
|
|
1296
1949
|
export async function executeBrowserFillCredential(
|
|
1297
1950
|
input: Record<string, unknown>,
|
|
@@ -1316,7 +1969,9 @@ export async function executeBrowserFillCredential(
|
|
|
1316
1969
|
? `element_id "${resolved!.eid}"`
|
|
1317
1970
|
: resolved!.selector;
|
|
1318
1971
|
|
|
1319
|
-
const
|
|
1972
|
+
const acquired = acquireCdpClientWithMode(input, context);
|
|
1973
|
+
if (acquired.errorResult) return acquired.errorResult;
|
|
1974
|
+
const cdp = acquired.cdp;
|
|
1320
1975
|
try {
|
|
1321
1976
|
let backendNodeId: number;
|
|
1322
1977
|
if (resolved!.kind === "backend") {
|
|
@@ -1347,7 +2002,7 @@ export async function executeBrowserFillCredential(
|
|
|
1347
2002
|
const result = await credentialBroker.browserFill({
|
|
1348
2003
|
service,
|
|
1349
2004
|
field,
|
|
1350
|
-
toolName:
|
|
2005
|
+
toolName: BROWSER_FILL_CAPABILITY,
|
|
1351
2006
|
domain: pageDomain,
|
|
1352
2007
|
fill: async (value) => {
|
|
1353
2008
|
// Clear-then-focus-then-insert via the shared helper. We
|
|
@@ -1405,6 +2060,13 @@ export async function executeBrowserFillCredential(
|
|
|
1405
2060
|
isError: false,
|
|
1406
2061
|
};
|
|
1407
2062
|
} catch (err) {
|
|
2063
|
+
const diagnosticMessage = formatCdpSendDiagnostics(
|
|
2064
|
+
err,
|
|
2065
|
+
acquired.browserMode,
|
|
2066
|
+
);
|
|
2067
|
+
if (diagnosticMessage) {
|
|
2068
|
+
return { content: diagnosticMessage, isError: true };
|
|
2069
|
+
}
|
|
1408
2070
|
const msg = err instanceof Error ? err.message : String(err);
|
|
1409
2071
|
log.error({ err }, "Fill credential failed");
|
|
1410
2072
|
return { content: `Error: Fill credential failed: ${msg}`, isError: true };
|
|
@@ -1412,3 +2074,508 @@ export async function executeBrowserFillCredential(
|
|
|
1412
2074
|
cdp.dispose();
|
|
1413
2075
|
}
|
|
1414
2076
|
}
|
|
2077
|
+
|
|
2078
|
+
function dedupeStrings(values: string[]): string[] {
|
|
2079
|
+
const seen = new Set<string>();
|
|
2080
|
+
const out: string[] = [];
|
|
2081
|
+
for (const value of values) {
|
|
2082
|
+
if (!value) continue;
|
|
2083
|
+
if (seen.has(value)) continue;
|
|
2084
|
+
seen.add(value);
|
|
2085
|
+
out.push(value);
|
|
2086
|
+
}
|
|
2087
|
+
return out;
|
|
2088
|
+
}
|
|
2089
|
+
|
|
2090
|
+
function modeTradeoffs(mode: StatusCheckMode): string[] {
|
|
2091
|
+
return MODE_TRADEOFFS[mode];
|
|
2092
|
+
}
|
|
2093
|
+
|
|
2094
|
+
function extensionSetupActions(): string[] {
|
|
2095
|
+
return [
|
|
2096
|
+
"Install the Vellum Assistant Chrome extension from the Chrome Web Store: https://chromewebstore.google.com/detail/vellum-assistant-browser/hphbdmpffeigpcdjkckleobjmhhokpne",
|
|
2097
|
+
"Open the extension and pair with your assistant.",
|
|
2098
|
+
];
|
|
2099
|
+
}
|
|
2100
|
+
|
|
2101
|
+
function cdpInspectSetupActions(): string[] {
|
|
2102
|
+
return [
|
|
2103
|
+
"Update Chrome to the latest version by going to chrome://settings/help",
|
|
2104
|
+
"Navigate directly to chrome://inspect/#remote-debugging",
|
|
2105
|
+
'Check the box next to "Allow remote debugging for this browser instance"',
|
|
2106
|
+
];
|
|
2107
|
+
}
|
|
2108
|
+
|
|
2109
|
+
function localSetupActions(): string[] {
|
|
2110
|
+
return [
|
|
2111
|
+
"Ask your assistant to install playwright and chromium on your host machine.",
|
|
2112
|
+
];
|
|
2113
|
+
}
|
|
2114
|
+
|
|
2115
|
+
function extractDiscoveryCodes(error: CdpError): string[] {
|
|
2116
|
+
const diagnostics = error.attemptDiagnostics ?? [];
|
|
2117
|
+
const codes: string[] = [];
|
|
2118
|
+
for (const diag of diagnostics) {
|
|
2119
|
+
if (diag.discoveryCode) codes.push(diag.discoveryCode);
|
|
2120
|
+
}
|
|
2121
|
+
return dedupeStrings(codes);
|
|
2122
|
+
}
|
|
2123
|
+
|
|
2124
|
+
function containsTokenCaseInsensitive(text: string, token: string): boolean {
|
|
2125
|
+
return text.toLowerCase().includes(token.toLowerCase());
|
|
2126
|
+
}
|
|
2127
|
+
|
|
2128
|
+
function probeFailureActions(mode: StatusCheckMode, error: CdpError): string[] {
|
|
2129
|
+
const actions: string[] = [];
|
|
2130
|
+
const message = error.message.toLowerCase();
|
|
2131
|
+
const discoveryCodes = extractDiscoveryCodes(error).map((c) =>
|
|
2132
|
+
c.toLowerCase(),
|
|
2133
|
+
);
|
|
2134
|
+
|
|
2135
|
+
if (mode === BROWSER_STATUS_MODE.EXTENSION) {
|
|
2136
|
+
actions.push(...extensionSetupActions());
|
|
2137
|
+
if (
|
|
2138
|
+
containsTokenCaseInsensitive(
|
|
2139
|
+
message,
|
|
2140
|
+
EXTENSION_STATUS_ERROR_MARKER.UNAUTHORIZED_ORIGIN,
|
|
2141
|
+
)
|
|
2142
|
+
) {
|
|
2143
|
+
actions.push(
|
|
2144
|
+
"Ensure this extension ID is present in meta/browser-extension/chrome-extension-allowlist.json and restart the assistant.",
|
|
2145
|
+
);
|
|
2146
|
+
}
|
|
2147
|
+
if (
|
|
2148
|
+
containsTokenCaseInsensitive(
|
|
2149
|
+
message,
|
|
2150
|
+
EXTENSION_STATUS_ERROR_MARKER.NATIVE_MESSAGING_HOST,
|
|
2151
|
+
)
|
|
2152
|
+
) {
|
|
2153
|
+
actions.push(
|
|
2154
|
+
"Reinstall the native messaging host manifest and confirm it allows this extension ID.",
|
|
2155
|
+
);
|
|
2156
|
+
}
|
|
2157
|
+
if (
|
|
2158
|
+
containsTokenCaseInsensitive(
|
|
2159
|
+
message,
|
|
2160
|
+
EXTENSION_STATUS_ERROR_MARKER.HTTP_401,
|
|
2161
|
+
)
|
|
2162
|
+
) {
|
|
2163
|
+
actions.push(
|
|
2164
|
+
"Re-pair the extension so it refreshes its local relay credential.",
|
|
2165
|
+
);
|
|
2166
|
+
}
|
|
2167
|
+
}
|
|
2168
|
+
|
|
2169
|
+
if (mode === BROWSER_STATUS_MODE.CDP_INSPECT) {
|
|
2170
|
+
actions.push(...cdpInspectSetupActions());
|
|
2171
|
+
if (discoveryCodes.includes(CDP_INSPECT_STATUS_DISCOVERY_CODE.NO_TARGETS)) {
|
|
2172
|
+
actions.push("Open at least one normal web page tab and retry.");
|
|
2173
|
+
}
|
|
2174
|
+
if (
|
|
2175
|
+
discoveryCodes.includes(
|
|
2176
|
+
CDP_INSPECT_STATUS_DISCOVERY_CODE.INVALID_RESPONSE,
|
|
2177
|
+
) ||
|
|
2178
|
+
discoveryCodes.includes(
|
|
2179
|
+
CDP_INSPECT_STATUS_DISCOVERY_CODE.WS_FALLBACK_FAILED,
|
|
2180
|
+
)
|
|
2181
|
+
) {
|
|
2182
|
+
actions.push(
|
|
2183
|
+
"Verify nothing else is bound to the configured CDP port and exposing non-DevTools responses.",
|
|
2184
|
+
);
|
|
2185
|
+
}
|
|
2186
|
+
}
|
|
2187
|
+
|
|
2188
|
+
if (mode === BROWSER_STATUS_MODE.LOCAL) {
|
|
2189
|
+
actions.push(...localSetupActions());
|
|
2190
|
+
}
|
|
2191
|
+
|
|
2192
|
+
return dedupeStrings(actions);
|
|
2193
|
+
}
|
|
2194
|
+
|
|
2195
|
+
async function probePinnedBrowserMode(
|
|
2196
|
+
mode: StatusCheckMode,
|
|
2197
|
+
context: ToolContext,
|
|
2198
|
+
): Promise<
|
|
2199
|
+
| {
|
|
2200
|
+
ok: true;
|
|
2201
|
+
backendKind: CdpClientKind;
|
|
2202
|
+
}
|
|
2203
|
+
| {
|
|
2204
|
+
ok: false;
|
|
2205
|
+
error: CdpError;
|
|
2206
|
+
diagnostic: string;
|
|
2207
|
+
}
|
|
2208
|
+
> {
|
|
2209
|
+
let cdp: ReturnType<typeof getCdpClient> | null = null;
|
|
2210
|
+
try {
|
|
2211
|
+
cdp = getCdpClient(context, { mode });
|
|
2212
|
+
await cdp.send(
|
|
2213
|
+
"Runtime.evaluate",
|
|
2214
|
+
{
|
|
2215
|
+
expression: "document.readyState",
|
|
2216
|
+
returnByValue: true,
|
|
2217
|
+
},
|
|
2218
|
+
context.signal,
|
|
2219
|
+
);
|
|
2220
|
+
return { ok: true, backendKind: cdp.kind };
|
|
2221
|
+
} catch (err) {
|
|
2222
|
+
if (err instanceof CdpError) {
|
|
2223
|
+
return {
|
|
2224
|
+
ok: false,
|
|
2225
|
+
error: err,
|
|
2226
|
+
diagnostic: formatModeSelectionFailure(mode, err),
|
|
2227
|
+
};
|
|
2228
|
+
}
|
|
2229
|
+
const wrapped = new CdpError(
|
|
2230
|
+
"transport_error",
|
|
2231
|
+
err instanceof Error ? err.message : String(err),
|
|
2232
|
+
{ underlying: err },
|
|
2233
|
+
);
|
|
2234
|
+
return {
|
|
2235
|
+
ok: false,
|
|
2236
|
+
error: wrapped,
|
|
2237
|
+
diagnostic: formatModeSelectionFailure(mode, wrapped),
|
|
2238
|
+
};
|
|
2239
|
+
} finally {
|
|
2240
|
+
cdp?.dispose();
|
|
2241
|
+
}
|
|
2242
|
+
}
|
|
2243
|
+
|
|
2244
|
+
async function checkExtensionModeStatus(
|
|
2245
|
+
context: ToolContext,
|
|
2246
|
+
autoCandidate: boolean,
|
|
2247
|
+
): Promise<BrowserStatusModeResult> {
|
|
2248
|
+
const proxyBound = Boolean(context.hostBrowserProxy);
|
|
2249
|
+
const proxyConnected = context.hostBrowserProxy?.isAvailable() ?? false;
|
|
2250
|
+
|
|
2251
|
+
if (!proxyBound) {
|
|
2252
|
+
return {
|
|
2253
|
+
mode: BROWSER_STATUS_MODE.EXTENSION,
|
|
2254
|
+
available: false,
|
|
2255
|
+
verified: "preflight",
|
|
2256
|
+
autoCandidate,
|
|
2257
|
+
summary:
|
|
2258
|
+
"Extension mode is unavailable: no host browser proxy is bound to this conversation.",
|
|
2259
|
+
userActions: extensionSetupActions(),
|
|
2260
|
+
tradeoffs: modeTradeoffs(BROWSER_STATUS_MODE.EXTENSION),
|
|
2261
|
+
details: {
|
|
2262
|
+
proxyBound,
|
|
2263
|
+
proxyConnected,
|
|
2264
|
+
},
|
|
2265
|
+
};
|
|
2266
|
+
}
|
|
2267
|
+
|
|
2268
|
+
if (!proxyConnected) {
|
|
2269
|
+
return {
|
|
2270
|
+
mode: BROWSER_STATUS_MODE.EXTENSION,
|
|
2271
|
+
available: false,
|
|
2272
|
+
verified: "preflight",
|
|
2273
|
+
autoCandidate,
|
|
2274
|
+
summary:
|
|
2275
|
+
"Extension mode is unavailable: the extension transport is currently disconnected.",
|
|
2276
|
+
userActions: extensionSetupActions(),
|
|
2277
|
+
tradeoffs: modeTradeoffs(BROWSER_STATUS_MODE.EXTENSION),
|
|
2278
|
+
details: {
|
|
2279
|
+
proxyBound,
|
|
2280
|
+
proxyConnected,
|
|
2281
|
+
},
|
|
2282
|
+
};
|
|
2283
|
+
}
|
|
2284
|
+
|
|
2285
|
+
const probe = await probePinnedBrowserMode(
|
|
2286
|
+
BROWSER_STATUS_MODE.EXTENSION,
|
|
2287
|
+
context,
|
|
2288
|
+
);
|
|
2289
|
+
if (probe.ok) {
|
|
2290
|
+
return {
|
|
2291
|
+
mode: BROWSER_STATUS_MODE.EXTENSION,
|
|
2292
|
+
available: true,
|
|
2293
|
+
verified: "active_probe",
|
|
2294
|
+
autoCandidate,
|
|
2295
|
+
summary: "Extension mode is ready and responded to an active CDP probe.",
|
|
2296
|
+
userActions: [],
|
|
2297
|
+
tradeoffs: modeTradeoffs(BROWSER_STATUS_MODE.EXTENSION),
|
|
2298
|
+
details: {
|
|
2299
|
+
proxyBound,
|
|
2300
|
+
proxyConnected,
|
|
2301
|
+
backendKind: probe.backendKind,
|
|
2302
|
+
},
|
|
2303
|
+
};
|
|
2304
|
+
}
|
|
2305
|
+
|
|
2306
|
+
if (isRestrictedChromePageProbeError(probe.error)) {
|
|
2307
|
+
return {
|
|
2308
|
+
mode: BROWSER_STATUS_MODE.EXTENSION,
|
|
2309
|
+
available: true,
|
|
2310
|
+
verified: "active_probe",
|
|
2311
|
+
autoCandidate,
|
|
2312
|
+
summary:
|
|
2313
|
+
"Extension mode transport is connected, but the active Chrome tab is a restricted chrome:// page. Switch to a regular website tab if browser actions fail.",
|
|
2314
|
+
userActions: [
|
|
2315
|
+
"Switch Chrome to a regular http(s) tab (not chrome://...) and retry.",
|
|
2316
|
+
],
|
|
2317
|
+
tradeoffs: modeTradeoffs(BROWSER_STATUS_MODE.EXTENSION),
|
|
2318
|
+
details: {
|
|
2319
|
+
proxyBound,
|
|
2320
|
+
proxyConnected,
|
|
2321
|
+
restrictedActiveTab: true,
|
|
2322
|
+
errorCode: probe.error.code,
|
|
2323
|
+
diagnostic: probe.diagnostic,
|
|
2324
|
+
attemptDiagnostics: probe.error.attemptDiagnostics ?? [],
|
|
2325
|
+
},
|
|
2326
|
+
};
|
|
2327
|
+
}
|
|
2328
|
+
|
|
2329
|
+
return {
|
|
2330
|
+
mode: BROWSER_STATUS_MODE.EXTENSION,
|
|
2331
|
+
available: false,
|
|
2332
|
+
verified: "active_probe",
|
|
2333
|
+
autoCandidate,
|
|
2334
|
+
summary: `Extension mode probe failed: ${probe.error.message}`,
|
|
2335
|
+
userActions: probeFailureActions(
|
|
2336
|
+
BROWSER_STATUS_MODE.EXTENSION,
|
|
2337
|
+
probe.error,
|
|
2338
|
+
),
|
|
2339
|
+
tradeoffs: modeTradeoffs(BROWSER_STATUS_MODE.EXTENSION),
|
|
2340
|
+
details: {
|
|
2341
|
+
proxyBound,
|
|
2342
|
+
proxyConnected,
|
|
2343
|
+
errorCode: probe.error.code,
|
|
2344
|
+
diagnostic: probe.diagnostic,
|
|
2345
|
+
attemptDiagnostics: probe.error.attemptDiagnostics ?? [],
|
|
2346
|
+
},
|
|
2347
|
+
};
|
|
2348
|
+
}
|
|
2349
|
+
|
|
2350
|
+
async function checkCdpInspectModeStatus(
|
|
2351
|
+
context: ToolContext,
|
|
2352
|
+
autoCandidate: boolean,
|
|
2353
|
+
): Promise<BrowserStatusModeResult> {
|
|
2354
|
+
const cdpInspectConfig = getConfig().hostBrowser.cdpInspect;
|
|
2355
|
+
const desktopAutoEnabled =
|
|
2356
|
+
context.transportInterface === "macos" &&
|
|
2357
|
+
cdpInspectConfig.desktopAuto.enabled;
|
|
2358
|
+
const cooldownActive =
|
|
2359
|
+
desktopAutoEnabled &&
|
|
2360
|
+
isDesktopAutoCooldownActive(cdpInspectConfig.desktopAuto.cooldownMs);
|
|
2361
|
+
|
|
2362
|
+
const probe = await probePinnedBrowserMode(
|
|
2363
|
+
BROWSER_STATUS_MODE.CDP_INSPECT,
|
|
2364
|
+
context,
|
|
2365
|
+
);
|
|
2366
|
+
if (probe.ok) {
|
|
2367
|
+
return {
|
|
2368
|
+
mode: BROWSER_STATUS_MODE.CDP_INSPECT,
|
|
2369
|
+
available: true,
|
|
2370
|
+
verified: "active_probe",
|
|
2371
|
+
autoCandidate,
|
|
2372
|
+
summary:
|
|
2373
|
+
"CDP inspect mode is ready and responded to an active CDP probe.",
|
|
2374
|
+
userActions: [],
|
|
2375
|
+
tradeoffs: modeTradeoffs(BROWSER_STATUS_MODE.CDP_INSPECT),
|
|
2376
|
+
details: {
|
|
2377
|
+
backendKind: probe.backendKind,
|
|
2378
|
+
configEnabled: cdpInspectConfig.enabled,
|
|
2379
|
+
configHost: cdpInspectConfig.host,
|
|
2380
|
+
configPort: cdpInspectConfig.port,
|
|
2381
|
+
desktopAutoEnabled,
|
|
2382
|
+
desktopAutoCooldownActive: cooldownActive,
|
|
2383
|
+
},
|
|
2384
|
+
};
|
|
2385
|
+
}
|
|
2386
|
+
|
|
2387
|
+
return {
|
|
2388
|
+
mode: BROWSER_STATUS_MODE.CDP_INSPECT,
|
|
2389
|
+
available: false,
|
|
2390
|
+
verified: "active_probe",
|
|
2391
|
+
autoCandidate,
|
|
2392
|
+
summary: `CDP inspect probe failed: ${probe.error.message}`,
|
|
2393
|
+
userActions: probeFailureActions(
|
|
2394
|
+
BROWSER_STATUS_MODE.CDP_INSPECT,
|
|
2395
|
+
probe.error,
|
|
2396
|
+
),
|
|
2397
|
+
tradeoffs: modeTradeoffs(BROWSER_STATUS_MODE.CDP_INSPECT),
|
|
2398
|
+
details: {
|
|
2399
|
+
errorCode: probe.error.code,
|
|
2400
|
+
discoveryCodes: extractDiscoveryCodes(probe.error),
|
|
2401
|
+
diagnostic: probe.diagnostic,
|
|
2402
|
+
attemptDiagnostics: probe.error.attemptDiagnostics ?? [],
|
|
2403
|
+
configEnabled: cdpInspectConfig.enabled,
|
|
2404
|
+
configHost: cdpInspectConfig.host,
|
|
2405
|
+
configPort: cdpInspectConfig.port,
|
|
2406
|
+
desktopAutoEnabled,
|
|
2407
|
+
desktopAutoCooldownActive: cooldownActive,
|
|
2408
|
+
},
|
|
2409
|
+
};
|
|
2410
|
+
}
|
|
2411
|
+
|
|
2412
|
+
async function checkLocalModeStatus(
|
|
2413
|
+
context: ToolContext,
|
|
2414
|
+
autoCandidate: boolean,
|
|
2415
|
+
checkLocalLaunch: boolean,
|
|
2416
|
+
): Promise<BrowserStatusModeResult> {
|
|
2417
|
+
const runtime = await checkBrowserRuntime();
|
|
2418
|
+
if (!runtime.playwrightAvailable || !runtime.chromiumInstalled) {
|
|
2419
|
+
return {
|
|
2420
|
+
mode: BROWSER_STATUS_MODE.LOCAL,
|
|
2421
|
+
available: false,
|
|
2422
|
+
verified: "preflight",
|
|
2423
|
+
autoCandidate,
|
|
2424
|
+
summary:
|
|
2425
|
+
runtime.error ??
|
|
2426
|
+
"Local mode preflight failed: Playwright Chromium runtime is not ready.",
|
|
2427
|
+
userActions: localSetupActions(),
|
|
2428
|
+
tradeoffs: modeTradeoffs(BROWSER_STATUS_MODE.LOCAL),
|
|
2429
|
+
details: {
|
|
2430
|
+
runtime,
|
|
2431
|
+
launchProbeRequested: checkLocalLaunch,
|
|
2432
|
+
},
|
|
2433
|
+
};
|
|
2434
|
+
}
|
|
2435
|
+
|
|
2436
|
+
if (!checkLocalLaunch) {
|
|
2437
|
+
return {
|
|
2438
|
+
mode: BROWSER_STATUS_MODE.LOCAL,
|
|
2439
|
+
available: true,
|
|
2440
|
+
verified: "preflight",
|
|
2441
|
+
autoCandidate,
|
|
2442
|
+
summary:
|
|
2443
|
+
"Local mode preflight passed (Playwright + Chromium are present). Launch probe was skipped.",
|
|
2444
|
+
userActions: [],
|
|
2445
|
+
tradeoffs: modeTradeoffs(BROWSER_STATUS_MODE.LOCAL),
|
|
2446
|
+
details: {
|
|
2447
|
+
runtime,
|
|
2448
|
+
launchProbeRequested: checkLocalLaunch,
|
|
2449
|
+
},
|
|
2450
|
+
};
|
|
2451
|
+
}
|
|
2452
|
+
|
|
2453
|
+
const probe = await probePinnedBrowserMode(
|
|
2454
|
+
BROWSER_STATUS_MODE.LOCAL,
|
|
2455
|
+
context,
|
|
2456
|
+
);
|
|
2457
|
+
if (probe.ok) {
|
|
2458
|
+
return {
|
|
2459
|
+
mode: BROWSER_STATUS_MODE.LOCAL,
|
|
2460
|
+
available: true,
|
|
2461
|
+
verified: "active_probe",
|
|
2462
|
+
autoCandidate,
|
|
2463
|
+
summary: "Local mode is ready and responded to an active CDP probe.",
|
|
2464
|
+
userActions: [],
|
|
2465
|
+
tradeoffs: modeTradeoffs(BROWSER_STATUS_MODE.LOCAL),
|
|
2466
|
+
details: {
|
|
2467
|
+
runtime,
|
|
2468
|
+
launchProbeRequested: checkLocalLaunch,
|
|
2469
|
+
backendKind: probe.backendKind,
|
|
2470
|
+
},
|
|
2471
|
+
};
|
|
2472
|
+
}
|
|
2473
|
+
|
|
2474
|
+
return {
|
|
2475
|
+
mode: BROWSER_STATUS_MODE.LOCAL,
|
|
2476
|
+
available: false,
|
|
2477
|
+
verified: "active_probe",
|
|
2478
|
+
autoCandidate,
|
|
2479
|
+
summary: `Local mode probe failed: ${probe.error.message}`,
|
|
2480
|
+
userActions: probeFailureActions(BROWSER_STATUS_MODE.LOCAL, probe.error),
|
|
2481
|
+
tradeoffs: modeTradeoffs(BROWSER_STATUS_MODE.LOCAL),
|
|
2482
|
+
details: {
|
|
2483
|
+
runtime,
|
|
2484
|
+
launchProbeRequested: checkLocalLaunch,
|
|
2485
|
+
errorCode: probe.error.code,
|
|
2486
|
+
diagnostic: probe.diagnostic,
|
|
2487
|
+
attemptDiagnostics: probe.error.attemptDiagnostics ?? [],
|
|
2488
|
+
},
|
|
2489
|
+
};
|
|
2490
|
+
}
|
|
2491
|
+
|
|
2492
|
+
// ── browser_status ────────────────────────────────────────────────────
|
|
2493
|
+
|
|
2494
|
+
export async function executeBrowserStatus(
|
|
2495
|
+
input: Record<string, unknown>,
|
|
2496
|
+
context: ToolContext,
|
|
2497
|
+
): Promise<ToolExecutionResult> {
|
|
2498
|
+
const parsedMode = parseBrowserMode(input);
|
|
2499
|
+
if (!parsedMode.ok) {
|
|
2500
|
+
return { content: parsedMode.error, isError: true };
|
|
2501
|
+
}
|
|
2502
|
+
|
|
2503
|
+
if (
|
|
2504
|
+
input[BROWSER_STATUS_INPUT_FIELD.CHECK_LOCAL_LAUNCH] !== undefined &&
|
|
2505
|
+
typeof input[BROWSER_STATUS_INPUT_FIELD.CHECK_LOCAL_LAUNCH] !== "boolean"
|
|
2506
|
+
) {
|
|
2507
|
+
return {
|
|
2508
|
+
content: `Error: ${BROWSER_STATUS_INPUT_FIELD.CHECK_LOCAL_LAUNCH} must be a boolean when provided.`,
|
|
2509
|
+
isError: true,
|
|
2510
|
+
};
|
|
2511
|
+
}
|
|
2512
|
+
|
|
2513
|
+
const checkLocalLaunch =
|
|
2514
|
+
input[BROWSER_STATUS_INPUT_FIELD.CHECK_LOCAL_LAUNCH] === true;
|
|
2515
|
+
const requestedMode = parsedMode.mode;
|
|
2516
|
+
const modesToCheck: readonly StatusCheckMode[] =
|
|
2517
|
+
requestedMode === BROWSER_MODE.AUTO
|
|
2518
|
+
? BROWSER_STATUS_MODES
|
|
2519
|
+
: [requestedMode];
|
|
2520
|
+
|
|
2521
|
+
const autoCandidateKinds = buildCandidateList(context).map((c) => c.kind);
|
|
2522
|
+
const autoCandidateSet = new Set<CdpClientKind>(autoCandidateKinds);
|
|
2523
|
+
|
|
2524
|
+
try {
|
|
2525
|
+
const modeResults: BrowserStatusModeResult[] = [];
|
|
2526
|
+
for (const mode of modesToCheck) {
|
|
2527
|
+
const autoCandidate = autoCandidateSet.has(mode);
|
|
2528
|
+
if (mode === BROWSER_STATUS_MODE.EXTENSION) {
|
|
2529
|
+
modeResults.push(
|
|
2530
|
+
await checkExtensionModeStatus(context, autoCandidate),
|
|
2531
|
+
);
|
|
2532
|
+
} else if (mode === BROWSER_STATUS_MODE.CDP_INSPECT) {
|
|
2533
|
+
modeResults.push(
|
|
2534
|
+
await checkCdpInspectModeStatus(context, autoCandidate),
|
|
2535
|
+
);
|
|
2536
|
+
} else {
|
|
2537
|
+
modeResults.push(
|
|
2538
|
+
await checkLocalModeStatus(context, autoCandidate, checkLocalLaunch),
|
|
2539
|
+
);
|
|
2540
|
+
}
|
|
2541
|
+
}
|
|
2542
|
+
|
|
2543
|
+
const stickyMode = browserManager.getPreferredBackendKind(
|
|
2544
|
+
context.conversationId,
|
|
2545
|
+
);
|
|
2546
|
+
const availableModes = modeResults
|
|
2547
|
+
.filter((r) => r.available)
|
|
2548
|
+
.map((r) => r.mode);
|
|
2549
|
+
const recommendedMode =
|
|
2550
|
+
autoCandidateKinds.find((candidate) =>
|
|
2551
|
+
modeResults.some(
|
|
2552
|
+
(result) => result.mode === candidate && result.available,
|
|
2553
|
+
),
|
|
2554
|
+
) ??
|
|
2555
|
+
availableModes[0] ??
|
|
2556
|
+
null;
|
|
2557
|
+
|
|
2558
|
+
return {
|
|
2559
|
+
content: JSON.stringify(
|
|
2560
|
+
{
|
|
2561
|
+
requestedMode,
|
|
2562
|
+
checkedModes: modesToCheck,
|
|
2563
|
+
autoCandidateOrder: autoCandidateKinds,
|
|
2564
|
+
stickyConversationMode: stickyMode,
|
|
2565
|
+
recommendedMode,
|
|
2566
|
+
checkLocalLaunch,
|
|
2567
|
+
modes: modeResults,
|
|
2568
|
+
},
|
|
2569
|
+
null,
|
|
2570
|
+
2,
|
|
2571
|
+
),
|
|
2572
|
+
isError: false,
|
|
2573
|
+
};
|
|
2574
|
+
} catch (err) {
|
|
2575
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
2576
|
+
return {
|
|
2577
|
+
content: `Error: browser status check failed: ${msg}`,
|
|
2578
|
+
isError: true,
|
|
2579
|
+
};
|
|
2580
|
+
}
|
|
2581
|
+
}
|