@vellumai/assistant 0.6.2 → 0.6.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +273 -10
- package/Dockerfile +2 -3
- package/bun.lock +41 -49
- package/bunfig.toml +3 -0
- package/docs/architecture/memory.md +1 -1
- package/docs/backup-troubleshooting.md +52 -0
- package/docs/browser-use-architecture-phase2.md +174 -0
- package/docs/stt-provider-onboarding.md +120 -0
- package/knip.json +12 -2
- package/node_modules/@vellumai/ces-contracts/bun.lock +8 -6
- package/node_modules/@vellumai/ces-contracts/package.json +3 -3
- package/node_modules/@vellumai/ces-contracts/src/rpc.ts +42 -0
- package/openapi.yaml +1111 -86
- package/package.json +40 -42
- package/scripts/generate-openapi.ts +0 -2
- package/scripts/test.sh +73 -18
- package/src/__tests__/acp-session.test.ts +43 -0
- package/src/__tests__/agent-image-optimize.test.ts +28 -0
- package/src/__tests__/agent-loop.test.ts +123 -0
- package/src/__tests__/anthropic-provider.test.ts +263 -10
- package/src/__tests__/app-builder-tool-scripts.test.ts +1 -0
- package/src/__tests__/app-executors.test.ts +1 -0
- package/src/__tests__/app-source-watcher.test.ts +37 -11
- package/src/__tests__/approval-routes-http.test.ts +178 -1
- package/src/__tests__/auto-analysis-end-to-end.test.ts +550 -0
- package/src/__tests__/auto-analysis-prompt.test.ts +50 -0
- package/src/__tests__/browser-fill-credential.test.ts +240 -94
- package/src/__tests__/browser-manager.test.ts +40 -27
- package/src/__tests__/browser-skill-baseline-tool-payload.test.ts +2 -2
- package/src/__tests__/browser-skill-endstate.test.ts +31 -7
- package/src/__tests__/btw-routes.test.ts +7 -0
- package/src/__tests__/call-controller.test.ts +581 -20
- package/src/__tests__/catalog-files.test.ts +1000 -0
- package/src/__tests__/channel-approvals.test.ts +53 -0
- package/src/__tests__/channel-invite-transport.test.ts +2 -2
- package/src/__tests__/channel-readiness-routes.test.ts +16 -20
- package/src/__tests__/channel-readiness-service.test.ts +12 -7
- package/src/__tests__/checker.test.ts +157 -10
- package/src/__tests__/clawhub-files.test.ts +347 -0
- package/src/__tests__/commit-message-enrichment-service.test.ts +36 -19
- package/src/__tests__/config-analysis.test.ts +100 -0
- package/src/__tests__/config-managed-gemini-defaults.test.ts +326 -0
- package/src/__tests__/config-schema-cmd.test.ts +2 -2
- package/src/__tests__/config-schema.test.ts +1248 -224
- package/src/__tests__/config-watcher-cleanup-throttle.test.ts +339 -0
- package/src/__tests__/config-watcher.test.ts +43 -8
- package/src/__tests__/confirmation-request-guardian-bridge.test.ts +23 -0
- package/src/__tests__/contact-store-user-file.test.ts +512 -0
- package/src/__tests__/contacts-write.test.ts +197 -0
- package/src/__tests__/context-overflow-approval.test.ts +16 -1
- package/src/__tests__/context-window-manager.test.ts +88 -0
- package/src/__tests__/conversation-abort-tool-results.test.ts +2 -0
- package/src/__tests__/conversation-agent-loop-overflow.test.ts +2 -1
- package/src/__tests__/conversation-agent-loop.test.ts +99 -3
- package/src/__tests__/conversation-analysis-routes.test.ts +2 -2
- package/src/__tests__/conversation-attachments.test.ts +80 -4
- package/src/__tests__/conversation-confirmation-signals.test.ts +290 -0
- package/src/__tests__/conversation-error.test.ts +70 -0
- package/src/__tests__/conversation-fork-crud.test.ts +17 -0
- package/src/__tests__/conversation-history-web-search.test.ts +12 -4
- package/src/__tests__/conversation-host-access-routes.test.ts +229 -0
- package/src/__tests__/conversation-init.benchmark.test.ts +6 -1
- package/src/__tests__/conversation-inject-context.test.ts +103 -0
- package/src/__tests__/conversation-launcher-skill-regression.test.ts +51 -0
- package/src/__tests__/conversation-list-source.test.ts +145 -0
- package/src/__tests__/conversation-pre-run-repair.test.ts +2 -0
- package/src/__tests__/conversation-provider-retry-repair.test.ts +2 -0
- package/src/__tests__/conversation-queue.test.ts +946 -62
- package/src/__tests__/conversation-routes-disk-view.test.ts +275 -0
- package/src/__tests__/conversation-routes-guardian-reply.test.ts +16 -0
- package/src/__tests__/conversation-routes-slash-commands.test.ts +1 -0
- package/src/__tests__/conversation-runtime-assembly.test.ts +324 -46
- package/src/__tests__/conversation-skill-tools.test.ts +7 -4
- package/src/__tests__/conversation-slash-commands.test.ts +33 -0
- package/src/__tests__/conversation-slash-queue.test.ts +89 -18
- package/src/__tests__/conversation-slash-unknown.test.ts +2 -0
- package/src/__tests__/conversation-starter-routes.test.ts +126 -0
- package/src/__tests__/conversation-starters-cadence.test.ts +161 -0
- package/src/__tests__/conversation-store.test.ts +195 -0
- package/src/__tests__/conversation-tool-setup-batch-authorized.test.ts +226 -0
- package/src/__tests__/conversation-workspace-cache-state.test.ts +193 -0
- package/src/__tests__/conversation-workspace-injection.test.ts +2 -0
- package/src/__tests__/conversation-workspace-tool-tracking.test.ts +2 -0
- package/src/__tests__/credential-execution-approval-bridge.test.ts +32 -1
- package/src/__tests__/credential-health-service.test.ts +352 -0
- package/src/__tests__/credential-security-invariants.test.ts +6 -3
- package/src/__tests__/credential-vault-unit.test.ts +383 -7
- package/src/__tests__/credential-vault.test.ts +152 -13
- package/src/__tests__/credentials-cli.test.ts +42 -18
- package/src/__tests__/cross-provider-web-search.test.ts +146 -35
- package/src/__tests__/date-context.test.ts +4 -4
- package/src/__tests__/deterministic-verification-control-plane.test.ts +10 -1
- package/src/__tests__/device-id.test.ts +112 -0
- package/src/__tests__/docker-signing-key-bootstrap.test.ts +167 -4
- package/src/__tests__/dynamic-skill-workflow-prompt.test.ts +1 -3
- package/src/__tests__/email-html-renderer.test.ts +71 -0
- package/src/__tests__/email-invite-adapter.test.ts +36 -32
- package/src/__tests__/embedding-managed-proxy-selection.test.ts +256 -0
- package/src/__tests__/emit-event-signal.test.ts +71 -0
- package/src/__tests__/extension-id-sync-guard.test.ts +222 -0
- package/src/__tests__/fixtures/mock-chrome-extension.ts +386 -0
- package/src/__tests__/gateway-only-enforcement.test.ts +206 -1
- package/src/__tests__/gateway-only-guard.test.ts +2 -0
- package/src/__tests__/gemini-provider.test.ts +66 -2
- package/src/__tests__/get-skill-detail-audit.test.ts +325 -0
- package/src/__tests__/gmail-archive-fallback.test.ts +193 -0
- package/src/__tests__/gmail-archive-gate.test.ts +246 -0
- package/src/__tests__/gmail-preferences.test.ts +117 -0
- package/src/__tests__/guardian-routing-invariants.test.ts +70 -2
- package/src/__tests__/headless-browser-interactions.test.ts +738 -359
- package/src/__tests__/headless-browser-mode.test.ts +614 -0
- package/src/__tests__/headless-browser-navigate.test.ts +528 -49
- package/src/__tests__/headless-browser-read-tools.test.ts +274 -100
- package/src/__tests__/headless-browser-snapshot.test.ts +250 -77
- package/src/__tests__/heartbeat-service.test.ts +70 -17
- package/src/__tests__/home-state-routes.test.ts +162 -0
- package/src/__tests__/host-bash-proxy.test.ts +145 -1
- package/src/__tests__/host-browser-e2e-cloud.test.ts +596 -0
- package/src/__tests__/host-browser-e2e-self-hosted-capability.test.ts +286 -0
- package/src/__tests__/host-browser-e2e-self-hosted.test.ts +374 -0
- package/src/__tests__/host-browser-event-routes.test.ts +350 -0
- package/src/__tests__/host-browser-proxy.test.ts +444 -0
- package/src/__tests__/host-browser-routes.test.ts +198 -0
- package/src/__tests__/host-browser-ws-events-e2e.test.ts +423 -0
- package/src/__tests__/host-cu-proxy.test.ts +166 -1
- package/src/__tests__/host-file-proxy.test.ts +185 -1
- package/src/__tests__/host-file-read-tool.test.ts +52 -0
- package/src/__tests__/host-proxy-interface.test.ts +165 -0
- package/src/__tests__/host-shell-tool.test.ts +1 -11
- package/src/__tests__/http-user-message-parity.test.ts +1 -0
- package/src/__tests__/identity-intro-cache.test.ts +40 -10
- package/src/__tests__/init-feature-flag-overrides.test.ts +38 -112
- package/src/__tests__/integration-status.test.ts +6 -7
- package/src/__tests__/jobs-store-upsert-debounced.test.ts +141 -0
- package/src/__tests__/list-messages-tool-merge.test.ts +37 -12
- package/src/__tests__/llm-context-normalization.test.ts +488 -0
- package/src/__tests__/llm-context-route-provider.test.ts +86 -5
- package/src/__tests__/llm-usage-store.test.ts +363 -0
- package/src/__tests__/mcp-client-auth.test.ts +40 -4
- package/src/__tests__/mcp-health-check.test.ts +10 -3
- package/src/__tests__/media-stream-output.test.ts +555 -0
- package/src/__tests__/media-stream-parser.test.ts +374 -0
- package/src/__tests__/media-stream-server-integration.test.ts +1234 -0
- package/src/__tests__/media-stream-stt-session.test.ts +588 -0
- package/src/__tests__/media-turn-detector.test.ts +440 -0
- package/src/__tests__/message-queue.test.ts +125 -0
- package/src/__tests__/migration-cross-version-compatibility.test.ts +3 -1
- package/src/__tests__/migration-export-http.test.ts +67 -8
- package/src/__tests__/migration-export-streaming.test.ts +66 -0
- package/src/__tests__/migration-import-commit-http.test.ts +109 -7
- package/src/__tests__/migration-import-preflight-http.test.ts +6 -5
- package/src/__tests__/migration-validate-http.test.ts +3 -3
- package/src/__tests__/mock-gateway-ipc.ts +151 -0
- package/src/__tests__/model-intents.test.ts +2 -2
- package/src/__tests__/native-host-marker-sync-guard.test.ts +157 -0
- package/src/__tests__/oauth-apps-routes.test.ts +18 -12
- package/src/__tests__/oauth-cli.test.ts +709 -60
- package/src/__tests__/oauth-connect-orchestrator.test.ts +118 -24
- package/src/__tests__/oauth-provider-seed-logos.test.ts +23 -0
- package/src/__tests__/oauth-provider-serializer.test.ts +147 -10
- package/src/__tests__/oauth-provider-visibility.test.ts +19 -21
- package/src/__tests__/oauth-providers-routes.test.ts +52 -14
- package/src/__tests__/oauth-store.test.ts +1465 -176
- package/src/__tests__/oauth2-gateway-transport.test.ts +460 -26
- package/src/__tests__/onboarding-template-contract.test.ts +81 -70
- package/src/__tests__/openai-provider.test.ts +178 -2
- package/src/__tests__/openai-responses-cutover-guard.test.ts +184 -0
- package/src/__tests__/openai-responses-provider.test.ts +1105 -0
- package/src/__tests__/openrouter-token-estimation.test.ts +100 -0
- package/src/__tests__/outlook-categories.test.ts +1 -1
- package/src/__tests__/outlook-client-automation.test.ts +1 -1
- package/src/__tests__/outlook-compose-tools.test.ts +1 -1
- package/src/__tests__/outlook-email-watcher.test.ts +1 -1
- package/src/__tests__/outlook-follow-up.test.ts +1 -1
- package/src/__tests__/outlook-messaging-provider.test.ts +2 -2
- package/src/__tests__/outlook-trash.test.ts +1 -1
- package/src/__tests__/outlook-unsubscribe.test.ts +32 -3
- package/src/__tests__/permission-checker-host-gate.test.ts +74 -14
- package/src/__tests__/permission-mode.test.ts +28 -56
- package/src/__tests__/persona-resolver.test.ts +251 -0
- package/src/__tests__/platform-bash-auto-approve.test.ts +4 -0
- package/src/__tests__/platform-callback-registration.test.ts +19 -0
- package/src/__tests__/platform.test.ts +92 -1
- package/src/__tests__/post-turn-tool-result-truncation.test.ts +343 -0
- package/src/__tests__/prechat-onboarding-contract.test.ts +267 -0
- package/src/__tests__/pricing.test.ts +174 -0
- package/src/__tests__/proxy-approval-callback.test.ts +18 -0
- package/src/__tests__/qdrant-manager.test.ts +29 -8
- package/src/__tests__/regenerate-fire-and-forget-trace.test.ts +194 -0
- package/src/__tests__/relationship-state-contract.test.ts +175 -0
- package/src/__tests__/relay-server.test.ts +423 -5
- package/src/__tests__/require-fresh-approval.test.ts +40 -1
- package/src/__tests__/sanitize-config-for-transfer.test.ts +132 -0
- package/src/__tests__/schedule-routes.test.ts +162 -0
- package/src/__tests__/search-skills-unified.test.ts +118 -0
- package/src/__tests__/secret-detection-handler.test.ts +84 -0
- package/src/__tests__/secret-ingress-http.test.ts +1 -0
- package/src/__tests__/secret-scanner-executor.test.ts +4 -0
- package/src/__tests__/secure-keys.test.ts +107 -0
- package/src/__tests__/send-endpoint-busy.test.ts +8 -1
- package/src/__tests__/sequence-store.test.ts +1 -1
- package/src/__tests__/server-history-render.test.ts +49 -0
- package/src/__tests__/set-permission-mode.test.ts +13 -250
- package/src/__tests__/settings-routes.test.ts +201 -0
- package/src/__tests__/skill-load-feature-flag.test.ts +1 -0
- package/src/__tests__/skills-file-content-endpoint.test.ts +801 -0
- package/src/__tests__/skills-files-catalog-fallback.test.ts +738 -0
- package/src/__tests__/skills.test.ts +5 -2
- package/src/__tests__/skillssh-files.test.ts +446 -0
- package/src/__tests__/slack-block-formatting.test.ts +110 -0
- package/src/__tests__/slack-channel-config.test.ts +576 -16
- package/src/__tests__/stt-catalog-parity.test.ts +282 -0
- package/src/__tests__/stt-stream-session.test.ts +535 -0
- package/src/__tests__/subagent-detail.test.ts +44 -2
- package/src/__tests__/subagent-disposal.test.ts +1 -0
- package/src/__tests__/subagent-fork-notifications.test.ts +291 -0
- package/src/__tests__/subagent-fork-spawn.test.ts +384 -0
- package/src/__tests__/subagent-manager-notify.test.ts +1 -0
- package/src/__tests__/subagent-notify-parent.test.ts +1 -0
- package/src/__tests__/subagent-spawn-tool-fork.test.ts +411 -0
- package/src/__tests__/subagent-tools.test.ts +1 -0
- package/src/__tests__/subagent-types.test.ts +1 -0
- package/src/__tests__/system-prompt-ask-mode.test.ts +27 -71
- package/src/__tests__/system-prompt.test.ts +184 -27
- package/src/__tests__/task-scheduler.test.ts +32 -6
- package/src/__tests__/telegram-config.test.ts +10 -13
- package/src/__tests__/telephony-stt-routing.test.ts +329 -0
- package/src/__tests__/terminal-tools.test.ts +25 -5
- package/src/__tests__/test-preload.ts +18 -0
- package/src/__tests__/test-support/browser-skill-harness.ts +4 -1
- package/src/__tests__/tool-approval-handler.test.ts +73 -0
- package/src/__tests__/tool-executor-lifecycle-events.test.ts +9 -5
- package/src/__tests__/tool-executor-shell-integration.test.ts +4 -0
- package/src/__tests__/tool-executor.test.ts +33 -24
- package/src/__tests__/tool-result-truncation.test.ts +36 -0
- package/src/__tests__/tool-side-effects-slack-dm.test.ts +22 -0
- package/src/__tests__/top-level-renderer.test.ts +73 -1
- package/src/__tests__/transport-hints-queue.test.ts +14 -29
- package/src/__tests__/trust-store.test.ts +7 -1
- package/src/__tests__/trusted-contact-approval-notifier.test.ts +1 -1
- package/src/__tests__/trusted-contact-inline-approval-integration.test.ts +109 -0
- package/src/__tests__/tts-catalog-parity.test.ts +345 -0
- package/src/__tests__/twilio-routes-twiml.test.ts +512 -114
- package/src/__tests__/twilio-routes.test.ts +376 -0
- package/src/__tests__/unicode.test.ts +293 -0
- package/src/__tests__/update-bulletin-format.test.ts +59 -0
- package/src/__tests__/update-bulletin.test.ts +206 -5
- package/src/__tests__/usage-routes.test.ts +25 -4
- package/src/__tests__/user-reference.test.ts +46 -61
- package/src/__tests__/v2-consent-policy.test.ts +103 -0
- package/src/__tests__/verification-control-plane-policy.test.ts +4 -0
- package/src/__tests__/voice-config-update.test.ts +403 -0
- package/src/__tests__/voice-quality.test.ts +434 -19
- package/src/__tests__/workspace-heartbeat-service.test.ts +7 -0
- package/src/__tests__/workspace-migration-033-stt-service-explicit-config.test.ts +547 -0
- package/src/__tests__/workspace-migration-034-remove-calls-voice-transcription-provider.test.ts +596 -0
- package/src/__tests__/workspace-migration-drop-user-md.test.ts +368 -0
- package/src/__tests__/workspace-migration-meets.test.ts +244 -0
- package/src/__tests__/workspace-migration-seed-device-id.test.ts +14 -20
- package/src/__tests__/workspace-policy.test.ts +2 -0
- package/src/acp/client-handler.ts +30 -4
- package/src/agent/image-optimize.ts +24 -12
- package/src/agent/loop.ts +55 -9
- package/src/approvals/guardian-request-resolvers.ts +21 -15
- package/src/backup/__tests__/backup-key.test.ts +152 -0
- package/src/backup/__tests__/backup-worker.test.ts +767 -0
- package/src/backup/__tests__/list-snapshots.test.ts +87 -0
- package/src/backup/__tests__/local-writer.test.ts +218 -0
- package/src/backup/__tests__/offsite-writer.test.ts +641 -0
- package/src/backup/__tests__/paths.test.ts +300 -0
- package/src/backup/__tests__/restore.test.ts +498 -0
- package/src/backup/__tests__/snapshot-lock.test.ts +352 -0
- package/src/backup/__tests__/stream-crypt.test.ts +228 -0
- package/src/backup/backup-key.ts +137 -0
- package/src/backup/backup-worker.ts +459 -0
- package/src/backup/list-snapshots.ts +147 -0
- package/src/backup/local-writer.ts +133 -0
- package/src/backup/offsite-writer.ts +222 -0
- package/src/backup/paths.ts +226 -0
- package/src/backup/restore.ts +322 -0
- package/src/backup/snapshot-lock.ts +431 -0
- package/src/backup/stream-crypt.ts +263 -0
- package/src/browser-session/__tests__/manager.test.ts +297 -0
- package/src/browser-session/backends/cdp-inspect.ts +30 -0
- package/src/browser-session/backends/extension.ts +26 -0
- package/src/browser-session/backends/local.ts +24 -0
- package/src/browser-session/events.ts +164 -0
- package/src/browser-session/index.ts +27 -0
- package/src/browser-session/manager.ts +159 -0
- package/src/browser-session/types.ts +28 -0
- package/src/bundler/package-resolver.ts +4 -0
- package/src/calls/audio-store.ts +11 -5
- package/src/calls/call-controller.ts +226 -71
- package/src/calls/call-domain.ts +9 -0
- package/src/calls/call-speech-output.ts +190 -0
- package/src/calls/call-transport.ts +77 -0
- package/src/calls/media-stream-audio-transcode.ts +173 -0
- package/src/calls/media-stream-output.ts +660 -0
- package/src/calls/media-stream-parser.ts +300 -0
- package/src/calls/media-stream-protocol.ts +166 -0
- package/src/calls/media-stream-server.ts +592 -0
- package/src/calls/media-stream-stt-session.ts +460 -0
- package/src/calls/media-turn-detector.ts +230 -0
- package/src/calls/relay-server.ts +90 -75
- package/src/calls/resolve-call-tts-provider.ts +136 -0
- package/src/calls/telephony-stt-routing.ts +145 -0
- package/src/calls/tts-call-strategy.ts +161 -0
- package/src/calls/tts-text-sanitizer.ts +32 -16
- package/src/calls/twilio-routes.ts +281 -17
- package/src/calls/voice-quality.ts +78 -35
- package/src/calls/voice-session-bridge.ts +8 -1
- package/src/channels/__tests__/types.test.ts +134 -0
- package/src/channels/types.ts +69 -3
- package/src/cli/__tests__/run-assistant-command.ts +11 -1
- package/src/cli/commands/__tests__/backup.test.ts +1165 -0
- package/src/cli/commands/__tests__/domain-register.test.ts +234 -0
- package/src/cli/commands/__tests__/domain-status.test.ts +132 -0
- package/src/cli/commands/__tests__/email-attachment.test.ts +422 -0
- package/src/cli/commands/__tests__/email-download.test.ts +16 -1
- package/src/cli/commands/__tests__/email-list.test.ts +22 -4
- package/src/cli/commands/__tests__/email-register.test.ts +4 -4
- package/src/cli/commands/__tests__/email-send.test.ts +37 -4
- package/src/cli/commands/__tests__/email-status.test.ts +5 -1
- package/src/cli/commands/__tests__/email-unregister.test.ts +34 -5
- package/src/cli/commands/backup.ts +993 -0
- package/src/cli/commands/conversations.ts +77 -0
- package/src/cli/commands/credentials.ts +3 -4
- package/src/cli/commands/domain.ts +210 -0
- package/src/cli/commands/email.ts +273 -16
- package/src/cli/commands/mcp.ts +16 -4
- package/src/cli/commands/oauth/__tests__/connect.test.ts +56 -44
- package/src/cli/commands/oauth/__tests__/disconnect.test.ts +21 -21
- package/src/cli/commands/oauth/__tests__/mode.test.ts +17 -17
- package/src/cli/commands/oauth/__tests__/ping.test.ts +16 -16
- package/src/cli/commands/oauth/__tests__/providers-delete.test.ts +32 -33
- package/src/cli/commands/oauth/__tests__/providers-register.test.ts +330 -0
- package/src/cli/commands/oauth/__tests__/providers-update.test.ts +117 -12
- package/src/cli/commands/oauth/__tests__/status.test.ts +10 -10
- package/src/cli/commands/oauth/__tests__/token.test.ts +7 -7
- package/src/cli/commands/oauth/apps.ts +7 -4
- package/src/cli/commands/oauth/connect.ts +6 -3
- package/src/cli/commands/oauth/disconnect.ts +1 -1
- package/src/cli/commands/oauth/mode.ts +12 -3
- package/src/cli/commands/oauth/providers.ts +215 -36
- package/src/cli/commands/oauth/shared.ts +7 -6
- package/src/cli/commands/platform/__tests__/callback-routes-list.test.ts +254 -0
- package/src/cli/commands/platform/__tests__/connect.test.ts +6 -0
- package/src/cli/commands/platform/__tests__/disconnect.test.ts +7 -1
- package/src/cli/commands/platform/__tests__/status.test.ts +6 -0
- package/src/cli/commands/platform/index.ts +107 -10
- package/src/cli/commands/usage.ts +10 -9
- package/src/cli/lib/daemon-credential-client.ts +4 -0
- package/src/cli/program.ts +30 -4
- package/src/config/__tests__/backup-schema.test.ts +134 -0
- package/src/config/assistant-feature-flags.ts +61 -62
- package/src/config/bundled-skills/app-builder/SKILL.md +26 -249
- package/src/config/bundled-skills/app-builder/references/CUSTOM_ROUTES.md +141 -0
- package/src/config/bundled-skills/app-builder/references/INTERACTION_HOOKS.md +56 -0
- package/src/config/bundled-skills/app-builder/references/WIDGETS.md +125 -0
- package/src/config/bundled-skills/browser/SKILL.md +30 -5
- package/src/config/bundled-skills/browser/TOOLS.json +123 -0
- package/src/config/bundled-skills/browser/tools/browser-attach.ts +12 -0
- package/src/config/bundled-skills/browser/tools/browser-detach.ts +12 -0
- package/src/config/bundled-skills/browser/tools/browser-status.ts +12 -0
- package/src/config/bundled-skills/browser/tools/browser-wait-for-download.ts +17 -0
- package/src/config/bundled-skills/contacts/SKILL.md +5 -2
- package/src/config/bundled-skills/document/SKILL.md +4 -0
- package/src/config/bundled-skills/gmail/SKILL.md +54 -8
- package/src/config/bundled-skills/gmail/TOOLS.json +33 -3
- package/src/config/bundled-skills/gmail/tools/gmail-archive.ts +116 -9
- package/src/config/bundled-skills/gmail/tools/gmail-outreach-scan.ts +138 -11
- package/src/config/bundled-skills/gmail/tools/gmail-preferences-tool.ts +59 -0
- package/src/config/bundled-skills/gmail/tools/gmail-preferences.ts +82 -0
- package/src/config/bundled-skills/gmail/tools/gmail-sender-digest.ts +113 -17
- package/src/config/bundled-skills/gmail/tools/gmail-unsubscribe.ts +2 -2
- package/src/config/bundled-skills/media-processing/SKILL.md +3 -9
- package/src/config/bundled-skills/media-processing/TOOLS.json +1 -6
- package/src/config/bundled-skills/media-processing/__tests__/audio-transcribe.test.ts +125 -0
- package/src/config/bundled-skills/media-processing/__tests__/extract-keyframes.test.ts +181 -0
- package/src/config/bundled-skills/media-processing/__tests__/preprocess-audio.test.ts +141 -0
- package/src/config/bundled-skills/media-processing/services/audio-transcribe.ts +32 -87
- package/src/config/bundled-skills/media-processing/services/preprocess.ts +8 -4
- package/src/config/bundled-skills/media-processing/tools/extract-keyframes.ts +0 -10
- package/src/config/bundled-skills/messaging/SKILL.md +3 -3
- package/src/config/bundled-skills/messaging/tools/messaging-archive-by-sender.ts +2 -2
- package/src/config/bundled-skills/outlook/SKILL.md +9 -2
- package/src/config/bundled-skills/outlook/tools/outlook-unsubscribe.ts +2 -2
- package/src/config/bundled-skills/phone-calls/SKILL.md +2 -2
- package/src/config/bundled-skills/phone-calls/references/CONFIG.md +27 -18
- package/src/config/bundled-skills/phone-calls/references/TROUBLESHOOTING.md +3 -3
- package/src/config/bundled-skills/settings/TOOLS.json +3 -3
- package/src/config/bundled-skills/settings/tools/voice-config-update.ts +26 -22
- package/src/config/bundled-skills/slack/SKILL.md +1 -0
- package/src/config/bundled-skills/subagent/SKILL.md +21 -0
- package/src/config/bundled-skills/subagent/TOOLS.json +8 -4
- package/src/config/bundled-skills/tasks/SKILL.md +5 -0
- package/src/config/bundled-skills/transcribe/SKILL.md +9 -14
- package/src/config/bundled-skills/transcribe/TOOLS.json +2 -7
- package/src/config/bundled-skills/transcribe/tools/transcribe-media.test.ts +256 -0
- package/src/config/bundled-skills/transcribe/tools/transcribe-media.ts +38 -188
- package/src/config/bundled-tool-registry.ts +8 -0
- package/src/config/env-registry.ts +38 -0
- package/src/config/env.ts +49 -4
- package/src/config/feature-flag-registry.json +85 -14
- package/src/config/loader.ts +82 -13
- package/src/config/sanitize-for-transfer.ts +47 -0
- package/src/config/schema.ts +81 -15
- package/src/config/schemas/__tests__/stt.test.ts +43 -0
- package/src/config/schemas/analysis.ts +51 -0
- package/src/config/schemas/backup.ts +72 -0
- package/src/config/schemas/calls.ts +1 -26
- package/src/config/schemas/elevenlabs.ts +0 -59
- package/src/config/schemas/filing.ts +47 -7
- package/src/config/schemas/heartbeat.ts +27 -5
- package/src/config/schemas/host-browser.ts +112 -0
- package/src/config/schemas/inference.ts +1 -1
- package/src/config/schemas/memory-lifecycle.ts +14 -2
- package/src/config/schemas/memory-retrieval.ts +103 -0
- package/src/config/schemas/security.ts +0 -6
- package/src/config/schemas/services.ts +52 -0
- package/src/config/schemas/stt.ts +59 -0
- package/src/config/schemas/tts.ts +230 -0
- package/src/config/schemas/updates.ts +14 -0
- package/src/config/skills.ts +4 -0
- package/src/config/types.ts +4 -1
- package/src/contacts/contact-store.ts +56 -11
- package/src/contacts/contacts-write.ts +38 -1
- package/src/context/post-turn-tool-result-truncation.ts +177 -0
- package/src/context/tool-result-truncation.ts +2 -1
- package/src/context/window-manager.ts +61 -10
- package/src/credential-execution/approval-bridge.ts +49 -15
- package/src/credential-execution/executable-discovery.ts +12 -2
- package/src/credential-execution/process-manager.ts +33 -2
- package/src/credential-health/credential-health-service.ts +366 -0
- package/src/daemon/__tests__/conversation-lifecycle-auto-analyze.test.ts +324 -0
- package/src/daemon/__tests__/conversation-surfaces-launch.test.ts +497 -0
- package/src/daemon/__tests__/conversation-tool-setup.test.ts +195 -0
- package/src/daemon/__tests__/lifecycle-startup-ordering.test.ts +127 -0
- package/src/daemon/app-source-watcher.ts +35 -0
- package/src/daemon/config-watcher.ts +99 -5
- package/src/daemon/context-overflow-approval.ts +5 -0
- package/src/daemon/conversation-agent-loop-handlers.ts +23 -2
- package/src/daemon/conversation-agent-loop.ts +153 -42
- package/src/daemon/conversation-attachments.ts +40 -0
- package/src/daemon/conversation-error.ts +11 -0
- package/src/daemon/conversation-history.ts +40 -6
- package/src/daemon/conversation-launch.ts +220 -0
- package/src/daemon/conversation-lifecycle.ts +59 -9
- package/src/daemon/conversation-messaging.ts +37 -3
- package/src/daemon/conversation-notifiers.ts +5 -0
- package/src/daemon/conversation-process.ts +622 -13
- package/src/daemon/conversation-queue-manager.ts +24 -0
- package/src/daemon/conversation-runtime-assembly.ts +128 -36
- package/src/daemon/conversation-slash.ts +36 -0
- package/src/daemon/conversation-surfaces.ts +131 -40
- package/src/daemon/conversation-tool-setup.ts +99 -8
- package/src/daemon/conversation-usage.ts +7 -4
- package/src/daemon/conversation-workspace.ts +12 -0
- package/src/daemon/conversation.ts +292 -16
- package/src/daemon/date-context.ts +10 -10
- package/src/daemon/first-greeting.ts +3 -2
- package/src/daemon/handlers/config-slack-channel.ts +269 -94
- package/src/daemon/handlers/conversations.ts +13 -141
- package/src/daemon/handlers/shared.ts +80 -0
- package/src/daemon/handlers/skills.ts +483 -44
- package/src/daemon/host-bash-proxy.ts +48 -13
- package/src/daemon/host-browser-proxy.ts +192 -0
- package/src/daemon/host-cu-proxy.ts +36 -11
- package/src/daemon/host-file-proxy.ts +57 -9
- package/src/daemon/lifecycle.ts +179 -28
- package/src/daemon/message-protocol.ts +13 -0
- package/src/daemon/message-types/conversations.ts +89 -14
- package/src/daemon/message-types/home.ts +40 -0
- package/src/daemon/message-types/host-browser.ts +100 -0
- package/src/daemon/message-types/meet.ts +143 -0
- package/src/daemon/message-types/messages.ts +19 -5
- package/src/daemon/message-types/schedules.ts +34 -2
- package/src/daemon/message-types/skills.ts +26 -0
- package/src/daemon/message-types/subagents.ts +2 -0
- package/src/daemon/message-types/surfaces.ts +2 -0
- package/src/daemon/server.ts +439 -14
- package/src/daemon/shutdown-handlers.ts +32 -4
- package/src/daemon/shutdown-registry.ts +40 -0
- package/src/daemon/tool-side-effects.ts +15 -0
- package/src/daemon/transport-hints.ts +5 -24
- package/src/email/html-renderer.ts +76 -0
- package/src/heartbeat/heartbeat-service.ts +93 -7
- package/src/home/__tests__/assistant-feed-authoring.test.ts +156 -0
- package/src/home/__tests__/emit-feed-event.test.ts +169 -0
- package/src/home/__tests__/feed-scheduler.test.ts +194 -0
- package/src/home/__tests__/feed-types.test.ts +275 -0
- package/src/home/__tests__/feed-writer.test.ts +688 -0
- package/src/home/__tests__/phase5-exit-criteria.test.ts +212 -0
- package/src/home/__tests__/platform-gmail-digest.test.ts +222 -0
- package/src/home/__tests__/progress-formula.test.ts +213 -0
- package/src/home/__tests__/relationship-state-writer.test.ts +740 -0
- package/src/home/__tests__/rollup-producer.test.ts +398 -0
- package/src/home/assistant-feed-authoring.ts +124 -0
- package/src/home/emit-feed-event.ts +158 -0
- package/src/home/feed-scheduler.ts +247 -0
- package/src/home/feed-types.ts +181 -0
- package/src/home/feed-writer.ts +469 -0
- package/src/home/platform-gmail-digest.ts +163 -0
- package/src/home/progress-formula.ts +86 -0
- package/src/home/relationship-state-writer.ts +824 -0
- package/src/home/relationship-state.ts +143 -0
- package/src/home/rollup-producer.ts +384 -0
- package/src/hooks/runner.ts +7 -0
- package/src/inbound/platform-callback-registration.ts +30 -20
- package/src/inbound/public-ingress-urls.ts +12 -0
- package/src/instrument.ts +1 -1
- package/src/ipc/__tests__/cli-ipc.test.ts +200 -0
- package/src/ipc/cli-client.ts +151 -0
- package/src/ipc/cli-server.ts +234 -0
- package/src/ipc/gateway-client.ts +180 -0
- package/src/ipc/routes/index.ts +5 -0
- package/src/ipc/routes/wake-conversation.ts +19 -0
- package/src/mcp/client.ts +59 -24
- package/src/memory/__tests__/auto-analysis-enqueue.test.ts +356 -0
- package/src/memory/__tests__/auto-analysis-guard.test.ts +57 -0
- package/src/memory/__tests__/conversation-analyze-job.test.ts +232 -0
- package/src/memory/__tests__/find-analysis-conversation.test.ts +196 -0
- package/src/memory/app-store.ts +31 -1
- package/src/memory/attachments-store.ts +70 -0
- package/src/memory/auto-analysis-enqueue.ts +127 -0
- package/src/memory/auto-analysis-guard.ts +27 -0
- package/src/memory/cleanup-schedule-state.ts +37 -0
- package/src/memory/conversation-analyze-job.ts +73 -0
- package/src/memory/conversation-crud.ts +122 -0
- package/src/memory/conversation-disk-view.ts +7 -0
- package/src/memory/conversation-group-migration.ts +34 -2
- package/src/memory/conversation-queries.ts +6 -5
- package/src/memory/conversation-starters-cadence.ts +76 -0
- package/src/memory/conversation-title-service.ts +5 -2
- package/src/memory/db-init.ts +18 -0
- package/src/memory/db-maintenance.ts +108 -0
- package/src/memory/db.ts +1 -0
- package/src/memory/embedding-backend.test.ts +75 -0
- package/src/memory/embedding-backend.ts +131 -5
- package/src/memory/embedding-gemini.test.ts +54 -0
- package/src/memory/embedding-gemini.ts +20 -9
- package/src/memory/embedding-local.ts +176 -17
- package/src/memory/graph/consolidation.ts +10 -23
- package/src/memory/graph/conversation-graph-memory.ts +15 -0
- package/src/memory/graph/extraction-job.ts +15 -0
- package/src/memory/graph/extraction.test.ts +23 -0
- package/src/memory/graph/extraction.ts +8 -0
- package/src/memory/graph/retriever.ts +67 -40
- package/src/memory/graph/scoring.test.ts +186 -0
- package/src/memory/graph/scoring.ts +31 -1
- package/src/memory/graph/store.test.ts +7 -3
- package/src/memory/graph/store.ts +47 -12
- package/src/memory/graph/tools.ts +1 -1
- package/src/memory/group-crud.ts +6 -1
- package/src/memory/indexer.ts +95 -16
- package/src/memory/job-handlers/cleanup.ts +11 -8
- package/src/memory/job-handlers/conversation-starters.ts +16 -10
- package/src/memory/jobs-store.ts +64 -4
- package/src/memory/jobs-worker.ts +22 -9
- package/src/memory/llm-usage-store.ts +137 -60
- package/src/memory/migrations/213-oauth-providers-scope-separator.ts +13 -0
- package/src/memory/migrations/214-oauth-providers-refresh-url.ts +11 -0
- package/src/memory/migrations/215-oauth-providers-revoke.ts +14 -0
- package/src/memory/migrations/216-oauth-providers-token-auth-method.ts +30 -0
- package/src/memory/migrations/217-conversation-host-access.ts +40 -0
- package/src/memory/migrations/218-oauth-providers-logo-url.ts +11 -0
- package/src/memory/migrations/219-oauth-providers-token-exchange-body-format.ts +15 -0
- package/src/memory/migrations/220-normalize-user-file-by-principal.ts +190 -0
- package/src/memory/migrations/221-conversations-archived-at.ts +16 -0
- package/src/memory/migrations/index.ts +12 -0
- package/src/memory/migrations/registry.ts +16 -0
- package/src/memory/qdrant-manager.ts +43 -16
- package/src/memory/schema/conversations.ts +3 -0
- package/src/memory/schema/oauth.ts +21 -13
- package/src/memory/usage-buckets.ts +396 -0
- package/src/messaging/providers/gmail/client.ts +57 -6
- package/src/messaging/providers/slack/__tests__/adapter-token-routing.test.ts +282 -0
- package/src/messaging/providers/slack/adapter.ts +143 -38
- package/src/messaging/providers/slack/client.ts +16 -0
- package/src/messaging/providers/slack/types.ts +4 -0
- package/src/notifications/decision-engine.ts +3 -3
- package/src/notifications/signal.ts +5 -0
- package/src/oauth/AGENTS.md +76 -0
- package/src/oauth/__tests__/identity-verifier.test.ts +25 -19
- package/src/oauth/__tests__/seed-providers-managed.test.ts +32 -0
- package/src/oauth/byo-connection.test.ts +26 -9
- package/src/oauth/byo-connection.ts +10 -8
- package/src/oauth/connect-orchestrator.ts +25 -21
- package/src/oauth/connect-types.ts +3 -3
- package/src/oauth/connection-resolver.test.ts +17 -4
- package/src/oauth/connection-resolver.ts +22 -18
- package/src/oauth/connection.ts +3 -1
- package/src/oauth/manual-token-connection.ts +13 -13
- package/src/oauth/oauth-store.ts +223 -100
- package/src/oauth/platform-connection.test.ts +101 -3
- package/src/oauth/platform-connection.ts +56 -35
- package/src/oauth/provider-serializer.ts +31 -5
- package/src/oauth/revoke.ts +76 -0
- package/src/oauth/seed-providers.ts +133 -87
- package/src/oauth/token-persistence.ts +1 -1
- package/src/permissions/checker.ts +16 -6
- package/src/permissions/defaults.ts +49 -1
- package/src/permissions/permission-mode.ts +4 -11
- package/src/permissions/prompter.ts +13 -1
- package/src/permissions/trust-store.ts +3 -3
- package/src/permissions/v2-consent-policy.ts +87 -0
- package/src/permissions/workspace-policy.ts +3 -0
- package/src/platform/client.test.ts +10 -0
- package/src/platform/sync-identity.ts +129 -0
- package/src/prompts/persona-resolver.ts +126 -2
- package/src/prompts/system-prompt.ts +76 -38
- package/src/prompts/templates/BOOTSTRAP-REFERENCE.md +3 -65
- package/src/prompts/templates/BOOTSTRAP.md +59 -105
- package/src/prompts/templates/SOUL.md +3 -1
- package/src/prompts/templates/UPDATES.md +12 -0
- package/src/prompts/templates/channels/slack.md +20 -0
- package/src/prompts/update-bulletin-format.ts +26 -9
- package/src/prompts/update-bulletin.ts +34 -23
- package/src/prompts/user-reference.ts +20 -17
- package/src/providers/__tests__/provider-secret-catalog.test.ts +42 -0
- package/src/providers/anthropic/client.ts +157 -60
- package/src/providers/fireworks/client.ts +2 -2
- package/src/providers/gemini/client.ts +9 -1
- package/src/providers/model-catalog.ts +6 -0
- package/src/providers/model-intents.ts +4 -4
- package/src/providers/ollama/client.ts +2 -2
- package/src/providers/openai/chat-completions-provider.ts +474 -0
- package/src/providers/openai/client.ts +25 -440
- package/src/providers/openai/responses-provider.ts +502 -0
- package/src/providers/openrouter/client.ts +101 -4
- package/src/providers/provider-secret-catalog.ts +139 -0
- package/src/providers/registry.ts +2 -2
- package/src/providers/retry.ts +14 -3
- package/src/providers/speech-to-text/__tests__/provider-catalog.test.ts +251 -0
- package/src/providers/speech-to-text/__tests__/resolve.test.ts +828 -0
- package/src/providers/speech-to-text/deepgram-realtime.test.ts +980 -0
- package/src/providers/speech-to-text/deepgram-realtime.ts +767 -0
- package/src/providers/speech-to-text/deepgram.test.ts +332 -0
- package/src/providers/speech-to-text/deepgram.ts +115 -0
- package/src/providers/speech-to-text/google-gemini-live-stream.test.ts +743 -0
- package/src/providers/speech-to-text/google-gemini-live-stream.ts +625 -0
- package/src/providers/speech-to-text/google-gemini.test.ts +226 -0
- package/src/providers/speech-to-text/google-gemini.ts +101 -0
- package/src/providers/speech-to-text/openai-whisper-stream.test.ts +564 -0
- package/src/providers/speech-to-text/openai-whisper-stream.ts +381 -0
- package/src/providers/speech-to-text/openai-whisper.test.ts +1 -37
- package/src/providers/speech-to-text/openai-whisper.ts +63 -33
- package/src/providers/speech-to-text/provider-catalog.ts +306 -0
- package/src/providers/speech-to-text/resolve.ts +386 -6
- package/src/providers/types.ts +10 -1
- package/src/runtime/AGENTS.md +65 -0
- package/src/runtime/__tests__/agent-wake.test.ts +831 -0
- package/src/runtime/__tests__/browser-extension-pair-routes.test.ts +715 -0
- package/src/runtime/__tests__/capability-tokens.test.ts +258 -0
- package/src/runtime/__tests__/chrome-extension-registry.test.ts +518 -0
- package/src/runtime/__tests__/runtime-mode.test.ts +62 -0
- package/src/runtime/__tests__/slack-block-formatting.test.ts +481 -0
- package/src/runtime/agent-wake.ts +512 -0
- package/src/runtime/assistant-event-hub.ts +2 -2
- package/src/runtime/auth/__tests__/guard-tests.test.ts +1 -0
- package/src/runtime/auth/__tests__/middleware.test.ts +116 -1
- package/src/runtime/auth/__tests__/route-policy.test.ts +48 -0
- package/src/runtime/auth/middleware.ts +98 -0
- package/src/runtime/auth/route-policy.ts +33 -9
- package/src/runtime/auth/token-service.ts +56 -1
- package/src/runtime/btw-sidechain.ts +2 -0
- package/src/runtime/capability-tokens.ts +414 -0
- package/src/runtime/channel-approvals.ts +18 -5
- package/src/runtime/channel-invite-transport.ts +1 -1
- package/src/runtime/channel-invite-transports/email.ts +14 -6
- package/src/runtime/channel-readiness-service.ts +12 -22
- package/src/runtime/chrome-extension-registry.ts +368 -0
- package/src/runtime/confirmation-request-guardian-bridge.ts +6 -0
- package/src/runtime/guardian-decision-types.ts +7 -0
- package/src/runtime/http-server.ts +815 -75
- package/src/runtime/http-types.ts +6 -2
- package/src/runtime/migrations/__tests__/rebind-secrets-credentials.test.ts +172 -0
- package/src/runtime/migrations/__tests__/vbundle-builder-credentials.test.ts +276 -0
- package/src/runtime/migrations/__tests__/vbundle-import-credentials.test.ts +198 -0
- package/src/runtime/migrations/__tests__/vbundle-legacy-user-md.test.ts +360 -0
- package/src/runtime/migrations/migration-transport.ts +7 -0
- package/src/runtime/migrations/migration-wizard.ts +23 -2
- package/src/runtime/migrations/rebind-secrets-screen.ts +76 -15
- package/src/runtime/migrations/vbundle-builder.ts +145 -38
- package/src/runtime/migrations/vbundle-import-analyzer.ts +96 -1
- package/src/runtime/migrations/vbundle-importer.ts +89 -5
- package/src/runtime/pending-interactions.ts +18 -13
- package/src/runtime/routes/__tests__/backup-routes.test.ts +967 -0
- package/src/runtime/routes/__tests__/home-feed-routes.test.ts +507 -0
- package/src/runtime/routes/__tests__/migration-import-credential-filter.test.ts +208 -0
- package/src/runtime/routes/__tests__/stt-routes.test.ts +406 -0
- package/src/runtime/routes/__tests__/tts-routes.test.ts +474 -0
- package/src/runtime/routes/__tests__/user-route-dispatcher.test.ts +148 -17
- package/src/runtime/routes/app-management-routes.ts +12 -18
- package/src/runtime/routes/approval-routes.ts +90 -16
- package/src/runtime/routes/attachment-routes.test.ts +9 -3
- package/src/runtime/routes/attachment-routes.ts +216 -17
- package/src/runtime/routes/backup-routes.ts +519 -0
- package/src/runtime/routes/browser-extension-pair-routes.ts +556 -0
- package/src/runtime/routes/btw-routes.ts +8 -6
- package/src/runtime/routes/contact-routes.test.ts +298 -0
- package/src/runtime/routes/contact-routes.ts +132 -5
- package/src/runtime/routes/conversation-analysis-routes.ts +22 -141
- package/src/runtime/routes/conversation-management-routes.ts +223 -0
- package/src/runtime/routes/conversation-routes.ts +598 -103
- package/src/runtime/routes/conversation-starter-routes.ts +78 -16
- package/src/runtime/routes/filing-routes.ts +93 -0
- package/src/runtime/routes/guardian-action-routes.ts +24 -13
- package/src/runtime/routes/home-feed-routes.ts +334 -0
- package/src/runtime/routes/home-state-routes.ts +138 -0
- package/src/runtime/routes/host-browser-routes.ts +268 -0
- package/src/runtime/routes/host-file-routes.ts +9 -1
- package/src/runtime/routes/identity-intro-cache.ts +7 -3
- package/src/runtime/routes/identity-routes.ts +262 -33
- package/src/runtime/routes/inbound-stages/transcribe-audio.test.ts +46 -39
- package/src/runtime/routes/inbound-stages/transcribe-audio.ts +15 -15
- package/src/runtime/routes/integrations/slack/__tests__/channel.test.ts +137 -0
- package/src/runtime/routes/integrations/slack/__tests__/share.test.ts +179 -0
- package/src/runtime/routes/integrations/slack/channel.ts +11 -3
- package/src/runtime/routes/integrations/slack/share.ts +45 -7
- package/src/runtime/routes/llm-context-normalization.ts +303 -0
- package/src/runtime/routes/log-export-routes.ts +42 -22
- package/src/runtime/routes/memory-item-routes.test.ts +3 -2
- package/src/runtime/routes/memory-item-routes.ts +1 -7
- package/src/runtime/routes/migration-routes.ts +122 -2
- package/src/runtime/routes/oauth-apps.ts +15 -17
- package/src/runtime/routes/oauth-providers.ts +4 -0
- package/src/runtime/routes/schedule-routes.ts +24 -11
- package/src/runtime/routes/settings-routes.ts +31 -102
- package/src/runtime/routes/skills-routes.ts +128 -9
- package/src/runtime/routes/stt-routes.ts +233 -0
- package/src/runtime/routes/subagents-routes.ts +14 -10
- package/src/runtime/routes/surface-action-routes.ts +41 -2
- package/src/runtime/routes/tts-routes.ts +108 -24
- package/src/runtime/routes/usage-routes.ts +38 -9
- package/src/runtime/routes/user-route-dispatcher.ts +50 -5
- package/src/runtime/routes/user-routes.ts +13 -1
- package/src/runtime/routes/work-items-routes.ts +8 -1
- package/src/runtime/routes/workspace-routes.test.ts +22 -0
- package/src/runtime/routes/workspace-routes.ts +8 -1
- package/src/runtime/routes/workspace-utils.ts +2 -0
- package/src/runtime/runtime-mode.ts +33 -0
- package/src/runtime/services/__tests__/analyze-conversation.test.ts +444 -0
- package/src/runtime/services/__tests__/analyze-deps-singleton.test.ts +67 -0
- package/src/runtime/services/__tests__/auto-analysis-prompt.test.ts +53 -0
- package/src/runtime/services/__tests__/manual-analysis-prompt.test.ts +41 -0
- package/src/runtime/services/analyze-conversation.ts +344 -0
- package/src/runtime/services/analyze-deps-singleton.ts +32 -0
- package/src/runtime/services/auto-analysis-prompt.ts +55 -0
- package/src/runtime/skill-route-registry.ts +49 -0
- package/src/runtime/slack-block-formatting.ts +437 -10
- package/src/schedule/scheduler.ts +57 -5
- package/src/security/ces-credential-client.ts +20 -0
- package/src/security/ces-rpc-credential-backend.ts +17 -0
- package/src/security/credential-backend.ts +5 -0
- package/src/security/oauth2.ts +68 -29
- package/src/security/secure-keys.ts +143 -27
- package/src/security/token-manager.ts +31 -10
- package/src/sequence/engine.ts +23 -0
- package/src/sequence/types.ts +1 -1
- package/src/skills/catalog-files.ts +554 -0
- package/src/skills/category-inference.ts +122 -0
- package/src/skills/clawhub-files.ts +213 -0
- package/src/skills/clawhub.ts +84 -23
- package/src/skills/skill-file-provider.ts +40 -0
- package/src/skills/skillssh-files.ts +395 -0
- package/src/skills/skillssh-registry.ts +4 -4
- package/src/stt/__tests__/daemon-batch-transcriber.test.ts +392 -0
- package/src/stt/__tests__/types.test.ts +89 -0
- package/src/stt/daemon-batch-transcriber.ts +195 -0
- package/src/stt/stt-stream-session.ts +499 -0
- package/src/stt/types.ts +330 -0
- package/src/stt/wav-encoder.test.ts +373 -0
- package/src/stt/wav-encoder.ts +175 -0
- package/src/subagent/manager.ts +169 -40
- package/src/subagent/types.ts +19 -0
- package/src/tools/apps/executors.ts +11 -2
- package/src/tools/browser/__tests__/auth-detector.test.ts +202 -108
- package/src/tools/browser/__tests__/browser-mode.test.ts +119 -0
- package/src/tools/browser/__tests__/browser-status.test.ts +123 -0
- package/src/tools/browser/auth-detector.ts +43 -12
- package/src/tools/browser/browser-execution.ts +1787 -342
- package/src/tools/browser/browser-manager.ts +81 -12
- package/src/tools/browser/browser-mode-constants.ts +12 -0
- package/src/tools/browser/browser-mode.ts +92 -0
- package/src/tools/browser/browser-status-constants.ts +33 -0
- package/src/tools/browser/cdp-client/__tests__/accessibility-snapshot.test.ts +318 -0
- package/src/tools/browser/cdp-client/__tests__/cdp-dom-helpers.test.ts +1175 -0
- package/src/tools/browser/cdp-client/__tests__/cdp-inspect-client.test.ts +1263 -0
- package/src/tools/browser/cdp-client/__tests__/extension-cdp-client.test.ts +359 -0
- package/src/tools/browser/cdp-client/__tests__/factory.test.ts +1993 -0
- package/src/tools/browser/cdp-client/__tests__/fixtures/ax-tree-nested-frames.json +64 -0
- package/src/tools/browser/cdp-client/__tests__/fixtures/ax-tree-simple.json +69 -0
- package/src/tools/browser/cdp-client/__tests__/local-cdp-client.test.ts +310 -0
- package/src/tools/browser/cdp-client/__tests__/types.test.ts +96 -0
- package/src/tools/browser/cdp-client/accessibility-snapshot.ts +387 -0
- package/src/tools/browser/cdp-client/cdp-dom-helpers.ts +695 -0
- package/src/tools/browser/cdp-client/cdp-inspect/__tests__/discovery.test.ts +1007 -0
- package/src/tools/browser/cdp-client/cdp-inspect/__tests__/ws-transport.test.ts +580 -0
- package/src/tools/browser/cdp-client/cdp-inspect/discovery.ts +744 -0
- package/src/tools/browser/cdp-client/cdp-inspect/ws-transport.ts +579 -0
- package/src/tools/browser/cdp-client/cdp-inspect-client.ts +868 -0
- package/src/tools/browser/cdp-client/errors.ts +49 -0
- package/src/tools/browser/cdp-client/extension-cdp-client.ts +148 -0
- package/src/tools/browser/cdp-client/factory.ts +914 -0
- package/src/tools/browser/cdp-client/index.ts +28 -0
- package/src/tools/browser/cdp-client/local-cdp-client.ts +187 -0
- package/src/tools/browser/cdp-client/types.ts +120 -0
- package/src/tools/credentials/vault.ts +35 -6
- package/src/tools/filesystem/edit.ts +1 -1
- package/src/tools/filesystem/list.ts +1 -1
- package/src/tools/filesystem/read.ts +1 -1
- package/src/tools/filesystem/write.ts +2 -1
- package/src/tools/host-filesystem/edit.ts +1 -1
- package/src/tools/host-filesystem/read.ts +12 -15
- package/src/tools/host-filesystem/write.ts +1 -1
- package/src/tools/host-terminal/host-shell.ts +21 -16
- package/src/tools/network/web-fetch.ts +5 -2
- package/src/tools/network/web-search.ts +5 -2
- package/src/tools/permission-checker.ts +77 -82
- package/src/tools/registry.ts +0 -2
- package/src/tools/secret-detection-handler.ts +34 -0
- package/src/tools/shared/filesystem/image-read.ts +61 -40
- package/src/tools/shared/shell-output.ts +3 -1
- package/src/tools/side-effects.ts +2 -0
- package/src/tools/skills/sandbox-runner.ts +3 -2
- package/src/tools/subagent/spawn.ts +47 -3
- package/src/tools/subagent/status.ts +2 -0
- package/src/tools/system/register.ts +2 -16
- package/src/tools/terminal/safe-env.ts +15 -0
- package/src/tools/terminal/shell.ts +36 -20
- package/src/tools/tool-approval-handler.ts +48 -2
- package/src/tools/tool-manifest.ts +21 -0
- package/src/tools/types.ts +19 -0
- package/src/tools/ui-surface/definitions.ts +6 -1
- package/src/tts/__tests__/provider-adapters.test.ts +834 -0
- package/src/tts/__tests__/provider-catalog-consistency.test.ts +196 -0
- package/src/tts/__tests__/provider-catalog.test.ts +183 -0
- package/src/tts/__tests__/provider-registry.test.ts +90 -0
- package/src/tts/provider-catalog.ts +201 -0
- package/src/tts/provider-registry.ts +73 -0
- package/src/tts/providers/deepgram-provider.ts +219 -0
- package/src/tts/providers/elevenlabs-provider.ts +211 -0
- package/src/tts/providers/fish-audio-provider.ts +183 -0
- package/src/tts/providers/index.ts +42 -0
- package/src/tts/providers/register-builtins.ts +130 -0
- package/src/tts/synthesize-text.ts +110 -0
- package/src/tts/tts-config-resolver.ts +78 -0
- package/src/tts/types.ts +153 -0
- package/src/types/onboarding-context.ts +7 -0
- package/src/util/abort-reasons.ts +58 -0
- package/src/util/device-id.ts +32 -16
- package/src/util/errors.ts +9 -1
- package/src/util/platform.ts +63 -24
- package/src/util/pricing.ts +66 -3
- package/src/util/spawn.ts +1 -1
- package/src/util/truncate.ts +4 -2
- package/src/util/unicode.ts +201 -0
- package/src/version.ts +19 -24
- package/src/watcher/engine.ts +23 -0
- package/src/watcher/watcher-store.ts +31 -0
- package/src/workspace/migrations/003-seed-device-id.ts +9 -3
- package/src/workspace/migrations/017-seed-persona-dirs.ts +68 -4
- package/src/workspace/migrations/029-seed-pkb.ts +1 -1
- package/src/workspace/migrations/031-drop-user-md.ts +317 -0
- package/src/workspace/migrations/031-llm-log-retention-zero-to-null.ts +73 -0
- package/src/workspace/migrations/032-tts-provider-unification.ts +227 -0
- package/src/workspace/migrations/033-stt-service-explicit-config.ts +122 -0
- package/src/workspace/migrations/034-remove-calls-voice-transcription-provider.ts +215 -0
- package/src/workspace/migrations/035-seed-slack-channel-persona.ts +50 -0
- package/src/workspace/migrations/036-update-pkb-index-bar.ts +37 -0
- package/src/workspace/migrations/037-create-meets-dir.ts +61 -0
- package/src/workspace/migrations/registry.ts +16 -0
- package/src/workspace/top-level-renderer.ts +31 -1
- package/src/workspace/turn-commit.ts +31 -0
- package/src/__tests__/chrome-cdp.test.ts +0 -419
- package/src/__tests__/email-cli.test.ts +0 -297
- package/src/__tests__/email-service-config-fallback.test.ts +0 -102
- package/src/__tests__/permission-mode-sse.test.ts +0 -418
- package/src/__tests__/permission-mode-store.test.ts +0 -277
- package/src/browser-extension-relay/protocol.ts +0 -63
- package/src/browser-extension-relay/server.ts +0 -203
- package/src/cli/commands/browser-relay.ts +0 -536
- package/src/config/schemas/sandbox.ts +0 -14
- package/src/email/guardrails.ts +0 -221
- package/src/email/provider.ts +0 -117
- package/src/email/providers/agentmail.ts +0 -361
- package/src/email/providers/index.ts +0 -65
- package/src/email/service.ts +0 -384
- package/src/email/types.ts +0 -126
- package/src/permissions/permission-mode-store.ts +0 -180
- package/src/prompts/templates/USER.md +0 -13
- package/src/providers/speech-to-text/types.ts +0 -17
- package/src/tools/browser/chrome-cdp.ts +0 -239
- package/src/tools/system/set-permission-mode.ts +0 -103
|
@@ -9,40 +9,103 @@ mock.module("../util/logger.js", () => ({
|
|
|
9
9
|
}),
|
|
10
10
|
}));
|
|
11
11
|
|
|
12
|
+
/**
|
|
13
|
+
* Fake CDP session used by every interaction tool that has been
|
|
14
|
+
* migrated to `CdpClient` (click, hover, type, press_key,
|
|
15
|
+
* select_option, scroll). Each `session.send(method, params)` call is
|
|
16
|
+
* recorded in `sendCalls` and routed to `sendHandler`, which tests
|
|
17
|
+
* configure per-case. The handler returns either a CDP response
|
|
18
|
+
* object or an `Error` to simulate transport failure. `detachCalls`
|
|
19
|
+
* counts `session.detach()` invocations so tests can assert that
|
|
20
|
+
* `CdpClient.dispose()` runs in the tool's `finally` block.
|
|
21
|
+
*
|
|
22
|
+
* The fake session is exposed via `mockPage.context().newCDPSession(
|
|
23
|
+
* page)` so the real `LocalCdpClient` drives it. Routing through the
|
|
24
|
+
* production client (instead of mocking the factory / cdp-client
|
|
25
|
+
* submodules) avoids polluting the global module cache that the CDP
|
|
26
|
+
* unit tests rely on.
|
|
27
|
+
*/
|
|
28
|
+
interface SendCall {
|
|
29
|
+
method: string;
|
|
30
|
+
params: Record<string, unknown> | undefined;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
let sendCalls: SendCall[];
|
|
34
|
+
let sendHandler: (
|
|
35
|
+
method: string,
|
|
36
|
+
params: Record<string, unknown> | undefined,
|
|
37
|
+
) => unknown;
|
|
38
|
+
let detachCalls: number;
|
|
39
|
+
|
|
40
|
+
function resetCdpMock() {
|
|
41
|
+
sendCalls = [];
|
|
42
|
+
detachCalls = 0;
|
|
43
|
+
sendHandler = () => ({});
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
const fakeCdpSession = {
|
|
47
|
+
send: async (method: string, params?: Record<string, unknown>) => {
|
|
48
|
+
sendCalls.push({ method, params });
|
|
49
|
+
const value = sendHandler(method, params);
|
|
50
|
+
if (value instanceof Error) throw value;
|
|
51
|
+
return value;
|
|
52
|
+
},
|
|
53
|
+
detach: async () => {
|
|
54
|
+
detachCalls += 1;
|
|
55
|
+
},
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* The mock page only needs to expose `context().newCDPSession()` so
|
|
60
|
+
* the real `LocalCdpClient` can obtain a CDP session. All interaction
|
|
61
|
+
* tools now route through CDP, so no Playwright `page.*` surface is
|
|
62
|
+
* required.
|
|
63
|
+
*/
|
|
12
64
|
let mockPage: {
|
|
13
|
-
click: ReturnType<typeof mock>;
|
|
14
|
-
fill: ReturnType<typeof mock>;
|
|
15
|
-
press: ReturnType<typeof mock>;
|
|
16
|
-
evaluate: ReturnType<typeof mock>;
|
|
17
|
-
title: ReturnType<typeof mock>;
|
|
18
|
-
url: ReturnType<typeof mock>;
|
|
19
|
-
goto: ReturnType<typeof mock>;
|
|
20
|
-
screenshot: ReturnType<typeof mock>;
|
|
21
|
-
selectOption: ReturnType<typeof mock>;
|
|
22
|
-
hover: ReturnType<typeof mock>;
|
|
23
65
|
close: () => Promise<void>;
|
|
24
66
|
isClosed: () => boolean;
|
|
25
|
-
|
|
26
|
-
|
|
67
|
+
context: () => {
|
|
68
|
+
newCDPSession: (page: unknown) => Promise<typeof fakeCdpSession>;
|
|
69
|
+
};
|
|
27
70
|
};
|
|
28
71
|
|
|
29
|
-
let
|
|
72
|
+
let snapshotBackendNodeMaps: Map<string, Map<string, number>>;
|
|
73
|
+
|
|
74
|
+
const preferredBackendKinds = new Map<string, string>();
|
|
30
75
|
|
|
31
76
|
mock.module("../tools/browser/browser-manager.js", () => {
|
|
32
|
-
|
|
77
|
+
snapshotBackendNodeMaps = new Map();
|
|
78
|
+
preferredBackendKinds.clear();
|
|
33
79
|
return {
|
|
34
80
|
browserManager: {
|
|
35
81
|
getOrCreateSessionPage: async () => mockPage,
|
|
36
82
|
closeSessionPage: async () => {},
|
|
37
83
|
closeAllPages: async () => {},
|
|
38
|
-
|
|
39
|
-
|
|
84
|
+
storeSnapshotBackendNodeMap: (
|
|
85
|
+
conversationId: string,
|
|
86
|
+
map: Map<string, number>,
|
|
87
|
+
) => {
|
|
88
|
+
snapshotBackendNodeMaps.set(conversationId, map);
|
|
40
89
|
},
|
|
41
|
-
|
|
42
|
-
|
|
90
|
+
resolveSnapshotBackendNodeId: (
|
|
91
|
+
conversationId: string,
|
|
92
|
+
elementId: string,
|
|
93
|
+
) => {
|
|
94
|
+
const map = snapshotBackendNodeMaps.get(conversationId);
|
|
43
95
|
if (!map) return null;
|
|
44
96
|
return map.get(elementId) ?? null;
|
|
45
97
|
},
|
|
98
|
+
clearSnapshotBackendNodeMap: (conversationId: string) => {
|
|
99
|
+
snapshotBackendNodeMaps.delete(conversationId);
|
|
100
|
+
},
|
|
101
|
+
getPreferredBackendKind: (conversationId: string) =>
|
|
102
|
+
preferredBackendKinds.get(conversationId) ?? null,
|
|
103
|
+
setPreferredBackendKind: (conversationId: string, kind: string) => {
|
|
104
|
+
preferredBackendKinds.set(conversationId, kind);
|
|
105
|
+
},
|
|
106
|
+
clearPreferredBackendKind: (conversationId: string) => {
|
|
107
|
+
preferredBackendKinds.delete(conversationId);
|
|
108
|
+
},
|
|
46
109
|
},
|
|
47
110
|
};
|
|
48
111
|
});
|
|
@@ -63,15 +126,14 @@ mock.module("../tools/browser/browser-screencast.js", () => ({
|
|
|
63
126
|
}));
|
|
64
127
|
|
|
65
128
|
import {
|
|
129
|
+
executeBrowserAttach,
|
|
66
130
|
executeBrowserClick,
|
|
67
131
|
executeBrowserClose,
|
|
68
|
-
|
|
132
|
+
executeBrowserDetach,
|
|
69
133
|
executeBrowserHover,
|
|
70
134
|
executeBrowserPressKey,
|
|
71
|
-
executeBrowserScreenshot,
|
|
72
135
|
executeBrowserScroll,
|
|
73
136
|
executeBrowserSelectOption,
|
|
74
|
-
executeBrowserSnapshot,
|
|
75
137
|
executeBrowserType,
|
|
76
138
|
} from "../tools/browser/browser-execution.js";
|
|
77
139
|
import type { ToolContext } from "../tools/types.js";
|
|
@@ -84,68 +146,234 @@ const ctx: ToolContext = {
|
|
|
84
146
|
|
|
85
147
|
function resetMockPage() {
|
|
86
148
|
mockPage = {
|
|
87
|
-
click: mock(async () => {}),
|
|
88
|
-
fill: mock(async () => {}),
|
|
89
|
-
press: mock(async () => {}),
|
|
90
|
-
evaluate: mock(async () => ""),
|
|
91
|
-
title: mock(async () => "Test Page"),
|
|
92
|
-
url: mock(() => "https://example.com/"),
|
|
93
|
-
goto: mock(async () => ({
|
|
94
|
-
status: () => 200,
|
|
95
|
-
url: () => "https://example.com/",
|
|
96
|
-
})),
|
|
97
|
-
screenshot: mock(async () => Buffer.from("fake-jpeg-data")),
|
|
98
|
-
selectOption: mock(async () => []),
|
|
99
|
-
hover: mock(async () => {}),
|
|
100
149
|
close: async () => {},
|
|
101
150
|
isClosed: () => false,
|
|
102
|
-
|
|
103
|
-
|
|
151
|
+
// `LocalCdpClient.ensureSession()` calls `page.context().newCDPSession(
|
|
152
|
+
// page)` to obtain a CDP session. Return the in-file `fakeCdpSession`
|
|
153
|
+
// so tests can assert on the exact CDP method sequence.
|
|
154
|
+
context: () => ({
|
|
155
|
+
newCDPSession: async (_page: unknown) => fakeCdpSession,
|
|
156
|
+
}),
|
|
157
|
+
};
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/**
|
|
161
|
+
* Default CDP send handler that answers the common plumbing calls
|
|
162
|
+
* used by the migrated tools (querySelectorBackendNodeId, DOM.focus,
|
|
163
|
+
* DOM.resolveNode, Runtime.callFunctionOn, Input.*, and
|
|
164
|
+
* Runtime.evaluate for viewport dimensions). Individual tests can
|
|
165
|
+
* override `sendHandler` to simulate failures or shape responses.
|
|
166
|
+
*/
|
|
167
|
+
function defaultCdpHandler(
|
|
168
|
+
method: string,
|
|
169
|
+
_params: Record<string, unknown> | undefined,
|
|
170
|
+
): unknown {
|
|
171
|
+
switch (method) {
|
|
172
|
+
case "DOM.getDocument":
|
|
173
|
+
return { root: { nodeId: 1 } };
|
|
174
|
+
case "DOM.querySelector":
|
|
175
|
+
return { nodeId: 42 };
|
|
176
|
+
case "DOM.describeNode":
|
|
177
|
+
return { node: { backendNodeId: 100 } };
|
|
178
|
+
case "DOM.resolveNode":
|
|
179
|
+
return { object: { objectId: "obj-1" } };
|
|
180
|
+
case "Runtime.evaluate":
|
|
181
|
+
return { result: { value: { w: 800, h: 600 } } };
|
|
182
|
+
case "Runtime.callFunctionOn":
|
|
183
|
+
// executeBrowserSelectOption invokes a function that returns
|
|
184
|
+
// a `matched` boolean — default to true so wrapper-contract
|
|
185
|
+
// tests don't need to know the inner select-option matching
|
|
186
|
+
// shape. Tests that exercise the no-match path override the
|
|
187
|
+
// handler explicitly.
|
|
188
|
+
return { result: { value: true } };
|
|
189
|
+
default:
|
|
190
|
+
return {};
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/**
|
|
195
|
+
* Install a CDP `sendHandler` tuned for the click + hover DOM →
|
|
196
|
+
* Input.dispatchMouseEvent chain (`DOM.getDocument`,
|
|
197
|
+
* `DOM.querySelector`, `DOM.describeNode`,
|
|
198
|
+
* `DOM.scrollIntoViewIfNeeded`, `DOM.getBoxModel`,
|
|
199
|
+
* `Input.dispatchMouseEvent`). Tests can override `throwFrom` to make
|
|
200
|
+
* one method reject, or override `backendNodeId` to control what
|
|
201
|
+
* `querySelectorBackendNodeId` resolves to.
|
|
202
|
+
*/
|
|
203
|
+
function installClickHoverCdpSend(
|
|
204
|
+
overrides: Partial<{
|
|
205
|
+
backendNodeId: number;
|
|
206
|
+
throwFrom: string;
|
|
207
|
+
}> = {},
|
|
208
|
+
) {
|
|
209
|
+
const backendNodeId = overrides.backendNodeId ?? 1234;
|
|
210
|
+
const throwFrom = overrides.throwFrom;
|
|
211
|
+
|
|
212
|
+
sendHandler = (method, _params) => {
|
|
213
|
+
if (throwFrom === method) {
|
|
214
|
+
return new Error("cdp boom");
|
|
215
|
+
}
|
|
216
|
+
switch (method) {
|
|
217
|
+
case "DOM.getDocument":
|
|
218
|
+
return { root: { nodeId: 1 } };
|
|
219
|
+
case "DOM.querySelector":
|
|
220
|
+
return { nodeId: 2 };
|
|
221
|
+
case "DOM.describeNode":
|
|
222
|
+
return { node: { backendNodeId } };
|
|
223
|
+
case "DOM.scrollIntoViewIfNeeded":
|
|
224
|
+
return {};
|
|
225
|
+
case "DOM.getBoxModel":
|
|
226
|
+
// Flat 8-number quad: (10,20) (30,20) (30,40) (10,40)
|
|
227
|
+
// → center (20, 30).
|
|
228
|
+
return { model: { content: [10, 20, 30, 20, 30, 40, 10, 40] } };
|
|
229
|
+
case "Input.dispatchMouseEvent":
|
|
230
|
+
return {};
|
|
231
|
+
case "Runtime.evaluate":
|
|
232
|
+
// cdpWaitForSelector (used by click/hover selector branches)
|
|
233
|
+
// polls Runtime.evaluate with the visible-state probe and
|
|
234
|
+
// expects { result: { value: boolean } }. Returning true on
|
|
235
|
+
// the first poll lets the test resolve immediately instead
|
|
236
|
+
// of timing out after ACTION_TIMEOUT_MS.
|
|
237
|
+
return { result: { value: true } };
|
|
238
|
+
default:
|
|
239
|
+
return {};
|
|
240
|
+
}
|
|
104
241
|
};
|
|
105
242
|
}
|
|
106
243
|
|
|
107
244
|
// ── browser_click ────────────────────────────────────────────────────
|
|
108
245
|
|
|
109
|
-
describe("executeBrowserClick", () => {
|
|
246
|
+
describe("executeBrowserClick (CDP)", () => {
|
|
110
247
|
beforeEach(() => {
|
|
111
248
|
resetMockPage();
|
|
112
|
-
|
|
249
|
+
resetCdpMock();
|
|
250
|
+
snapshotBackendNodeMaps.clear();
|
|
113
251
|
});
|
|
114
252
|
|
|
115
|
-
test("clicks by
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
);
|
|
120
|
-
const result = await executeBrowserClick({ element_id: "e1" }, ctx);
|
|
253
|
+
test("clicks by selector: runs full DOM → Input.dispatchMouseEvent chain", async () => {
|
|
254
|
+
installClickHoverCdpSend({ backendNodeId: 5555 });
|
|
255
|
+
const result = await executeBrowserClick({ selector: "#submit-btn" }, ctx);
|
|
256
|
+
|
|
121
257
|
expect(result.isError).toBe(false);
|
|
122
|
-
expect(result.content).toContain("Clicked element");
|
|
123
|
-
|
|
124
|
-
|
|
258
|
+
expect(result.content).toContain("Clicked element: #submit-btn");
|
|
259
|
+
|
|
260
|
+
// Expected CDP call sequence for the selector path. The leading
|
|
261
|
+
// Runtime.evaluate is the visible-state probe issued by
|
|
262
|
+
// cdpWaitForSelector before resolving the backend node — this
|
|
263
|
+
// matches Playwright's `page.click(selector, { timeout })`
|
|
264
|
+
// semantics and lets click work on async-hydrated pages.
|
|
265
|
+
const methods = sendCalls.map((c) => c.method);
|
|
266
|
+
expect(methods).toEqual([
|
|
267
|
+
"Runtime.evaluate",
|
|
268
|
+
"DOM.getDocument",
|
|
269
|
+
"DOM.querySelector",
|
|
270
|
+
"DOM.describeNode",
|
|
271
|
+
"DOM.scrollIntoViewIfNeeded",
|
|
272
|
+
"DOM.getBoxModel",
|
|
273
|
+
"Input.dispatchMouseEvent",
|
|
274
|
+
"Input.dispatchMouseEvent",
|
|
275
|
+
"Input.dispatchMouseEvent",
|
|
276
|
+
]);
|
|
277
|
+
|
|
278
|
+
// The leading Runtime.evaluate is the visible-state probe.
|
|
279
|
+
const visibleProbe = sendCalls.find(
|
|
280
|
+
(c) => c.method === "Runtime.evaluate",
|
|
281
|
+
)!;
|
|
282
|
+
expect(
|
|
283
|
+
(visibleProbe.params as { expression: string }).expression,
|
|
284
|
+
).toContain("getBoundingClientRect");
|
|
285
|
+
|
|
286
|
+
// Arguments threaded through correctly.
|
|
287
|
+
const qsCall = sendCalls.find((c) => c.method === "DOM.querySelector")!;
|
|
288
|
+
expect(qsCall.params).toMatchObject({ nodeId: 1, selector: "#submit-btn" });
|
|
289
|
+
const scrollCall = sendCalls.find(
|
|
290
|
+
(c) => c.method === "DOM.scrollIntoViewIfNeeded",
|
|
291
|
+
)!;
|
|
292
|
+
expect(scrollCall.params).toMatchObject({ backendNodeId: 5555 });
|
|
293
|
+
const boxCall = sendCalls.find((c) => c.method === "DOM.getBoxModel")!;
|
|
294
|
+
expect(boxCall.params).toMatchObject({ backendNodeId: 5555 });
|
|
295
|
+
|
|
296
|
+
// All three mouse events land on the quad midpoint (20, 30).
|
|
297
|
+
const mouseCalls = sendCalls.filter(
|
|
298
|
+
(c) => c.method === "Input.dispatchMouseEvent",
|
|
299
|
+
);
|
|
300
|
+
expect(mouseCalls).toHaveLength(3);
|
|
301
|
+
expect(mouseCalls[0]!.params).toMatchObject({
|
|
302
|
+
type: "mouseMoved",
|
|
303
|
+
x: 20,
|
|
304
|
+
y: 30,
|
|
305
|
+
button: "left",
|
|
306
|
+
clickCount: 1,
|
|
125
307
|
});
|
|
308
|
+
expect(mouseCalls[1]!.params).toMatchObject({
|
|
309
|
+
type: "mousePressed",
|
|
310
|
+
x: 20,
|
|
311
|
+
y: 30,
|
|
312
|
+
button: "left",
|
|
313
|
+
clickCount: 1,
|
|
314
|
+
});
|
|
315
|
+
expect(mouseCalls[2]!.params).toMatchObject({
|
|
316
|
+
type: "mouseReleased",
|
|
317
|
+
x: 20,
|
|
318
|
+
y: 30,
|
|
319
|
+
button: "left",
|
|
320
|
+
clickCount: 1,
|
|
321
|
+
});
|
|
322
|
+
|
|
323
|
+
// CdpClient disposed in finally → session.detach called.
|
|
324
|
+
await new Promise((resolve) => setTimeout(resolve, 0));
|
|
325
|
+
expect(detachCalls).toBe(1);
|
|
126
326
|
});
|
|
127
327
|
|
|
128
|
-
test("clicks by
|
|
129
|
-
|
|
328
|
+
test("clicks by element_id (backend path): skips DOM.querySelector", async () => {
|
|
329
|
+
snapshotBackendNodeMaps.set("test-conversation", new Map([["e1", 42]]));
|
|
330
|
+
installClickHoverCdpSend();
|
|
331
|
+
|
|
332
|
+
const result = await executeBrowserClick({ element_id: "e1" }, ctx);
|
|
333
|
+
|
|
130
334
|
expect(result.isError).toBe(false);
|
|
131
|
-
expect(
|
|
132
|
-
|
|
133
|
-
|
|
335
|
+
expect(result.content).toContain("Clicked element: eid=e1");
|
|
336
|
+
|
|
337
|
+
const methods = sendCalls.map((c) => c.method);
|
|
338
|
+
// Backend path jumps straight to scrollIntoViewIfNeeded — no
|
|
339
|
+
// DOM.getDocument / querySelector / describeNode round-trip.
|
|
340
|
+
expect(methods).not.toContain("DOM.getDocument");
|
|
341
|
+
expect(methods).not.toContain("DOM.querySelector");
|
|
342
|
+
expect(methods).not.toContain("DOM.describeNode");
|
|
343
|
+
expect(methods).toEqual([
|
|
344
|
+
"DOM.scrollIntoViewIfNeeded",
|
|
345
|
+
"DOM.getBoxModel",
|
|
346
|
+
"Input.dispatchMouseEvent",
|
|
347
|
+
"Input.dispatchMouseEvent",
|
|
348
|
+
"Input.dispatchMouseEvent",
|
|
349
|
+
]);
|
|
350
|
+
|
|
351
|
+
// Backend node id threaded directly from the snapshot map.
|
|
352
|
+
const scrollCall = sendCalls.find(
|
|
353
|
+
(c) => c.method === "DOM.scrollIntoViewIfNeeded",
|
|
354
|
+
)!;
|
|
355
|
+
expect(scrollCall.params).toMatchObject({ backendNodeId: 42 });
|
|
356
|
+
const boxCall = sendCalls.find((c) => c.method === "DOM.getBoxModel")!;
|
|
357
|
+
expect(boxCall.params).toMatchObject({ backendNodeId: 42 });
|
|
358
|
+
|
|
359
|
+
await new Promise((resolve) => setTimeout(resolve, 0));
|
|
360
|
+
expect(detachCalls).toBe(1);
|
|
134
361
|
});
|
|
135
362
|
|
|
136
|
-
test("prefers element_id over selector", async () => {
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
);
|
|
363
|
+
test("prefers element_id over selector when both provided", async () => {
|
|
364
|
+
snapshotBackendNodeMaps.set("test-conversation", new Map([["e1", 77]]));
|
|
365
|
+
installClickHoverCdpSend();
|
|
366
|
+
|
|
141
367
|
const result = await executeBrowserClick(
|
|
142
|
-
{ element_id: "e1", selector: "#
|
|
368
|
+
{ element_id: "e1", selector: "#ignored" },
|
|
143
369
|
ctx,
|
|
144
370
|
);
|
|
145
371
|
expect(result.isError).toBe(false);
|
|
146
|
-
expect(
|
|
147
|
-
|
|
148
|
-
|
|
372
|
+
expect(result.content).toContain("eid=e1");
|
|
373
|
+
|
|
374
|
+
// DOM.querySelector must NOT have been called (selector ignored).
|
|
375
|
+
const methods = sendCalls.map((c) => c.method);
|
|
376
|
+
expect(methods).not.toContain("DOM.querySelector");
|
|
149
377
|
});
|
|
150
378
|
|
|
151
379
|
test("errors when neither element_id nor selector provided", async () => {
|
|
@@ -154,29 +382,89 @@ describe("executeBrowserClick", () => {
|
|
|
154
382
|
expect(result.content).toContain(
|
|
155
383
|
"Either element_id or selector is required",
|
|
156
384
|
);
|
|
385
|
+
// No CDP session should have been opened at all.
|
|
386
|
+
expect(sendCalls).toHaveLength(0);
|
|
387
|
+
expect(detachCalls).toBe(0);
|
|
157
388
|
});
|
|
158
389
|
|
|
159
390
|
test("errors when element_id not found in snapshot map", async () => {
|
|
391
|
+
installClickHoverCdpSend();
|
|
160
392
|
const result = await executeBrowserClick({ element_id: "e99" }, ctx);
|
|
161
393
|
expect(result.isError).toBe(true);
|
|
162
394
|
expect(result.content).toContain('element_id "e99" not found');
|
|
163
395
|
expect(result.content).toContain("browser_snapshot");
|
|
396
|
+
// Resolution failed before acquiring a CdpClient.
|
|
397
|
+
expect(sendCalls).toHaveLength(0);
|
|
164
398
|
});
|
|
165
399
|
|
|
166
|
-
test("errors when snapshot map is missing for session", async () => {
|
|
400
|
+
test("errors when snapshot backend-node map is missing for session", async () => {
|
|
401
|
+
installClickHoverCdpSend();
|
|
167
402
|
const result = await executeBrowserClick({ element_id: "e1" }, ctx);
|
|
168
403
|
expect(result.isError).toBe(true);
|
|
169
404
|
expect(result.content).toContain("not found");
|
|
405
|
+
expect(sendCalls).toHaveLength(0);
|
|
170
406
|
});
|
|
171
407
|
|
|
172
|
-
test("
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
});
|
|
176
|
-
|
|
408
|
+
test("returns error + still disposes CdpClient when cdp.send throws", async () => {
|
|
409
|
+
installClickHoverCdpSend({ throwFrom: "Input.dispatchMouseEvent" });
|
|
410
|
+
|
|
411
|
+
const result = await executeBrowserClick({ selector: "#submit-btn" }, ctx);
|
|
412
|
+
|
|
177
413
|
expect(result.isError).toBe(true);
|
|
178
414
|
expect(result.content).toContain("Click failed");
|
|
179
|
-
expect(result.content).toContain("
|
|
415
|
+
expect(result.content).toContain("cdp boom");
|
|
416
|
+
|
|
417
|
+
// finally { cdp.dispose() } must still fire → detach called.
|
|
418
|
+
await new Promise((resolve) => setTimeout(resolve, 0));
|
|
419
|
+
expect(detachCalls).toBe(1);
|
|
420
|
+
});
|
|
421
|
+
|
|
422
|
+
test("waits for selector that initially doesn't exist but becomes visible", async () => {
|
|
423
|
+
// Simulates a hydrating page: the visible-state probe returns
|
|
424
|
+
// false for the first 2 polls, then true on the 3rd. The click
|
|
425
|
+
// tool must wait through these polls (instead of failing
|
|
426
|
+
// immediately) and then complete the click as normal.
|
|
427
|
+
let visibleProbeCount = 0;
|
|
428
|
+
sendHandler = (method, _params) => {
|
|
429
|
+
switch (method) {
|
|
430
|
+
case "Runtime.evaluate":
|
|
431
|
+
visibleProbeCount++;
|
|
432
|
+
return { result: { value: visibleProbeCount >= 3 } };
|
|
433
|
+
case "DOM.getDocument":
|
|
434
|
+
return { root: { nodeId: 1 } };
|
|
435
|
+
case "DOM.querySelector":
|
|
436
|
+
return { nodeId: 2 };
|
|
437
|
+
case "DOM.describeNode":
|
|
438
|
+
return { node: { backendNodeId: 8888 } };
|
|
439
|
+
case "DOM.scrollIntoViewIfNeeded":
|
|
440
|
+
return {};
|
|
441
|
+
case "DOM.getBoxModel":
|
|
442
|
+
return { model: { content: [10, 20, 30, 20, 30, 40, 10, 40] } };
|
|
443
|
+
case "Input.dispatchMouseEvent":
|
|
444
|
+
return {};
|
|
445
|
+
default:
|
|
446
|
+
return {};
|
|
447
|
+
}
|
|
448
|
+
};
|
|
449
|
+
|
|
450
|
+
const result = await executeBrowserClick({ selector: "#hydrated" }, ctx);
|
|
451
|
+
|
|
452
|
+
expect(result.isError).toBe(false);
|
|
453
|
+
expect(result.content).toContain("Clicked element: #hydrated");
|
|
454
|
+
// The visible-state probe was polled at least 3 times before
|
|
455
|
+
// succeeding, then the rest of the click pipeline ran exactly
|
|
456
|
+
// once.
|
|
457
|
+
expect(visibleProbeCount).toBeGreaterThanOrEqual(3);
|
|
458
|
+
const mouseCalls = sendCalls.filter(
|
|
459
|
+
(c) => c.method === "Input.dispatchMouseEvent",
|
|
460
|
+
);
|
|
461
|
+
expect(mouseCalls).toHaveLength(3);
|
|
462
|
+
// querySelectorBackendNodeId only ran once at the end (after the
|
|
463
|
+
// probe returned true) — not on every polling iteration.
|
|
464
|
+
const describeCalls = sendCalls.filter(
|
|
465
|
+
(c) => c.method === "DOM.describeNode",
|
|
466
|
+
);
|
|
467
|
+
expect(describeCalls).toHaveLength(1);
|
|
180
468
|
});
|
|
181
469
|
});
|
|
182
470
|
|
|
@@ -185,51 +473,68 @@ describe("executeBrowserClick", () => {
|
|
|
185
473
|
describe("executeBrowserType", () => {
|
|
186
474
|
beforeEach(() => {
|
|
187
475
|
resetMockPage();
|
|
188
|
-
|
|
476
|
+
resetCdpMock();
|
|
477
|
+
snapshotBackendNodeMaps.clear();
|
|
478
|
+
sendHandler = defaultCdpHandler;
|
|
189
479
|
});
|
|
190
480
|
|
|
191
481
|
test("types with element_id and default clear_first=true", async () => {
|
|
192
|
-
|
|
193
|
-
"test-conversation",
|
|
194
|
-
new Map([["e3", '[data-vellum-eid="e3"]']]),
|
|
195
|
-
);
|
|
482
|
+
snapshotBackendNodeMaps.set("test-conversation", new Map([["e3", 555]]));
|
|
196
483
|
const result = await executeBrowserType(
|
|
197
484
|
{ element_id: "e3", text: "hello" },
|
|
198
485
|
ctx,
|
|
199
486
|
);
|
|
200
487
|
expect(result.isError).toBe(false);
|
|
201
|
-
expect(result.content).toContain(
|
|
488
|
+
expect(result.content).toContain('Typed into element: element_id "e3"');
|
|
202
489
|
expect(result.content).toContain("cleared existing content");
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
);
|
|
490
|
+
|
|
491
|
+
// Expected CDP sequence when resolving by backendNodeId + clearFirst:
|
|
492
|
+
// DOM.focus → DOM.resolveNode → Runtime.callFunctionOn (clear) →
|
|
493
|
+
// DOM.focus → Input.insertText
|
|
494
|
+
const methods = sendCalls.map((c) => c.method);
|
|
495
|
+
expect(methods).toEqual([
|
|
496
|
+
"DOM.focus",
|
|
497
|
+
"DOM.resolveNode",
|
|
498
|
+
"Runtime.callFunctionOn",
|
|
499
|
+
"DOM.focus",
|
|
500
|
+
"Input.insertText",
|
|
501
|
+
]);
|
|
502
|
+
const focusCall = sendCalls[0]!;
|
|
503
|
+
expect(focusCall.params).toEqual({ backendNodeId: 555 });
|
|
504
|
+
const insertCall = sendCalls[sendCalls.length - 1]!;
|
|
505
|
+
expect(insertCall.params).toEqual({ text: "hello" });
|
|
208
506
|
});
|
|
209
507
|
|
|
210
|
-
test("types with raw selector", async () => {
|
|
508
|
+
test("types with raw selector (resolves via DOM.querySelector)", async () => {
|
|
211
509
|
const result = await executeBrowserType(
|
|
212
510
|
{ selector: 'input[name="email"]', text: "test" },
|
|
213
511
|
ctx,
|
|
214
512
|
);
|
|
215
513
|
expect(result.isError).toBe(false);
|
|
216
|
-
expect(
|
|
217
|
-
|
|
218
|
-
|
|
514
|
+
expect(result.content).toContain('Typed into element: input[name="email"]');
|
|
515
|
+
// Raw-selector path must resolve the backendNodeId first.
|
|
516
|
+
const methods = sendCalls.map((c) => c.method);
|
|
517
|
+
expect(methods[0]).toBe("DOM.getDocument");
|
|
518
|
+
expect(methods[1]).toBe("DOM.querySelector");
|
|
519
|
+
expect(methods[2]).toBe("DOM.describeNode");
|
|
520
|
+
expect(methods).toContain("Input.insertText");
|
|
219
521
|
});
|
|
220
522
|
|
|
221
523
|
test("appends text when clear_first=false", async () => {
|
|
222
|
-
mockPage.evaluate = mock(async () => "existing");
|
|
223
524
|
const result = await executeBrowserType(
|
|
224
525
|
{ selector: "#input", text: " more", clear_first: false },
|
|
225
526
|
ctx,
|
|
226
527
|
);
|
|
227
528
|
expect(result.isError).toBe(false);
|
|
228
|
-
expect(mockPage.evaluate).toHaveBeenCalled();
|
|
229
|
-
expect(mockPage.fill).toHaveBeenCalledWith("#input", "existing more", {
|
|
230
|
-
timeout: 10000,
|
|
231
|
-
});
|
|
232
529
|
expect(result.content).not.toContain("cleared");
|
|
530
|
+
// clear_first=false skips DOM.resolveNode + Runtime.callFunctionOn
|
|
531
|
+
// and the re-focus call, so we should see focus + insertText only.
|
|
532
|
+
const methods = sendCalls.map((c) => c.method);
|
|
533
|
+
expect(methods).not.toContain("DOM.resolveNode");
|
|
534
|
+
expect(methods).not.toContain("Runtime.callFunctionOn");
|
|
535
|
+
const focusCount = methods.filter((m) => m === "DOM.focus").length;
|
|
536
|
+
expect(focusCount).toBe(1);
|
|
537
|
+
expect(methods).toContain("Input.insertText");
|
|
233
538
|
});
|
|
234
539
|
|
|
235
540
|
test("presses Enter after typing when press_enter=true", async () => {
|
|
@@ -239,16 +544,32 @@ describe("executeBrowserType", () => {
|
|
|
239
544
|
);
|
|
240
545
|
expect(result.isError).toBe(false);
|
|
241
546
|
expect(result.content).toContain("pressed Enter");
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
547
|
+
const methods = sendCalls.map((c) => c.method);
|
|
548
|
+
// Input.insertText must come before the Enter keyDown/char/keyUp.
|
|
549
|
+
const insertIdx = methods.indexOf("Input.insertText");
|
|
550
|
+
const keyDownIdx = methods.findIndex(
|
|
551
|
+
(m, i) =>
|
|
552
|
+
m === "Input.dispatchKeyEvent" &&
|
|
553
|
+
(sendCalls[i]!.params as { type: string }).type === "keyDown",
|
|
554
|
+
);
|
|
555
|
+
expect(insertIdx).toBeGreaterThanOrEqual(0);
|
|
556
|
+
expect(keyDownIdx).toBeGreaterThan(insertIdx);
|
|
557
|
+
// Enter is text-producing → keyDown + char + keyUp.
|
|
558
|
+
const keyEvents = sendCalls.filter(
|
|
559
|
+
(c) => c.method === "Input.dispatchKeyEvent",
|
|
560
|
+
);
|
|
561
|
+
expect(keyEvents).toHaveLength(3);
|
|
562
|
+
expect((keyEvents[0]!.params as { key: string }).key).toBe("Enter");
|
|
563
|
+
expect((keyEvents[0]!.params as { type: string }).type).toBe("keyDown");
|
|
564
|
+
expect((keyEvents[1]!.params as { type: string }).type).toBe("char");
|
|
565
|
+
expect((keyEvents[2]!.params as { type: string }).type).toBe("keyUp");
|
|
246
566
|
});
|
|
247
567
|
|
|
248
568
|
test("errors when text is missing", async () => {
|
|
249
569
|
const result = await executeBrowserType({ selector: "#input" }, ctx);
|
|
250
570
|
expect(result.isError).toBe(true);
|
|
251
571
|
expect(result.content).toContain("text is required");
|
|
572
|
+
expect(sendCalls).toHaveLength(0);
|
|
252
573
|
});
|
|
253
574
|
|
|
254
575
|
test("errors when text is empty string", async () => {
|
|
@@ -258,6 +579,7 @@ describe("executeBrowserType", () => {
|
|
|
258
579
|
);
|
|
259
580
|
expect(result.isError).toBe(true);
|
|
260
581
|
expect(result.content).toContain("text is required");
|
|
582
|
+
expect(sendCalls).toHaveLength(0);
|
|
261
583
|
});
|
|
262
584
|
|
|
263
585
|
test("errors when neither element_id nor selector provided", async () => {
|
|
@@ -266,6 +588,7 @@ describe("executeBrowserType", () => {
|
|
|
266
588
|
expect(result.content).toContain(
|
|
267
589
|
"Either element_id or selector is required",
|
|
268
590
|
);
|
|
591
|
+
expect(sendCalls).toHaveLength(0);
|
|
269
592
|
});
|
|
270
593
|
|
|
271
594
|
test("errors when element_id not found", async () => {
|
|
@@ -275,143 +598,33 @@ describe("executeBrowserType", () => {
|
|
|
275
598
|
);
|
|
276
599
|
expect(result.isError).toBe(true);
|
|
277
600
|
expect(result.content).toContain('element_id "e99" not found');
|
|
601
|
+
expect(sendCalls).toHaveLength(0);
|
|
278
602
|
});
|
|
279
603
|
|
|
280
|
-
test("
|
|
281
|
-
|
|
282
|
-
throw new Error("Element is not an input");
|
|
283
|
-
});
|
|
604
|
+
test("surfaces CDP failure as a type error", async () => {
|
|
605
|
+
sendHandler = () => new Error("focus failed");
|
|
284
606
|
const result = await executeBrowserType(
|
|
285
607
|
{ selector: "#div", text: "hello" },
|
|
286
608
|
ctx,
|
|
287
609
|
);
|
|
288
610
|
expect(result.isError).toBe(true);
|
|
289
611
|
expect(result.content).toContain("Type failed");
|
|
290
|
-
expect(result.content).toContain("
|
|
291
|
-
});
|
|
292
|
-
});
|
|
293
|
-
|
|
294
|
-
// ── browser_snapshot ──────────────────────────────────────────────────
|
|
295
|
-
|
|
296
|
-
describe("executeBrowserSnapshot", () => {
|
|
297
|
-
beforeEach(() => {
|
|
298
|
-
resetMockPage();
|
|
299
|
-
snapshotMaps.clear();
|
|
300
|
-
});
|
|
301
|
-
|
|
302
|
-
test("returns element list with eid format", async () => {
|
|
303
|
-
const sampleElements = [
|
|
304
|
-
{ eid: "e1", tag: "a", attrs: { href: "/about" }, text: "About Us" },
|
|
305
|
-
{ eid: "e2", tag: "button", attrs: { type: "submit" }, text: "Submit" },
|
|
306
|
-
{
|
|
307
|
-
eid: "e3",
|
|
308
|
-
tag: "input",
|
|
309
|
-
attrs: { type: "text", name: "email", placeholder: "Enter email" },
|
|
310
|
-
text: "",
|
|
311
|
-
},
|
|
312
|
-
];
|
|
313
|
-
mockPage.evaluate = mock(async () => sampleElements);
|
|
314
|
-
const result = await executeBrowserSnapshot({}, ctx);
|
|
315
|
-
expect(result.isError).toBe(false);
|
|
316
|
-
expect(result.content).toContain("[e1]");
|
|
317
|
-
expect(result.content).toContain("[e2]");
|
|
318
|
-
expect(result.content).toContain("[e3]");
|
|
319
|
-
expect(result.content).toContain("<a");
|
|
320
|
-
expect(result.content).toContain("<button");
|
|
321
|
-
expect(result.content).toContain("<input");
|
|
322
|
-
expect(result.content).toContain("3 interactive elements found");
|
|
323
|
-
});
|
|
324
|
-
|
|
325
|
-
test("stores snapshot map for later element resolution", async () => {
|
|
326
|
-
const sampleElements = [
|
|
327
|
-
{ eid: "e1", tag: "a", attrs: { href: "/" }, text: "Home" },
|
|
328
|
-
];
|
|
329
|
-
mockPage.evaluate = mock(async () => sampleElements);
|
|
330
|
-
await executeBrowserSnapshot({}, ctx);
|
|
331
|
-
const map = snapshotMaps.get("test-conversation");
|
|
332
|
-
expect(map).toBeDefined();
|
|
333
|
-
expect(map!.get("e1")).toBe('[data-vellum-eid="e1"]');
|
|
334
|
-
});
|
|
335
|
-
|
|
336
|
-
test("reports no interactive elements when page is empty", async () => {
|
|
337
|
-
mockPage.evaluate = mock(async () => []);
|
|
338
|
-
const result = await executeBrowserSnapshot({}, ctx);
|
|
339
|
-
expect(result.isError).toBe(false);
|
|
340
|
-
expect(result.content).toContain("no interactive elements found");
|
|
341
|
-
});
|
|
342
|
-
|
|
343
|
-
test("includes page URL and title", async () => {
|
|
344
|
-
mockPage.evaluate = mock(async () => []);
|
|
345
|
-
const result = await executeBrowserSnapshot({}, ctx);
|
|
346
|
-
expect(result.content).toContain("URL: https://example.com/");
|
|
347
|
-
expect(result.content).toContain("Title: Test Page");
|
|
348
|
-
});
|
|
349
|
-
|
|
350
|
-
test("handles snapshot error from page", async () => {
|
|
351
|
-
mockPage.evaluate = mock(async () => {
|
|
352
|
-
throw new Error("Page crashed");
|
|
353
|
-
});
|
|
354
|
-
const result = await executeBrowserSnapshot({}, ctx);
|
|
355
|
-
expect(result.isError).toBe(true);
|
|
356
|
-
expect(result.content).toContain("Snapshot failed");
|
|
357
|
-
expect(result.content).toContain("Page crashed");
|
|
612
|
+
expect(result.content).toContain("focus failed");
|
|
358
613
|
});
|
|
359
614
|
});
|
|
360
615
|
|
|
361
|
-
//
|
|
362
|
-
|
|
363
|
-
describe("executeBrowserScreenshot", () => {
|
|
364
|
-
beforeEach(() => {
|
|
365
|
-
resetMockPage();
|
|
366
|
-
});
|
|
616
|
+
// NOTE: executeBrowserSnapshot tests live in
|
|
617
|
+
// `headless-browser-snapshot.test.ts`.
|
|
367
618
|
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
mockPage.screenshot = mock(async () => fakeBuffer);
|
|
371
|
-
const result = await executeBrowserScreenshot({}, ctx);
|
|
372
|
-
expect(result.isError).toBe(false);
|
|
373
|
-
expect(result.content).toContain("Screenshot captured");
|
|
374
|
-
expect(result.content).toContain(`${fakeBuffer.length} bytes`);
|
|
375
|
-
expect(result.content).toContain("viewport");
|
|
376
|
-
expect(result.contentBlocks).toBeDefined();
|
|
377
|
-
expect(result.contentBlocks!.length).toBe(1);
|
|
378
|
-
const imageBlock = result.contentBlocks![0] as {
|
|
379
|
-
type: string;
|
|
380
|
-
source: { type: string; media_type: string; data: string };
|
|
381
|
-
};
|
|
382
|
-
expect(imageBlock.type).toBe("image");
|
|
383
|
-
expect(imageBlock.source.media_type).toBe("image/jpeg");
|
|
384
|
-
expect(imageBlock.source.data).toBe(fakeBuffer.toString("base64"));
|
|
385
|
-
});
|
|
386
|
-
|
|
387
|
-
test("supports full_page mode", async () => {
|
|
388
|
-
mockPage.screenshot = mock(async () => Buffer.from("full"));
|
|
389
|
-
const result = await executeBrowserScreenshot({ full_page: true }, ctx);
|
|
390
|
-
expect(result.isError).toBe(false);
|
|
391
|
-
expect(result.content).toContain("full page");
|
|
392
|
-
expect(mockPage.screenshot).toHaveBeenCalledWith({
|
|
393
|
-
type: "jpeg",
|
|
394
|
-
quality: 80,
|
|
395
|
-
fullPage: true,
|
|
396
|
-
});
|
|
397
|
-
});
|
|
398
|
-
|
|
399
|
-
test("handles screenshot error from page", async () => {
|
|
400
|
-
mockPage.screenshot = mock(async () => {
|
|
401
|
-
throw new Error("Render failed");
|
|
402
|
-
});
|
|
403
|
-
const result = await executeBrowserScreenshot({}, ctx);
|
|
404
|
-
expect(result.isError).toBe(true);
|
|
405
|
-
expect(result.content).toContain("Screenshot failed");
|
|
406
|
-
expect(result.content).toContain("Render failed");
|
|
407
|
-
});
|
|
408
|
-
});
|
|
619
|
+
// browser_screenshot tests live in headless-browser-read-tools.test.ts
|
|
620
|
+
// (alongside browser_extract / browser_wait_for).
|
|
409
621
|
|
|
410
622
|
// ── browser_close ────────────────────────────────────────────────────
|
|
411
623
|
|
|
412
624
|
describe("executeBrowserClose", () => {
|
|
413
625
|
beforeEach(() => {
|
|
414
626
|
resetMockPage();
|
|
627
|
+
resetCdpMock();
|
|
415
628
|
});
|
|
416
629
|
|
|
417
630
|
test("closes session page", async () => {
|
|
@@ -429,92 +642,93 @@ describe("executeBrowserClose", () => {
|
|
|
429
642
|
});
|
|
430
643
|
});
|
|
431
644
|
|
|
432
|
-
// ──
|
|
645
|
+
// ── browser_attach ──────────────────────────────────────────────────
|
|
433
646
|
|
|
434
|
-
describe("
|
|
647
|
+
describe("executeBrowserAttach", () => {
|
|
435
648
|
beforeEach(() => {
|
|
436
649
|
resetMockPage();
|
|
650
|
+
resetCdpMock();
|
|
437
651
|
});
|
|
438
652
|
|
|
439
|
-
test("
|
|
440
|
-
|
|
441
|
-
async () => "Hello, this is the page text content.",
|
|
442
|
-
);
|
|
443
|
-
const result = await executeBrowserExtract({}, ctx);
|
|
444
|
-
expect(result.isError).toBe(false);
|
|
445
|
-
expect(result.content).toContain("URL: https://example.com/");
|
|
446
|
-
expect(result.content).toContain("Title: Test Page");
|
|
447
|
-
expect(result.content).toContain("Hello, this is the page text content.");
|
|
448
|
-
});
|
|
449
|
-
|
|
450
|
-
test("includes links when include_links=true", async () => {
|
|
451
|
-
// First call returns text content, second returns link list
|
|
452
|
-
let callCount = 0;
|
|
453
|
-
mockPage.evaluate = mock(async () => {
|
|
454
|
-
callCount++;
|
|
455
|
-
if (callCount === 1) return "Some text";
|
|
456
|
-
return [
|
|
457
|
-
{ text: "Example Link", href: "https://example.com/link1" },
|
|
458
|
-
{ text: "Another", href: "https://example.com/link2" },
|
|
459
|
-
];
|
|
460
|
-
});
|
|
461
|
-
const result = await executeBrowserExtract({ include_links: true }, ctx);
|
|
653
|
+
test("returns success on non-extension (local) backend", async () => {
|
|
654
|
+
const result = await executeBrowserAttach({}, ctx);
|
|
462
655
|
expect(result.isError).toBe(false);
|
|
463
|
-
expect(result.content).toContain("
|
|
464
|
-
expect(result.content).toContain(
|
|
465
|
-
"[Example Link](https://example.com/link1)",
|
|
466
|
-
);
|
|
467
|
-
expect(result.content).toContain("[Another](https://example.com/link2)");
|
|
656
|
+
expect(result.content).toContain("Browser session ready");
|
|
468
657
|
});
|
|
658
|
+
});
|
|
469
659
|
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
660
|
+
// ── browser_detach ──────────────────────────────────────────────────
|
|
661
|
+
|
|
662
|
+
describe("executeBrowserDetach", () => {
|
|
663
|
+
beforeEach(() => {
|
|
664
|
+
resetMockPage();
|
|
665
|
+
resetCdpMock();
|
|
475
666
|
});
|
|
476
667
|
|
|
477
|
-
test("
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
const result = await executeBrowserExtract({}, ctx);
|
|
482
|
-
expect(result.isError).toBe(true);
|
|
483
|
-
expect(result.content).toContain("Extract failed");
|
|
484
|
-
expect(result.content).toContain("Page not loaded");
|
|
668
|
+
test("clears snapshot state and returns success on non-extension backend", async () => {
|
|
669
|
+
const result = await executeBrowserDetach({}, ctx);
|
|
670
|
+
expect(result.isError).toBe(false);
|
|
671
|
+
expect(result.content).toContain("Browser debugger detached");
|
|
485
672
|
});
|
|
486
673
|
});
|
|
487
674
|
|
|
675
|
+
// browser_extract tests live in headless-browser-read-tools.test.ts
|
|
676
|
+
// because it drives CDP via getCdpClient() rather than the
|
|
677
|
+
// Playwright page mock this file uses.
|
|
678
|
+
|
|
488
679
|
// ── browser_press_key ────────────────────────────────────────────────
|
|
489
680
|
|
|
490
681
|
describe("executeBrowserPressKey", () => {
|
|
491
682
|
beforeEach(() => {
|
|
492
683
|
resetMockPage();
|
|
493
|
-
|
|
684
|
+
resetCdpMock();
|
|
685
|
+
snapshotBackendNodeMaps.clear();
|
|
686
|
+
sendHandler = defaultCdpHandler;
|
|
494
687
|
});
|
|
495
688
|
|
|
496
|
-
test("presses key on
|
|
689
|
+
test("presses key on focused element when no target", async () => {
|
|
497
690
|
const result = await executeBrowserPressKey({ key: "Enter" }, ctx);
|
|
498
691
|
expect(result.isError).toBe(false);
|
|
499
692
|
expect(result.content).toContain('Pressed "Enter"');
|
|
500
|
-
|
|
693
|
+
// No target => no DOM.focus, no selector resolution. Enter is a
|
|
694
|
+
// text-producing key (text "\r") so dispatchKeyPress emits
|
|
695
|
+
// keyDown + char + keyUp.
|
|
696
|
+
const methods = sendCalls.map((c) => c.method);
|
|
697
|
+
expect(methods).toEqual([
|
|
698
|
+
"Input.dispatchKeyEvent",
|
|
699
|
+
"Input.dispatchKeyEvent",
|
|
700
|
+
"Input.dispatchKeyEvent",
|
|
701
|
+
]);
|
|
702
|
+
const keyDown = sendCalls[0]!.params as Record<string, unknown>;
|
|
703
|
+
const charEvt = sendCalls[1]!.params as Record<string, unknown>;
|
|
704
|
+
const keyUp = sendCalls[2]!.params as Record<string, unknown>;
|
|
705
|
+
expect(keyDown.type).toBe("keyDown");
|
|
706
|
+
expect(keyDown.key).toBe("Enter");
|
|
707
|
+
expect(keyDown.windowsVirtualKeyCode).toBe(13);
|
|
708
|
+
expect(charEvt.type).toBe("char");
|
|
709
|
+
expect(keyUp.type).toBe("keyUp");
|
|
710
|
+
expect(keyUp.key).toBe("Enter");
|
|
501
711
|
});
|
|
502
712
|
|
|
503
713
|
test("presses key on targeted element via element_id", async () => {
|
|
504
|
-
|
|
505
|
-
"test-conversation",
|
|
506
|
-
new Map([["e5", '[data-vellum-eid="e5"]']]),
|
|
507
|
-
);
|
|
714
|
+
snapshotBackendNodeMaps.set("test-conversation", new Map([["e5", 555]]));
|
|
508
715
|
const result = await executeBrowserPressKey(
|
|
509
716
|
{ key: "Tab", element_id: "e5" },
|
|
510
717
|
ctx,
|
|
511
718
|
);
|
|
512
719
|
expect(result.isError).toBe(false);
|
|
513
720
|
expect(result.content).toContain('Pressed "Tab" on element');
|
|
514
|
-
expect(
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
);
|
|
721
|
+
expect(result.content).toContain('element_id "e5"');
|
|
722
|
+
// Backend-resolved path: focus → dispatchKeyEvent × 3 (Tab is
|
|
723
|
+
// text-producing so we also dispatch a char event).
|
|
724
|
+
const methods = sendCalls.map((c) => c.method);
|
|
725
|
+
expect(methods).toEqual([
|
|
726
|
+
"DOM.focus",
|
|
727
|
+
"Input.dispatchKeyEvent",
|
|
728
|
+
"Input.dispatchKeyEvent",
|
|
729
|
+
"Input.dispatchKeyEvent",
|
|
730
|
+
]);
|
|
731
|
+
expect(sendCalls[0]!.params).toEqual({ backendNodeId: 555 });
|
|
518
732
|
});
|
|
519
733
|
|
|
520
734
|
test("presses key on targeted element via selector", async () => {
|
|
@@ -524,13 +738,24 @@ describe("executeBrowserPressKey", () => {
|
|
|
524
738
|
);
|
|
525
739
|
expect(result.isError).toBe(false);
|
|
526
740
|
expect(result.content).toContain('Pressed "Escape" on element');
|
|
527
|
-
|
|
741
|
+
// Selector path: DOM.getDocument → DOM.querySelector → DOM.describeNode
|
|
742
|
+
// → DOM.focus → dispatchKeyEvent × 2 (Escape has no text, so no char event).
|
|
743
|
+
const methods = sendCalls.map((c) => c.method);
|
|
744
|
+
expect(methods).toEqual([
|
|
745
|
+
"DOM.getDocument",
|
|
746
|
+
"DOM.querySelector",
|
|
747
|
+
"DOM.describeNode",
|
|
748
|
+
"DOM.focus",
|
|
749
|
+
"Input.dispatchKeyEvent",
|
|
750
|
+
"Input.dispatchKeyEvent",
|
|
751
|
+
]);
|
|
528
752
|
});
|
|
529
753
|
|
|
530
754
|
test("errors when key is missing", async () => {
|
|
531
755
|
const result = await executeBrowserPressKey({}, ctx);
|
|
532
756
|
expect(result.isError).toBe(true);
|
|
533
757
|
expect(result.content).toContain("key is required");
|
|
758
|
+
expect(sendCalls).toHaveLength(0);
|
|
534
759
|
});
|
|
535
760
|
|
|
536
761
|
test("errors when element_id not found", async () => {
|
|
@@ -540,12 +765,11 @@ describe("executeBrowserPressKey", () => {
|
|
|
540
765
|
);
|
|
541
766
|
expect(result.isError).toBe(true);
|
|
542
767
|
expect(result.content).toContain('element_id "e99" not found');
|
|
768
|
+
expect(sendCalls).toHaveLength(0);
|
|
543
769
|
});
|
|
544
770
|
|
|
545
|
-
test("
|
|
546
|
-
|
|
547
|
-
throw new Error("Key not recognized");
|
|
548
|
-
});
|
|
771
|
+
test("surfaces CDP failure as a press-key error", async () => {
|
|
772
|
+
sendHandler = () => new Error("Key not recognized");
|
|
549
773
|
const result = await executeBrowserPressKey({ key: "InvalidKey" }, ctx);
|
|
550
774
|
expect(result.isError).toBe(true);
|
|
551
775
|
expect(result.content).toContain("Press key failed");
|
|
@@ -558,14 +782,32 @@ describe("executeBrowserPressKey", () => {
|
|
|
558
782
|
describe("executeBrowserScroll", () => {
|
|
559
783
|
beforeEach(() => {
|
|
560
784
|
resetMockPage();
|
|
561
|
-
|
|
785
|
+
resetCdpMock();
|
|
786
|
+
sendHandler = defaultCdpHandler;
|
|
562
787
|
});
|
|
563
788
|
|
|
564
789
|
test("scrolls down by default amount", async () => {
|
|
565
790
|
const result = await executeBrowserScroll({ direction: "down" }, ctx);
|
|
566
791
|
expect(result.isError).toBe(false);
|
|
567
792
|
expect(result.content).toContain("Scrolled down by 500px");
|
|
568
|
-
|
|
793
|
+
// Runtime.evaluate for viewport dimensions, then a single
|
|
794
|
+
// Input.dispatchMouseEvent mouseWheel at the viewport center.
|
|
795
|
+
const evaluateCall = sendCalls.find((c) => c.method === "Runtime.evaluate");
|
|
796
|
+
expect(evaluateCall).toBeDefined();
|
|
797
|
+
expect((evaluateCall!.params as { expression: string }).expression).toBe(
|
|
798
|
+
"({ w: window.innerWidth, h: window.innerHeight })",
|
|
799
|
+
);
|
|
800
|
+
const wheelCall = sendCalls.find(
|
|
801
|
+
(c) => c.method === "Input.dispatchMouseEvent",
|
|
802
|
+
);
|
|
803
|
+
expect(wheelCall).toBeDefined();
|
|
804
|
+
expect(wheelCall!.params).toEqual({
|
|
805
|
+
type: "mouseWheel",
|
|
806
|
+
x: 400,
|
|
807
|
+
y: 300,
|
|
808
|
+
deltaX: 0,
|
|
809
|
+
deltaY: 500,
|
|
810
|
+
});
|
|
569
811
|
});
|
|
570
812
|
|
|
571
813
|
test("scrolls up by custom amount", async () => {
|
|
@@ -575,7 +817,16 @@ describe("executeBrowserScroll", () => {
|
|
|
575
817
|
);
|
|
576
818
|
expect(result.isError).toBe(false);
|
|
577
819
|
expect(result.content).toContain("Scrolled up by 300px");
|
|
578
|
-
|
|
820
|
+
const wheelCall = sendCalls.find(
|
|
821
|
+
(c) => c.method === "Input.dispatchMouseEvent",
|
|
822
|
+
);
|
|
823
|
+
expect(wheelCall!.params).toEqual({
|
|
824
|
+
type: "mouseWheel",
|
|
825
|
+
x: 400,
|
|
826
|
+
y: 300,
|
|
827
|
+
deltaX: 0,
|
|
828
|
+
deltaY: -300,
|
|
829
|
+
});
|
|
579
830
|
});
|
|
580
831
|
|
|
581
832
|
test("scrolls left", async () => {
|
|
@@ -584,7 +835,16 @@ describe("executeBrowserScroll", () => {
|
|
|
584
835
|
ctx,
|
|
585
836
|
);
|
|
586
837
|
expect(result.isError).toBe(false);
|
|
587
|
-
|
|
838
|
+
const wheelCall = sendCalls.find(
|
|
839
|
+
(c) => c.method === "Input.dispatchMouseEvent",
|
|
840
|
+
);
|
|
841
|
+
expect(wheelCall!.params).toEqual({
|
|
842
|
+
type: "mouseWheel",
|
|
843
|
+
x: 400,
|
|
844
|
+
y: 300,
|
|
845
|
+
deltaX: -200,
|
|
846
|
+
deltaY: 0,
|
|
847
|
+
});
|
|
588
848
|
});
|
|
589
849
|
|
|
590
850
|
test("scrolls right", async () => {
|
|
@@ -593,35 +853,79 @@ describe("executeBrowserScroll", () => {
|
|
|
593
853
|
ctx,
|
|
594
854
|
);
|
|
595
855
|
expect(result.isError).toBe(false);
|
|
596
|
-
|
|
856
|
+
const wheelCall = sendCalls.find(
|
|
857
|
+
(c) => c.method === "Input.dispatchMouseEvent",
|
|
858
|
+
);
|
|
859
|
+
expect(wheelCall!.params).toEqual({
|
|
860
|
+
type: "mouseWheel",
|
|
861
|
+
x: 400,
|
|
862
|
+
y: 300,
|
|
863
|
+
deltaX: 200,
|
|
864
|
+
deltaY: 0,
|
|
865
|
+
});
|
|
597
866
|
});
|
|
598
867
|
|
|
599
868
|
test("errors when direction is missing", async () => {
|
|
600
869
|
const result = await executeBrowserScroll({}, ctx);
|
|
601
870
|
expect(result.isError).toBe(true);
|
|
602
871
|
expect(result.content).toContain("direction is required");
|
|
872
|
+
expect(sendCalls).toHaveLength(0);
|
|
603
873
|
});
|
|
604
874
|
|
|
605
875
|
test("errors when direction is invalid", async () => {
|
|
606
876
|
const result = await executeBrowserScroll({ direction: "diagonal" }, ctx);
|
|
607
877
|
expect(result.isError).toBe(true);
|
|
608
878
|
expect(result.content).toContain("direction is required");
|
|
879
|
+
expect(sendCalls).toHaveLength(0);
|
|
880
|
+
});
|
|
881
|
+
|
|
882
|
+
test("surfaces CDP failure as a scroll error", async () => {
|
|
883
|
+
sendHandler = () => new Error("viewport unavailable");
|
|
884
|
+
const result = await executeBrowserScroll({ direction: "down" }, ctx);
|
|
885
|
+
expect(result.isError).toBe(true);
|
|
886
|
+
expect(result.content).toContain("Scroll failed");
|
|
887
|
+
expect(result.content).toContain("viewport unavailable");
|
|
609
888
|
});
|
|
610
889
|
});
|
|
611
890
|
|
|
612
891
|
// ── browser_select_option ────────────────────────────────────────────
|
|
613
892
|
|
|
893
|
+
/**
|
|
894
|
+
* Default handler tuned for select-option tests. The Runtime.callFunctionOn
|
|
895
|
+
* call now returns whether an option matched; tests assert on this
|
|
896
|
+
* via `result.value`.
|
|
897
|
+
*/
|
|
898
|
+
function selectOptionHandler(
|
|
899
|
+
matched = true,
|
|
900
|
+
): (method: string, params?: Record<string, unknown>) => unknown {
|
|
901
|
+
return (method, _params) => {
|
|
902
|
+
switch (method) {
|
|
903
|
+
case "DOM.getDocument":
|
|
904
|
+
return { root: { nodeId: 1 } };
|
|
905
|
+
case "DOM.querySelector":
|
|
906
|
+
return { nodeId: 42 };
|
|
907
|
+
case "DOM.describeNode":
|
|
908
|
+
return { node: { backendNodeId: 100 } };
|
|
909
|
+
case "DOM.resolveNode":
|
|
910
|
+
return { object: { objectId: "obj-1" } };
|
|
911
|
+
case "Runtime.callFunctionOn":
|
|
912
|
+
return { result: { value: matched } };
|
|
913
|
+
default:
|
|
914
|
+
return {};
|
|
915
|
+
}
|
|
916
|
+
};
|
|
917
|
+
}
|
|
918
|
+
|
|
614
919
|
describe("executeBrowserSelectOption", () => {
|
|
615
920
|
beforeEach(() => {
|
|
616
921
|
resetMockPage();
|
|
617
|
-
|
|
922
|
+
resetCdpMock();
|
|
923
|
+
snapshotBackendNodeMaps.clear();
|
|
924
|
+
sendHandler = selectOptionHandler();
|
|
618
925
|
});
|
|
619
926
|
|
|
620
927
|
test("selects by value via element_id", async () => {
|
|
621
|
-
|
|
622
|
-
"test-conversation",
|
|
623
|
-
new Map([["e4", '[data-vellum-eid="e4"]']]),
|
|
624
|
-
);
|
|
928
|
+
snapshotBackendNodeMaps.set("test-conversation", new Map([["e4", 777]]));
|
|
625
929
|
const result = await executeBrowserSelectOption(
|
|
626
930
|
{ element_id: "e4", value: "ca" },
|
|
627
931
|
ctx,
|
|
@@ -629,10 +933,26 @@ describe("executeBrowserSelectOption", () => {
|
|
|
629
933
|
expect(result.isError).toBe(false);
|
|
630
934
|
expect(result.content).toContain("Selected option");
|
|
631
935
|
expect(result.content).toContain('value="ca"');
|
|
632
|
-
expect(
|
|
633
|
-
|
|
936
|
+
expect(result.content).toContain('element_id "e4"');
|
|
937
|
+
|
|
938
|
+
// Expected CDP sequence: DOM.resolveNode → Runtime.callFunctionOn
|
|
939
|
+
const methods = sendCalls.map((c) => c.method);
|
|
940
|
+
expect(methods).toEqual(["DOM.resolveNode", "Runtime.callFunctionOn"]);
|
|
941
|
+
expect(sendCalls[0]!.params).toEqual({ backendNodeId: 777 });
|
|
942
|
+
const callFn = sendCalls[1]!.params as {
|
|
943
|
+
objectId: string;
|
|
944
|
+
arguments: Array<{ value: unknown }>;
|
|
945
|
+
returnByValue?: boolean;
|
|
946
|
+
};
|
|
947
|
+
expect(callFn.objectId).toBe("obj-1");
|
|
948
|
+
expect(callFn.arguments).toEqual([
|
|
634
949
|
{ value: "ca" },
|
|
635
|
-
|
|
950
|
+
{ value: null },
|
|
951
|
+
{ value: null },
|
|
952
|
+
]);
|
|
953
|
+
// returnByValue must be true so the matched boolean comes back
|
|
954
|
+
// primitive instead of as a RemoteObject reference.
|
|
955
|
+
expect(callFn.returnByValue).toBe(true);
|
|
636
956
|
});
|
|
637
957
|
|
|
638
958
|
test("selects by label", async () => {
|
|
@@ -642,9 +962,23 @@ describe("executeBrowserSelectOption", () => {
|
|
|
642
962
|
);
|
|
643
963
|
expect(result.isError).toBe(false);
|
|
644
964
|
expect(result.content).toContain('label="California"');
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
965
|
+
// Selector path: querySelectorBackendNodeId sequence + DOM.resolveNode + Runtime.callFunctionOn
|
|
966
|
+
const methods = sendCalls.map((c) => c.method);
|
|
967
|
+
expect(methods).toEqual([
|
|
968
|
+
"DOM.getDocument",
|
|
969
|
+
"DOM.querySelector",
|
|
970
|
+
"DOM.describeNode",
|
|
971
|
+
"DOM.resolveNode",
|
|
972
|
+
"Runtime.callFunctionOn",
|
|
973
|
+
]);
|
|
974
|
+
const callFn = sendCalls[4]!.params as {
|
|
975
|
+
arguments: Array<{ value: unknown }>;
|
|
976
|
+
};
|
|
977
|
+
expect(callFn.arguments).toEqual([
|
|
978
|
+
{ value: null },
|
|
979
|
+
{ value: "California" },
|
|
980
|
+
{ value: null },
|
|
981
|
+
]);
|
|
648
982
|
});
|
|
649
983
|
|
|
650
984
|
test("selects by index", async () => {
|
|
@@ -654,7 +988,39 @@ describe("executeBrowserSelectOption", () => {
|
|
|
654
988
|
);
|
|
655
989
|
expect(result.isError).toBe(false);
|
|
656
990
|
expect(result.content).toContain("index=2");
|
|
657
|
-
|
|
991
|
+
const callFn = sendCalls.find((c) => c.method === "Runtime.callFunctionOn")!
|
|
992
|
+
.params as { arguments: Array<{ value: unknown }> };
|
|
993
|
+
expect(callFn.arguments).toEqual([
|
|
994
|
+
{ value: null },
|
|
995
|
+
{ value: null },
|
|
996
|
+
{ value: 2 },
|
|
997
|
+
]);
|
|
998
|
+
});
|
|
999
|
+
|
|
1000
|
+
test("returns error when no option matches", async () => {
|
|
1001
|
+
sendHandler = selectOptionHandler(false);
|
|
1002
|
+
const result = await executeBrowserSelectOption(
|
|
1003
|
+
{ selector: "#state", value: "nope" },
|
|
1004
|
+
ctx,
|
|
1005
|
+
);
|
|
1006
|
+
expect(result.isError).toBe(true);
|
|
1007
|
+
expect(result.content).toContain("Select option failed");
|
|
1008
|
+
expect(result.content).toContain("no option matched");
|
|
1009
|
+
expect(result.content).toContain('value="nope"');
|
|
1010
|
+
});
|
|
1011
|
+
|
|
1012
|
+
test("dispatches input + change events via the function declaration", async () => {
|
|
1013
|
+
await executeBrowserSelectOption({ selector: "#state", value: "ca" }, ctx);
|
|
1014
|
+
const callFn = sendCalls.find((c) => c.method === "Runtime.callFunctionOn")!
|
|
1015
|
+
.params as { functionDeclaration: string };
|
|
1016
|
+
// The function body must dispatch BOTH input and change events
|
|
1017
|
+
// (HTML spec order: input fires before change for <select>).
|
|
1018
|
+
expect(callFn.functionDeclaration).toContain('new Event("input"');
|
|
1019
|
+
expect(callFn.functionDeclaration).toContain('new Event("change"');
|
|
1020
|
+
const inputIdx = callFn.functionDeclaration.indexOf('new Event("input"');
|
|
1021
|
+
const changeIdx = callFn.functionDeclaration.indexOf('new Event("change"');
|
|
1022
|
+
expect(inputIdx).toBeGreaterThanOrEqual(0);
|
|
1023
|
+
expect(changeIdx).toBeGreaterThan(inputIdx);
|
|
658
1024
|
});
|
|
659
1025
|
|
|
660
1026
|
test("errors when no option specifier provided", async () => {
|
|
@@ -666,6 +1032,7 @@ describe("executeBrowserSelectOption", () => {
|
|
|
666
1032
|
expect(result.content).toContain(
|
|
667
1033
|
"One of value, label, or index is required",
|
|
668
1034
|
);
|
|
1035
|
+
expect(sendCalls).toHaveLength(0);
|
|
669
1036
|
});
|
|
670
1037
|
|
|
671
1038
|
test("errors when neither element_id nor selector provided", async () => {
|
|
@@ -674,12 +1041,11 @@ describe("executeBrowserSelectOption", () => {
|
|
|
674
1041
|
expect(result.content).toContain(
|
|
675
1042
|
"Either element_id or selector is required",
|
|
676
1043
|
);
|
|
1044
|
+
expect(sendCalls).toHaveLength(0);
|
|
677
1045
|
});
|
|
678
1046
|
|
|
679
|
-
test("
|
|
680
|
-
|
|
681
|
-
throw new Error("Not a select element");
|
|
682
|
-
});
|
|
1047
|
+
test("surfaces CDP failure as a select-option error", async () => {
|
|
1048
|
+
sendHandler = () => new Error("Not a select element");
|
|
683
1049
|
const result = await executeBrowserSelectOption(
|
|
684
1050
|
{ selector: "#div", value: "x" },
|
|
685
1051
|
ctx,
|
|
@@ -692,34 +1058,72 @@ describe("executeBrowserSelectOption", () => {
|
|
|
692
1058
|
|
|
693
1059
|
// ── browser_hover ────────────────────────────────────────────────────
|
|
694
1060
|
|
|
695
|
-
describe("executeBrowserHover", () => {
|
|
1061
|
+
describe("executeBrowserHover (CDP)", () => {
|
|
696
1062
|
beforeEach(() => {
|
|
697
1063
|
resetMockPage();
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
test("hovers by element_id via snapshot map", async () => {
|
|
702
|
-
snapshotMaps.set(
|
|
703
|
-
"test-conversation",
|
|
704
|
-
new Map([["e2", '[data-vellum-eid="e2"]']]),
|
|
705
|
-
);
|
|
706
|
-
const result = await executeBrowserHover({ element_id: "e2" }, ctx);
|
|
707
|
-
expect(result.isError).toBe(false);
|
|
708
|
-
expect(result.content).toContain("Hovered element");
|
|
709
|
-
expect(mockPage.hover).toHaveBeenCalledWith('[data-vellum-eid="e2"]', {
|
|
710
|
-
timeout: 10000,
|
|
711
|
-
});
|
|
1064
|
+
resetCdpMock();
|
|
1065
|
+
snapshotBackendNodeMaps.clear();
|
|
712
1066
|
});
|
|
713
1067
|
|
|
714
|
-
test("hovers by
|
|
1068
|
+
test("hovers by selector: emits a single mouseMoved event", async () => {
|
|
1069
|
+
installClickHoverCdpSend({ backendNodeId: 9000 });
|
|
715
1070
|
const result = await executeBrowserHover(
|
|
716
1071
|
{ selector: ".menu-trigger" },
|
|
717
1072
|
ctx,
|
|
718
1073
|
);
|
|
719
1074
|
expect(result.isError).toBe(false);
|
|
720
|
-
expect(
|
|
721
|
-
|
|
1075
|
+
expect(result.content).toContain("Hovered element: .menu-trigger");
|
|
1076
|
+
|
|
1077
|
+
// Selector path waits for the element to become visible via
|
|
1078
|
+
// cdpWaitForSelector before resolving the backend node.
|
|
1079
|
+
const methods = sendCalls.map((c) => c.method);
|
|
1080
|
+
expect(methods).toEqual([
|
|
1081
|
+
"Runtime.evaluate",
|
|
1082
|
+
"DOM.getDocument",
|
|
1083
|
+
"DOM.querySelector",
|
|
1084
|
+
"DOM.describeNode",
|
|
1085
|
+
"DOM.scrollIntoViewIfNeeded",
|
|
1086
|
+
"DOM.getBoxModel",
|
|
1087
|
+
"Input.dispatchMouseEvent",
|
|
1088
|
+
]);
|
|
1089
|
+
|
|
1090
|
+
// Exactly ONE mouseMoved event (no press/release) → hover semantics.
|
|
1091
|
+
const mouseCalls = sendCalls.filter(
|
|
1092
|
+
(c) => c.method === "Input.dispatchMouseEvent",
|
|
1093
|
+
);
|
|
1094
|
+
expect(mouseCalls).toHaveLength(1);
|
|
1095
|
+
expect(mouseCalls[0]!.params).toMatchObject({
|
|
1096
|
+
type: "mouseMoved",
|
|
1097
|
+
x: 20,
|
|
1098
|
+
y: 30,
|
|
1099
|
+
button: "none",
|
|
722
1100
|
});
|
|
1101
|
+
|
|
1102
|
+
await new Promise((resolve) => setTimeout(resolve, 0));
|
|
1103
|
+
expect(detachCalls).toBe(1);
|
|
1104
|
+
});
|
|
1105
|
+
|
|
1106
|
+
test("hovers by element_id (backend path): skips DOM.querySelector", async () => {
|
|
1107
|
+
snapshotBackendNodeMaps.set("test-conversation", new Map([["e2", 12]]));
|
|
1108
|
+
installClickHoverCdpSend();
|
|
1109
|
+
|
|
1110
|
+
const result = await executeBrowserHover({ element_id: "e2" }, ctx);
|
|
1111
|
+
|
|
1112
|
+
expect(result.isError).toBe(false);
|
|
1113
|
+
expect(result.content).toContain("Hovered element: eid=e2");
|
|
1114
|
+
|
|
1115
|
+
const methods = sendCalls.map((c) => c.method);
|
|
1116
|
+
expect(methods).not.toContain("DOM.querySelector");
|
|
1117
|
+
expect(methods).toEqual([
|
|
1118
|
+
"DOM.scrollIntoViewIfNeeded",
|
|
1119
|
+
"DOM.getBoxModel",
|
|
1120
|
+
"Input.dispatchMouseEvent",
|
|
1121
|
+
]);
|
|
1122
|
+
|
|
1123
|
+
const scrollCall = sendCalls.find(
|
|
1124
|
+
(c) => c.method === "DOM.scrollIntoViewIfNeeded",
|
|
1125
|
+
)!;
|
|
1126
|
+
expect(scrollCall.params).toMatchObject({ backendNodeId: 12 });
|
|
723
1127
|
});
|
|
724
1128
|
|
|
725
1129
|
test("errors when neither element_id nor selector provided", async () => {
|
|
@@ -728,22 +1132,28 @@ describe("executeBrowserHover", () => {
|
|
|
728
1132
|
expect(result.content).toContain(
|
|
729
1133
|
"Either element_id or selector is required",
|
|
730
1134
|
);
|
|
1135
|
+
expect(sendCalls).toHaveLength(0);
|
|
731
1136
|
});
|
|
732
1137
|
|
|
733
1138
|
test("errors when element_id not found in snapshot map", async () => {
|
|
1139
|
+
installClickHoverCdpSend();
|
|
734
1140
|
const result = await executeBrowserHover({ element_id: "e99" }, ctx);
|
|
735
1141
|
expect(result.isError).toBe(true);
|
|
736
1142
|
expect(result.content).toContain('element_id "e99" not found');
|
|
1143
|
+
expect(sendCalls).toHaveLength(0);
|
|
737
1144
|
});
|
|
738
1145
|
|
|
739
|
-
test("
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
});
|
|
1146
|
+
test("returns error + still disposes CdpClient when cdp.send throws", async () => {
|
|
1147
|
+
installClickHoverCdpSend({ throwFrom: "DOM.getBoxModel" });
|
|
1148
|
+
|
|
743
1149
|
const result = await executeBrowserHover({ selector: "#gone" }, ctx);
|
|
1150
|
+
|
|
744
1151
|
expect(result.isError).toBe(true);
|
|
745
1152
|
expect(result.content).toContain("Hover failed");
|
|
746
|
-
expect(result.content).toContain("
|
|
1153
|
+
expect(result.content).toContain("cdp boom");
|
|
1154
|
+
|
|
1155
|
+
await new Promise((resolve) => setTimeout(resolve, 0));
|
|
1156
|
+
expect(detachCalls).toBe(1);
|
|
747
1157
|
});
|
|
748
1158
|
});
|
|
749
1159
|
|
|
@@ -754,14 +1164,14 @@ describe("executeBrowserHover", () => {
|
|
|
754
1164
|
describe("browser execution wrapper contract", () => {
|
|
755
1165
|
beforeEach(() => {
|
|
756
1166
|
resetMockPage();
|
|
757
|
-
|
|
1167
|
+
resetCdpMock();
|
|
1168
|
+
sendHandler = defaultCdpHandler;
|
|
1169
|
+
snapshotBackendNodeMaps.clear();
|
|
758
1170
|
});
|
|
759
1171
|
|
|
760
1172
|
test("executeBrowserClick matches wrapper contract (input, context) → result", async () => {
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
new Map([["e1", '[data-vellum-eid="e1"]']]),
|
|
764
|
-
);
|
|
1173
|
+
installClickHoverCdpSend();
|
|
1174
|
+
snapshotBackendNodeMaps.set("test-conversation", new Map([["e1", 1]]));
|
|
765
1175
|
const result = await executeBrowserClick({ element_id: "e1" }, ctx);
|
|
766
1176
|
expect(result).toHaveProperty("content");
|
|
767
1177
|
expect(result).toHaveProperty("isError");
|
|
@@ -771,10 +1181,7 @@ describe("browser execution wrapper contract", () => {
|
|
|
771
1181
|
});
|
|
772
1182
|
|
|
773
1183
|
test("executeBrowserType matches wrapper contract", async () => {
|
|
774
|
-
|
|
775
|
-
"test-conversation",
|
|
776
|
-
new Map([["e3", '[data-vellum-eid="e3"]']]),
|
|
777
|
-
);
|
|
1184
|
+
snapshotBackendNodeMaps.set("test-conversation", new Map([["e3", 555]]));
|
|
778
1185
|
const result = await executeBrowserType(
|
|
779
1186
|
{ element_id: "e3", text: "hello" },
|
|
780
1187
|
ctx,
|
|
@@ -784,27 +1191,12 @@ describe("browser execution wrapper contract", () => {
|
|
|
784
1191
|
expect(result.isError).toBe(false);
|
|
785
1192
|
});
|
|
786
1193
|
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
{ eid: "e1", tag: "button", attrs: {}, text: "Click me" },
|
|
790
|
-
]);
|
|
791
|
-
mockPage.title = mock(async () => "Test");
|
|
792
|
-
mockPage.url = mock(() => "https://example.com");
|
|
793
|
-
const result = await executeBrowserSnapshot({}, ctx);
|
|
794
|
-
expect(result).toHaveProperty("content");
|
|
795
|
-
expect(result).toHaveProperty("isError");
|
|
796
|
-
expect(result.isError).toBe(false);
|
|
797
|
-
});
|
|
1194
|
+
// executeBrowserSnapshot wrapper-contract check lives in
|
|
1195
|
+
// `headless-browser-snapshot.test.ts`.
|
|
798
1196
|
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
mockPage.url = mock(() => "https://example.com");
|
|
803
|
-
const result = await executeBrowserExtract({}, ctx);
|
|
804
|
-
expect(result).toHaveProperty("content");
|
|
805
|
-
expect(result).toHaveProperty("isError");
|
|
806
|
-
expect(result.isError).toBe(false);
|
|
807
|
-
});
|
|
1197
|
+
// wrapper contract for executeBrowserExtract and
|
|
1198
|
+
// executeBrowserScreenshot lives in
|
|
1199
|
+
// headless-browser-read-tools.test.ts.
|
|
808
1200
|
|
|
809
1201
|
test("executeBrowserPressKey matches wrapper contract", async () => {
|
|
810
1202
|
const result = await executeBrowserPressKey({ key: "Enter" }, ctx);
|
|
@@ -813,14 +1205,6 @@ describe("browser execution wrapper contract", () => {
|
|
|
813
1205
|
expect(result.isError).toBe(false);
|
|
814
1206
|
});
|
|
815
1207
|
|
|
816
|
-
test("executeBrowserScreenshot matches wrapper contract", async () => {
|
|
817
|
-
mockPage.screenshot = mock(async () => Buffer.from("fake-image"));
|
|
818
|
-
const result = await executeBrowserScreenshot({}, ctx);
|
|
819
|
-
expect(result).toHaveProperty("content");
|
|
820
|
-
expect(result).toHaveProperty("isError");
|
|
821
|
-
expect(result.isError).toBe(false);
|
|
822
|
-
});
|
|
823
|
-
|
|
824
1208
|
test("executeBrowserClose matches wrapper contract", async () => {
|
|
825
1209
|
const result = await executeBrowserClose({}, ctx);
|
|
826
1210
|
expect(result).toHaveProperty("content");
|
|
@@ -836,10 +1220,7 @@ describe("browser execution wrapper contract", () => {
|
|
|
836
1220
|
});
|
|
837
1221
|
|
|
838
1222
|
test("executeBrowserSelectOption matches wrapper contract", async () => {
|
|
839
|
-
|
|
840
|
-
"test-conversation",
|
|
841
|
-
new Map([["e4", '[data-vellum-eid="e4"]']]),
|
|
842
|
-
);
|
|
1223
|
+
snapshotBackendNodeMaps.set("test-conversation", new Map([["e4", 777]]));
|
|
843
1224
|
const result = await executeBrowserSelectOption(
|
|
844
1225
|
{ element_id: "e4", value: "opt1" },
|
|
845
1226
|
ctx,
|
|
@@ -850,10 +1231,8 @@ describe("browser execution wrapper contract", () => {
|
|
|
850
1231
|
});
|
|
851
1232
|
|
|
852
1233
|
test("executeBrowserHover matches wrapper contract", async () => {
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
new Map([["e2", '[data-vellum-eid="e2"]']]),
|
|
856
|
-
);
|
|
1234
|
+
installClickHoverCdpSend();
|
|
1235
|
+
snapshotBackendNodeMaps.set("test-conversation", new Map([["e2", 2]]));
|
|
857
1236
|
const result = await executeBrowserHover({ element_id: "e2" }, ctx);
|
|
858
1237
|
expect(result).toHaveProperty("content");
|
|
859
1238
|
expect(result).toHaveProperty("isError");
|