@rudderjs/ai 1.17.3 → 1.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -1274
- package/dist/budget-orm/index.d.ts +1 -95
- package/dist/budget-orm/index.d.ts.map +1 -1
- package/dist/budget-orm/index.js +4 -176
- package/dist/budget-orm/index.js.map +1 -1
- package/dist/chat-mentions.d.ts +1 -58
- package/dist/chat-mentions.d.ts.map +1 -1
- package/dist/chat-mentions.js +4 -80
- package/dist/chat-mentions.js.map +1 -1
- package/dist/commands/ai-eval.d.ts +1 -92
- package/dist/commands/ai-eval.d.ts.map +1 -1
- package/dist/commands/ai-eval.js +4 -377
- package/dist/commands/ai-eval.js.map +1 -1
- package/dist/commands/make-agent.d.ts +1 -2
- package/dist/commands/make-agent.d.ts.map +1 -1
- package/dist/commands/make-agent.js +4 -22
- package/dist/commands/make-agent.js.map +1 -1
- package/dist/computer-use/index.d.ts +1 -52
- package/dist/computer-use/index.d.ts.map +1 -1
- package/dist/computer-use/index.js +4 -50
- package/dist/computer-use/index.js.map +1 -1
- package/dist/conversation-orm/index.d.ts +1 -108
- package/dist/conversation-orm/index.d.ts.map +1 -1
- package/dist/conversation-orm/index.js +4 -214
- package/dist/conversation-orm/index.js.map +1 -1
- package/dist/doctor.d.ts +1 -1
- package/dist/doctor.d.ts.map +1 -1
- package/dist/doctor.js +4 -65
- package/dist/doctor.js.map +1 -1
- package/dist/eval/index.d.ts +1 -270
- package/dist/eval/index.d.ts.map +1 -1
- package/dist/eval/index.js +4 -509
- package/dist/eval/index.js.map +1 -1
- package/dist/gateway/index.d.ts +1 -10
- package/dist/gateway/index.d.ts.map +1 -1
- package/dist/gateway/index.js +4 -10
- package/dist/gateway/index.js.map +1 -1
- package/dist/index.d.ts +1 -66
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -78
- package/dist/index.js.map +1 -1
- package/dist/mcp/index.d.ts +1 -15
- package/dist/mcp/index.d.ts.map +1 -1
- package/dist/mcp/index.js +4 -14
- package/dist/mcp/index.js.map +1 -1
- package/dist/memory-embedding/index.d.ts +1 -120
- package/dist/memory-embedding/index.d.ts.map +1 -1
- package/dist/memory-embedding/index.js +4 -228
- package/dist/memory-embedding/index.js.map +1 -1
- package/dist/memory-orm/index.d.ts +1 -117
- package/dist/memory-orm/index.d.ts.map +1 -1
- package/dist/memory-orm/index.js +4 -186
- package/dist/memory-orm/index.js.map +1 -1
- package/dist/node/index.d.ts +1 -2
- package/dist/node/index.d.ts.map +1 -1
- package/dist/node/index.js +4 -2
- package/dist/node/index.js.map +1 -1
- package/dist/observers.d.ts +1 -129
- package/dist/observers.d.ts.map +1 -1
- package/dist/observers.js +4 -39
- package/dist/observers.js.map +1 -1
- package/dist/react/index.d.ts +1 -15
- package/dist/react/index.d.ts.map +1 -1
- package/dist/react/index.js +4 -15
- package/dist/react/index.js.map +1 -1
- package/dist/server/index.d.ts +1 -1
- package/dist/server/index.d.ts.map +1 -1
- package/dist/server/index.js +4 -1
- package/dist/server/index.js.map +1 -1
- package/package.json +9 -13
- package/boost/guidelines.md +0 -260
- package/boost/skills/ai-agents/SKILL.md +0 -240
- package/boost/skills/ai-tools/SKILL.md +0 -260
- package/dist/agent-run-store.d.ts +0 -161
- package/dist/agent-run-store.d.ts.map +0 -1
- package/dist/agent-run-store.js +0 -98
- package/dist/agent-run-store.js.map +0 -1
- package/dist/agent-sse.d.ts +0 -153
- package/dist/agent-sse.d.ts.map +0 -1
- package/dist/agent-sse.js +0 -282
- package/dist/agent-sse.js.map +0 -1
- package/dist/agent.d.ts +0 -508
- package/dist/agent.d.ts.map +0 -1
- package/dist/agent.js +0 -1538
- package/dist/agent.js.map +0 -1
- package/dist/attachment.d.ts +0 -31
- package/dist/attachment.d.ts.map +0 -1
- package/dist/attachment.js +0 -89
- package/dist/attachment.js.map +0 -1
- package/dist/audio.d.ts +0 -45
- package/dist/audio.d.ts.map +0 -1
- package/dist/audio.js +0 -93
- package/dist/audio.js.map +0 -1
- package/dist/base64.d.ts +0 -7
- package/dist/base64.d.ts.map +0 -1
- package/dist/base64.js +0 -39
- package/dist/base64.js.map +0 -1
- package/dist/budget/pricing.d.ts +0 -124
- package/dist/budget/pricing.d.ts.map +0 -1
- package/dist/budget/pricing.js +0 -175
- package/dist/budget/pricing.js.map +0 -1
- package/dist/budget/storage.d.ts +0 -104
- package/dist/budget/storage.d.ts.map +0 -1
- package/dist/budget/storage.js +0 -0
- package/dist/budget/storage.js.map +0 -1
- package/dist/budget/with-budget.d.ts +0 -119
- package/dist/budget/with-budget.d.ts.map +0 -1
- package/dist/budget/with-budget.js +0 -175
- package/dist/budget/with-budget.js.map +0 -1
- package/dist/cached-embedding.d.ts +0 -14
- package/dist/cached-embedding.d.ts.map +0 -1
- package/dist/cached-embedding.js +0 -44
- package/dist/cached-embedding.js.map +0 -1
- package/dist/computer-use/actions.d.ts +0 -214
- package/dist/computer-use/actions.d.ts.map +0 -1
- package/dist/computer-use/actions.js +0 -48
- package/dist/computer-use/actions.js.map +0 -1
- package/dist/computer-use/errors.d.ts +0 -57
- package/dist/computer-use/errors.d.ts.map +0 -1
- package/dist/computer-use/errors.js +0 -76
- package/dist/computer-use/errors.js.map +0 -1
- package/dist/computer-use/playwright.d.ts +0 -76
- package/dist/computer-use/playwright.d.ts.map +0 -1
- package/dist/computer-use/playwright.js +0 -270
- package/dist/computer-use/playwright.js.map +0 -1
- package/dist/computer-use/tool.d.ts +0 -154
- package/dist/computer-use/tool.d.ts.map +0 -1
- package/dist/computer-use/tool.js +0 -210
- package/dist/computer-use/tool.js.map +0 -1
- package/dist/continuation-validation.d.ts +0 -85
- package/dist/continuation-validation.d.ts.map +0 -1
- package/dist/continuation-validation.js +0 -166
- package/dist/continuation-validation.js.map +0 -1
- package/dist/conversation-persistence.d.ts +0 -46
- package/dist/conversation-persistence.d.ts.map +0 -1
- package/dist/conversation-persistence.js +0 -176
- package/dist/conversation-persistence.js.map +0 -1
- package/dist/conversation.d.ts +0 -11
- package/dist/conversation.d.ts.map +0 -1
- package/dist/conversation.js +0 -55
- package/dist/conversation.js.map +0 -1
- package/dist/eval/fixtures.d.ts +0 -65
- package/dist/eval/fixtures.d.ts.map +0 -1
- package/dist/eval/fixtures.js +0 -110
- package/dist/eval/fixtures.js.map +0 -1
- package/dist/eval/html-reporter.d.ts +0 -25
- package/dist/eval/html-reporter.d.ts.map +0 -1
- package/dist/eval/html-reporter.js +0 -209
- package/dist/eval/html-reporter.js.map +0 -1
- package/dist/eval/json-reporter.d.ts +0 -43
- package/dist/eval/json-reporter.d.ts.map +0 -1
- package/dist/eval/json-reporter.js +0 -40
- package/dist/eval/json-reporter.js.map +0 -1
- package/dist/facade.d.ts +0 -96
- package/dist/facade.d.ts.map +0 -1
- package/dist/facade.js +0 -146
- package/dist/facade.js.map +0 -1
- package/dist/fake.d.ts +0 -201
- package/dist/fake.d.ts.map +0 -1
- package/dist/fake.js +0 -428
- package/dist/fake.js.map +0 -1
- package/dist/file-search.d.ts +0 -168
- package/dist/file-search.d.ts.map +0 -1
- package/dist/file-search.js +0 -158
- package/dist/file-search.js.map +0 -1
- package/dist/files.d.ts +0 -27
- package/dist/files.d.ts.map +0 -1
- package/dist/files.js +0 -44
- package/dist/files.js.map +0 -1
- package/dist/gateway/http-gateway-adapter.d.ts +0 -94
- package/dist/gateway/http-gateway-adapter.d.ts.map +0 -1
- package/dist/gateway/http-gateway-adapter.js +0 -106
- package/dist/gateway/http-gateway-adapter.js.map +0 -1
- package/dist/gateway/sse.d.ts +0 -28
- package/dist/gateway/sse.d.ts.map +0 -1
- package/dist/gateway/sse.js +0 -78
- package/dist/gateway/sse.js.map +0 -1
- package/dist/handoff.d.ts +0 -95
- package/dist/handoff.d.ts.map +0 -1
- package/dist/handoff.js +0 -78
- package/dist/handoff.js.map +0 -1
- package/dist/handoffs-driver.d.ts +0 -58
- package/dist/handoffs-driver.d.ts.map +0 -1
- package/dist/handoffs-driver.js +0 -103
- package/dist/handoffs-driver.js.map +0 -1
- package/dist/image.d.ts +0 -40
- package/dist/image.d.ts.map +0 -1
- package/dist/image.js +0 -109
- package/dist/image.js.map +0 -1
- package/dist/mcp/client-tools.d.ts +0 -39
- package/dist/mcp/client-tools.d.ts.map +0 -1
- package/dist/mcp/client-tools.js +0 -147
- package/dist/mcp/client-tools.js.map +0 -1
- package/dist/mcp/server-from-agent.d.ts +0 -24
- package/dist/mcp/server-from-agent.d.ts.map +0 -1
- package/dist/mcp/server-from-agent.js +0 -113
- package/dist/mcp/server-from-agent.js.map +0 -1
- package/dist/mcp/types.d.ts +0 -64
- package/dist/mcp/types.d.ts.map +0 -1
- package/dist/mcp/types.js +0 -6
- package/dist/mcp/types.js.map +0 -1
- package/dist/memory-extract.d.ts +0 -60
- package/dist/memory-extract.d.ts.map +0 -1
- package/dist/memory-extract.js +0 -163
- package/dist/memory-extract.js.map +0 -1
- package/dist/memory-inject.d.ts +0 -39
- package/dist/memory-inject.d.ts.map +0 -1
- package/dist/memory-inject.js +0 -135
- package/dist/memory-inject.js.map +0 -1
- package/dist/memory.d.ts +0 -55
- package/dist/memory.d.ts.map +0 -1
- package/dist/memory.js +0 -132
- package/dist/memory.js.map +0 -1
- package/dist/middleware.d.ts +0 -18
- package/dist/middleware.d.ts.map +0 -1
- package/dist/middleware.js +0 -72
- package/dist/middleware.js.map +0 -1
- package/dist/node/attachment.d.ts +0 -6
- package/dist/node/attachment.d.ts.map +0 -1
- package/dist/node/attachment.js +0 -35
- package/dist/node/attachment.js.map +0 -1
- package/dist/node/transcription.d.ts +0 -4
- package/dist/node/transcription.d.ts.map +0 -1
- package/dist/node/transcription.js +0 -8
- package/dist/node/transcription.js.map +0 -1
- package/dist/output.d.ts +0 -22
- package/dist/output.d.ts.map +0 -1
- package/dist/output.js +0 -60
- package/dist/output.js.map +0 -1
- package/dist/provider-tools.d.ts +0 -87
- package/dist/provider-tools.d.ts.map +0 -1
- package/dist/provider-tools.js +0 -189
- package/dist/provider-tools.js.map +0 -1
- package/dist/providers/anthropic.d.ts +0 -24
- package/dist/providers/anthropic.d.ts.map +0 -1
- package/dist/providers/anthropic.js +0 -405
- package/dist/providers/anthropic.js.map +0 -1
- package/dist/providers/azure.d.ts +0 -13
- package/dist/providers/azure.d.ts.map +0 -1
- package/dist/providers/azure.js +0 -15
- package/dist/providers/azure.js.map +0 -1
- package/dist/providers/bedrock.d.ts +0 -75
- package/dist/providers/bedrock.d.ts.map +0 -1
- package/dist/providers/bedrock.js +0 -181
- package/dist/providers/bedrock.js.map +0 -1
- package/dist/providers/cohere.d.ts +0 -13
- package/dist/providers/cohere.d.ts.map +0 -1
- package/dist/providers/cohere.js +0 -87
- package/dist/providers/cohere.js.map +0 -1
- package/dist/providers/deepseek.d.ts +0 -12
- package/dist/providers/deepseek.d.ts.map +0 -1
- package/dist/providers/deepseek.js +0 -15
- package/dist/providers/deepseek.js.map +0 -1
- package/dist/providers/elevenlabs.d.ts +0 -98
- package/dist/providers/elevenlabs.d.ts.map +0 -1
- package/dist/providers/elevenlabs.js +0 -229
- package/dist/providers/elevenlabs.js.map +0 -1
- package/dist/providers/google-cache-registry.d.ts +0 -132
- package/dist/providers/google-cache-registry.d.ts.map +0 -1
- package/dist/providers/google-cache-registry.js +0 -209
- package/dist/providers/google-cache-registry.js.map +0 -1
- package/dist/providers/google.d.ts +0 -38
- package/dist/providers/google.d.ts.map +0 -1
- package/dist/providers/google.js +0 -903
- package/dist/providers/google.js.map +0 -1
- package/dist/providers/groq.d.ts +0 -12
- package/dist/providers/groq.d.ts.map +0 -1
- package/dist/providers/groq.js +0 -15
- package/dist/providers/groq.js.map +0 -1
- package/dist/providers/jina.d.ts +0 -13
- package/dist/providers/jina.d.ts.map +0 -1
- package/dist/providers/jina.js +0 -90
- package/dist/providers/jina.js.map +0 -1
- package/dist/providers/mistral.d.ts +0 -13
- package/dist/providers/mistral.d.ts.map +0 -1
- package/dist/providers/mistral.js +0 -46
- package/dist/providers/mistral.js.map +0 -1
- package/dist/providers/ollama.d.ts +0 -11
- package/dist/providers/ollama.d.ts.map +0 -1
- package/dist/providers/ollama.js +0 -15
- package/dist/providers/ollama.js.map +0 -1
- package/dist/providers/openai.d.ts +0 -79
- package/dist/providers/openai.d.ts.map +0 -1
- package/dist/providers/openai.js +0 -792
- package/dist/providers/openai.js.map +0 -1
- package/dist/providers/openrouter.d.ts +0 -43
- package/dist/providers/openrouter.d.ts.map +0 -1
- package/dist/providers/openrouter.js +0 -21
- package/dist/providers/openrouter.js.map +0 -1
- package/dist/providers/voyage.d.ts +0 -91
- package/dist/providers/voyage.d.ts.map +0 -1
- package/dist/providers/voyage.js +0 -166
- package/dist/providers/voyage.js.map +0 -1
- package/dist/providers/xai.d.ts +0 -12
- package/dist/providers/xai.d.ts.map +0 -1
- package/dist/providers/xai.js +0 -15
- package/dist/providers/xai.js.map +0 -1
- package/dist/queue-job.d.ts +0 -100
- package/dist/queue-job.d.ts.map +0 -1
- package/dist/queue-job.js +0 -185
- package/dist/queue-job.js.map +0 -1
- package/dist/react/agent-run.d.ts +0 -111
- package/dist/react/agent-run.d.ts.map +0 -1
- package/dist/react/agent-run.js +0 -107
- package/dist/react/agent-run.js.map +0 -1
- package/dist/react/useAgentRun.d.ts +0 -68
- package/dist/react/useAgentRun.d.ts.map +0 -1
- package/dist/react/useAgentRun.js +0 -125
- package/dist/react/useAgentRun.js.map +0 -1
- package/dist/registry.d.ts +0 -45
- package/dist/registry.d.ts.map +0 -1
- package/dist/registry.js +0 -131
- package/dist/registry.js.map +0 -1
- package/dist/rerank.d.ts +0 -20
- package/dist/rerank.d.ts.map +0 -1
- package/dist/rerank.js +0 -40
- package/dist/rerank.js.map +0 -1
- package/dist/resume-approval.d.ts +0 -30
- package/dist/resume-approval.d.ts.map +0 -1
- package/dist/resume-approval.js +0 -147
- package/dist/resume-approval.js.map +0 -1
- package/dist/sanitize-conversation.d.ts +0 -43
- package/dist/sanitize-conversation.d.ts.map +0 -1
- package/dist/sanitize-conversation.js +0 -85
- package/dist/sanitize-conversation.js.map +0 -1
- package/dist/scoped-tool.d.ts +0 -98
- package/dist/scoped-tool.d.ts.map +0 -1
- package/dist/scoped-tool.js +0 -174
- package/dist/scoped-tool.js.map +0 -1
- package/dist/server/provider.d.ts +0 -22
- package/dist/server/provider.d.ts.map +0 -1
- package/dist/server/provider.js +0 -194
- package/dist/server/provider.js.map +0 -1
- package/dist/similarity-search.d.ts +0 -163
- package/dist/similarity-search.d.ts.map +0 -1
- package/dist/similarity-search.js +0 -147
- package/dist/similarity-search.js.map +0 -1
- package/dist/sub-agent-run-store.d.ts +0 -157
- package/dist/sub-agent-run-store.d.ts.map +0 -1
- package/dist/sub-agent-run-store.js +0 -87
- package/dist/sub-agent-run-store.js.map +0 -1
- package/dist/tool-execution.d.ts +0 -16
- package/dist/tool-execution.d.ts.map +0 -1
- package/dist/tool-execution.js +0 -498
- package/dist/tool-execution.js.map +0 -1
- package/dist/tool-helpers.d.ts +0 -77
- package/dist/tool-helpers.d.ts.map +0 -1
- package/dist/tool-helpers.js +0 -117
- package/dist/tool-helpers.js.map +0 -1
- package/dist/tool.d.ts +0 -216
- package/dist/tool.d.ts.map +0 -1
- package/dist/tool.js +0 -175
- package/dist/tool.js.map +0 -1
- package/dist/transcription.d.ts +0 -42
- package/dist/transcription.d.ts.map +0 -1
- package/dist/transcription.js +0 -77
- package/dist/transcription.js.map +0 -1
- package/dist/types.d.ts +0 -1020
- package/dist/types.d.ts.map +0 -1
- package/dist/types.js +0 -2
- package/dist/types.js.map +0 -1
- package/dist/util/hash.d.ts +0 -11
- package/dist/util/hash.d.ts.map +0 -1
- package/dist/util/hash.js +0 -23
- package/dist/util/hash.js.map +0 -1
- package/dist/vector-stores/index.d.ts +0 -96
- package/dist/vector-stores/index.d.ts.map +0 -1
- package/dist/vector-stores/index.js +0 -153
- package/dist/vector-stores/index.js.map +0 -1
- package/dist/vercel-protocol.d.ts +0 -18
- package/dist/vercel-protocol.d.ts.map +0 -1
- package/dist/vercel-protocol.js +0 -75
- package/dist/vercel-protocol.js.map +0 -1
- package/dist/zod-to-json-schema.d.ts +0 -16
- package/dist/zod-to-json-schema.d.ts.map +0 -1
- package/dist/zod-to-json-schema.js +0 -17
- package/dist/zod-to-json-schema.js.map +0 -1
|
@@ -1,270 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Playwright executor for {@link ComputerAction} (#A7 Phase 1).
|
|
3
|
-
*
|
|
4
|
-
* Translates Anthropic's `computer_20250124` action vocabulary into
|
|
5
|
-
* Playwright `Page` calls. Stateless apart from {@link ComputerExecutorState},
|
|
6
|
-
* which the caller threads through so `cursor_position` can answer.
|
|
7
|
-
*
|
|
8
|
-
* # Errors
|
|
9
|
-
*
|
|
10
|
-
* Playwright throws on failures (selector misses, timeouts, navigation
|
|
11
|
-
* during action). The executor catches and returns
|
|
12
|
-
* `{ type: 'error', text: <message> }` so the agent loop can hand the
|
|
13
|
-
* failure back to the model as a tool-result with `is_error: true`. The
|
|
14
|
-
* model decides whether to retry, recover, or give up — failing the
|
|
15
|
-
* whole agent run on a single missed click is too brittle.
|
|
16
|
-
*
|
|
17
|
-
* # Modifier keys & key chords
|
|
18
|
-
*
|
|
19
|
-
* Anthropic uses xdotool naming (`ctrl`, `cmd`, `Return`); Playwright
|
|
20
|
-
* uses its own (`Control`, `Meta`, `Enter`). {@link normalizeKey} maps
|
|
21
|
-
* the common ones; unmapped keys pass through verbatim and Playwright
|
|
22
|
-
* either accepts them (single chars, function keys) or throws — which
|
|
23
|
-
* surfaces as the standard error path above.
|
|
24
|
-
*
|
|
25
|
-
* # Scroll units
|
|
26
|
-
*
|
|
27
|
-
* Anthropic's `scroll_amount` is in mouse-wheel "clicks". A typical
|
|
28
|
-
* desktop wheel click is ~100px; the executor multiplies through
|
|
29
|
-
* {@link SCROLL_PIXELS_PER_CLICK}. Tunable later if customer feedback
|
|
30
|
-
* shows drift.
|
|
31
|
-
*/
|
|
32
|
-
/**
|
|
33
|
-
* Pixels per `scroll_amount` unit. Roughly matches a desktop mouse
|
|
34
|
-
* wheel click on most platforms.
|
|
35
|
-
*/
|
|
36
|
-
export const SCROLL_PIXELS_PER_CLICK = 100;
|
|
37
|
-
/**
|
|
38
|
-
* Number of intermediate move steps Playwright interpolates when the
|
|
39
|
-
* executor moves the mouse to a target. Higher = smoother (more hover
|
|
40
|
-
* events fire), but slower. 5 is a reasonable default — instant enough
|
|
41
|
-
* for most pages, slow enough that hover-driven UI (tooltips, menus)
|
|
42
|
-
* has a chance to react.
|
|
43
|
-
*/
|
|
44
|
-
export const MOUSE_MOVE_STEPS = 5;
|
|
45
|
-
/**
|
|
46
|
-
* Dispatch a single {@link ComputerAction} against a Playwright `Page`.
|
|
47
|
-
*
|
|
48
|
-
* Updates `state.cursor` after every action that targets a coordinate
|
|
49
|
-
* (move / click / drag / scroll). Caller owns `state` and threads it
|
|
50
|
-
* through every call within an agent run.
|
|
51
|
-
*
|
|
52
|
-
* Returns a {@link ComputerActionResult} suitable for forwarding to the
|
|
53
|
-
* model as a tool-result. Never throws — Playwright failures surface as
|
|
54
|
-
* `{ type: 'error', text }`.
|
|
55
|
-
*/
|
|
56
|
-
export async function executeComputerAction(page, action, state) {
|
|
57
|
-
try {
|
|
58
|
-
switch (action.action) {
|
|
59
|
-
case 'screenshot': {
|
|
60
|
-
const data = await page.screenshot({ type: 'png' });
|
|
61
|
-
return { type: 'image', media_type: 'image/png', data };
|
|
62
|
-
}
|
|
63
|
-
case 'cursor_position': {
|
|
64
|
-
return { type: 'text', text: `X=${state.cursor.x}, Y=${state.cursor.y}` };
|
|
65
|
-
}
|
|
66
|
-
case 'wait': {
|
|
67
|
-
await sleep(action.duration * 1000);
|
|
68
|
-
return { type: 'text', text: `waited ${action.duration}s` };
|
|
69
|
-
}
|
|
70
|
-
case 'mouse_move': {
|
|
71
|
-
const [x, y] = action.coordinate;
|
|
72
|
-
await page.mouse.move(x, y, { steps: MOUSE_MOVE_STEPS });
|
|
73
|
-
state.cursor = { x, y };
|
|
74
|
-
return { type: 'text', text: `moved to (${x}, ${y})` };
|
|
75
|
-
}
|
|
76
|
-
case 'left_click':
|
|
77
|
-
case 'right_click':
|
|
78
|
-
case 'middle_click': {
|
|
79
|
-
const button = action.action === 'left_click'
|
|
80
|
-
? 'left'
|
|
81
|
-
: action.action === 'right_click'
|
|
82
|
-
? 'right'
|
|
83
|
-
: 'middle';
|
|
84
|
-
const [x, y] = action.coordinate;
|
|
85
|
-
const modifiers = parseModifiers(action.text);
|
|
86
|
-
await pressDown(page, modifiers);
|
|
87
|
-
try {
|
|
88
|
-
await page.mouse.move(x, y, { steps: MOUSE_MOVE_STEPS });
|
|
89
|
-
await page.mouse.click(x, y, { button });
|
|
90
|
-
}
|
|
91
|
-
finally {
|
|
92
|
-
await pressUp(page, modifiers);
|
|
93
|
-
}
|
|
94
|
-
state.cursor = { x, y };
|
|
95
|
-
return { type: 'text', text: `${button}-clicked at (${x}, ${y})` };
|
|
96
|
-
}
|
|
97
|
-
case 'double_click':
|
|
98
|
-
case 'triple_click': {
|
|
99
|
-
const clickCount = action.action === 'double_click' ? 2 : 3;
|
|
100
|
-
const label = action.action === 'double_click' ? 'double' : 'triple';
|
|
101
|
-
const [x, y] = action.coordinate;
|
|
102
|
-
const modifiers = parseModifiers(action.text);
|
|
103
|
-
await pressDown(page, modifiers);
|
|
104
|
-
try {
|
|
105
|
-
await page.mouse.move(x, y, { steps: MOUSE_MOVE_STEPS });
|
|
106
|
-
await page.mouse.click(x, y, { button: 'left', clickCount });
|
|
107
|
-
}
|
|
108
|
-
finally {
|
|
109
|
-
await pressUp(page, modifiers);
|
|
110
|
-
}
|
|
111
|
-
state.cursor = { x, y };
|
|
112
|
-
return { type: 'text', text: `${label}-clicked at (${x}, ${y})` };
|
|
113
|
-
}
|
|
114
|
-
case 'left_mouse_down': {
|
|
115
|
-
if (action.coordinate) {
|
|
116
|
-
const [x, y] = action.coordinate;
|
|
117
|
-
await page.mouse.move(x, y, { steps: MOUSE_MOVE_STEPS });
|
|
118
|
-
state.cursor = { x, y };
|
|
119
|
-
}
|
|
120
|
-
await page.mouse.down({ button: 'left' });
|
|
121
|
-
return { type: 'text', text: `mouse down at (${state.cursor.x}, ${state.cursor.y})` };
|
|
122
|
-
}
|
|
123
|
-
case 'left_mouse_up': {
|
|
124
|
-
if (action.coordinate) {
|
|
125
|
-
const [x, y] = action.coordinate;
|
|
126
|
-
await page.mouse.move(x, y, { steps: MOUSE_MOVE_STEPS });
|
|
127
|
-
state.cursor = { x, y };
|
|
128
|
-
}
|
|
129
|
-
await page.mouse.up({ button: 'left' });
|
|
130
|
-
return { type: 'text', text: `mouse up at (${state.cursor.x}, ${state.cursor.y})` };
|
|
131
|
-
}
|
|
132
|
-
case 'type': {
|
|
133
|
-
await page.keyboard.type(action.text);
|
|
134
|
-
return { type: 'text', text: `typed ${JSON.stringify(action.text)}` };
|
|
135
|
-
}
|
|
136
|
-
case 'key': {
|
|
137
|
-
await page.keyboard.press(normalizeChord(action.text));
|
|
138
|
-
return { type: 'text', text: `pressed ${action.text}` };
|
|
139
|
-
}
|
|
140
|
-
case 'hold_key': {
|
|
141
|
-
const key = normalizeKey(action.text);
|
|
142
|
-
await page.keyboard.down(key);
|
|
143
|
-
try {
|
|
144
|
-
await sleep(action.duration * 1000);
|
|
145
|
-
}
|
|
146
|
-
finally {
|
|
147
|
-
await page.keyboard.up(key);
|
|
148
|
-
}
|
|
149
|
-
return { type: 'text', text: `held ${action.text} for ${action.duration}s` };
|
|
150
|
-
}
|
|
151
|
-
case 'scroll': {
|
|
152
|
-
const [x, y] = action.coordinate;
|
|
153
|
-
const modifiers = parseModifiers(action.text);
|
|
154
|
-
await pressDown(page, modifiers);
|
|
155
|
-
try {
|
|
156
|
-
await page.mouse.move(x, y, { steps: MOUSE_MOVE_STEPS });
|
|
157
|
-
const px = action.scroll_amount * SCROLL_PIXELS_PER_CLICK;
|
|
158
|
-
let dx = 0;
|
|
159
|
-
let dy = 0;
|
|
160
|
-
switch (action.scroll_direction) {
|
|
161
|
-
case 'up':
|
|
162
|
-
dy = -px;
|
|
163
|
-
break;
|
|
164
|
-
case 'down':
|
|
165
|
-
dy = px;
|
|
166
|
-
break;
|
|
167
|
-
case 'left':
|
|
168
|
-
dx = -px;
|
|
169
|
-
break;
|
|
170
|
-
case 'right':
|
|
171
|
-
dx = px;
|
|
172
|
-
break;
|
|
173
|
-
}
|
|
174
|
-
await page.mouse.wheel(dx, dy);
|
|
175
|
-
}
|
|
176
|
-
finally {
|
|
177
|
-
await pressUp(page, modifiers);
|
|
178
|
-
}
|
|
179
|
-
state.cursor = { x, y };
|
|
180
|
-
return {
|
|
181
|
-
type: 'text',
|
|
182
|
-
text: `scrolled ${action.scroll_direction} ${action.scroll_amount} clicks at (${x}, ${y})`,
|
|
183
|
-
};
|
|
184
|
-
}
|
|
185
|
-
default: {
|
|
186
|
-
// Exhaustiveness guard — TS errors if a future action variant is
|
|
187
|
-
// added to ComputerAction without a handler here.
|
|
188
|
-
const _exhaustive = action;
|
|
189
|
-
throw new Error(`Unknown computer action: ${JSON.stringify(_exhaustive)}`);
|
|
190
|
-
}
|
|
191
|
-
}
|
|
192
|
-
}
|
|
193
|
-
catch (err) {
|
|
194
|
-
const text = err instanceof Error ? err.message : String(err);
|
|
195
|
-
return { type: 'error', text };
|
|
196
|
-
}
|
|
197
|
-
}
|
|
198
|
-
// ─── Helpers ──────────────────────────────────────────────
|
|
199
|
-
function sleep(ms) {
|
|
200
|
-
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
201
|
-
}
|
|
202
|
-
/**
|
|
203
|
-
* Parse Anthropic's `+`-separated modifier text into Playwright key
|
|
204
|
-
* names. Empty / undefined input → empty array (no modifiers).
|
|
205
|
-
*/
|
|
206
|
-
export function parseModifiers(text) {
|
|
207
|
-
if (!text)
|
|
208
|
-
return [];
|
|
209
|
-
return text
|
|
210
|
-
.split('+')
|
|
211
|
-
.map((s) => normalizeKey(s.trim()))
|
|
212
|
-
.filter(Boolean);
|
|
213
|
-
}
|
|
214
|
-
/**
|
|
215
|
-
* Map Anthropic / xdotool key names to Playwright key names.
|
|
216
|
-
* Unmapped keys (single chars, F-keys, anything we haven't covered)
|
|
217
|
-
* pass through verbatim — Playwright either accepts them or throws,
|
|
218
|
-
* which is the right failure mode.
|
|
219
|
-
*/
|
|
220
|
-
export function normalizeKey(key) {
|
|
221
|
-
switch (key.toLowerCase()) {
|
|
222
|
-
case 'ctrl':
|
|
223
|
-
case 'control': return 'Control';
|
|
224
|
-
case 'shift': return 'Shift';
|
|
225
|
-
case 'alt': return 'Alt';
|
|
226
|
-
case 'cmd':
|
|
227
|
-
case 'super':
|
|
228
|
-
case 'meta': return 'Meta';
|
|
229
|
-
case 'return':
|
|
230
|
-
case 'enter': return 'Enter';
|
|
231
|
-
case 'tab': return 'Tab';
|
|
232
|
-
case 'space': return 'Space';
|
|
233
|
-
case 'escape':
|
|
234
|
-
case 'esc': return 'Escape';
|
|
235
|
-
case 'backspace': return 'Backspace';
|
|
236
|
-
case 'delete':
|
|
237
|
-
case 'del': return 'Delete';
|
|
238
|
-
case 'up': return 'ArrowUp';
|
|
239
|
-
case 'down': return 'ArrowDown';
|
|
240
|
-
case 'left': return 'ArrowLeft';
|
|
241
|
-
case 'right': return 'ArrowRight';
|
|
242
|
-
case 'page_up':
|
|
243
|
-
case 'pageup': return 'PageUp';
|
|
244
|
-
case 'page_down':
|
|
245
|
-
case 'pagedown': return 'PageDown';
|
|
246
|
-
case 'home': return 'Home';
|
|
247
|
-
case 'end': return 'End';
|
|
248
|
-
default: return key;
|
|
249
|
-
}
|
|
250
|
-
}
|
|
251
|
-
/**
|
|
252
|
-
* Normalize a key chord (`+`-separated). Playwright's
|
|
253
|
-
* `keyboard.press()` parses chords natively, so we just normalize each
|
|
254
|
-
* segment and rejoin.
|
|
255
|
-
*/
|
|
256
|
-
export function normalizeChord(chord) {
|
|
257
|
-
return chord
|
|
258
|
-
.split('+')
|
|
259
|
-
.map((s) => normalizeKey(s.trim()))
|
|
260
|
-
.join('+');
|
|
261
|
-
}
|
|
262
|
-
async function pressDown(page, keys) {
|
|
263
|
-
for (const k of keys)
|
|
264
|
-
await page.keyboard.down(k);
|
|
265
|
-
}
|
|
266
|
-
async function pressUp(page, keys) {
|
|
267
|
-
for (const k of [...keys].reverse())
|
|
268
|
-
await page.keyboard.up(k);
|
|
269
|
-
}
|
|
270
|
-
//# sourceMappingURL=playwright.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"playwright.js","sourceRoot":"","sources":["../../src/computer-use/playwright.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AASH;;;GAGG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,GAAG,CAAA;AAE1C;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAA;AAEjC;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,IAAc,EACd,MAAsB,EACtB,KAA4B;IAE5B,IAAI,CAAC;QACH,QAAQ,MAAM,CAAC,MAAM,EAAE,CAAC;YACtB,KAAK,YAAY,CAAC,CAAC,CAAC;gBAClB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAA;gBACnD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,IAAI,EAAE,CAAA;YACzD,CAAC;YAED,KAAK,iBAAiB,CAAC,CAAC,CAAC;gBACvB,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,KAAK,CAAC,MAAM,CAAC,CAAC,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,CAAA;YAC3E,CAAC;YAED,KAAK,MAAM,CAAC,CAAC,CAAC;gBACZ,MAAM,KAAK,CAAC,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAA;gBACnC,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,MAAM,CAAC,QAAQ,GAAG,EAAE,CAAA;YAC7D,CAAC;YAED,KAAK,YAAY,CAAC,CAAC,CAAC;gBAClB,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,UAAU,CAAA;gBAChC,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,KAAK,EAAE,gBAAgB,EAAE,CAAC,CAAA;gBACxD,KAAK,CAAC,MAAM,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,CAAA;gBACvB,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,CAAC,KAAK,CAAC,GAAG,EAAE,CAAA;YACxD,CAAC;YAED,KAAK,YAAY,CAAC;YAClB,KAAK,aAAa,CAAC;YACnB,KAAK,cAAc,CAAC,CAAC,CAAC;gBACpB,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,KAAK,YAAY;oBAC3C,CAAC,CAAC,MAAM;oBACR,CAAC,CAAC,MAAM,CAAC,MAAM,KAAK,aAAa;wBACjC,CAAC,CAAC,OAAO;wBACT,CAAC,CAAC,QAAQ,CAAA;gBACZ,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,UAAU,CAAA;gBAChC,MAAM,SAAS,GAAG,cAAc,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;gBAC7C,MAAM,SAAS,CAAC,IAAI,EAAE,SAAS,CAAC,CAAA;gBAChC,IAAI,CAAC;oBACH,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,KAAK,EAAE,gBAAgB,EAAE,CAAC,CAAA;oBACxD,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,CAAA;gBAC1C,CAAC;wBAAS,CAAC;oBACT,MAAM,OAAO,CAAC,IAAI,EAAE,SAAS,CAAC,CAAA;gBAChC,CAAC;gBACD,KAAK,CAAC,MAAM,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,CAAA;gBACvB,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,gBAAgB,CAAC,KAAK,CAAC,GAAG,EAAE,CAAA;YACpE,CAAC;YAED,KAAK,cAAc,CAAC;YACpB,KAAK,cAAc,CAAC,CAAC,CAAC;gBACpB,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,KAAK,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;gBAC3D,MAAM,KAAK,GAAO,MAAM,CAAC,MAAM,KAAK,cAAc,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAA;gBACxE,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,UAAU,CAAA;gBAChC,MAAM,SAAS,GAAG,cAAc,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;gBAC7C,MAAM,SAAS,CAAC,IAAI,EAAE,SAAS,CAAC,CAAA;gBAChC,IAAI,CAAC;oBACH,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,KAAK,EAAE,gBAAgB,EAAE,CAAC,CAAA;oBACxD,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAA;gBAC9D,CAAC;wBAAS,CAAC;oBACT,MAAM,OAAO,CAAC,IAAI,EAAE,SAAS,CAAC,CAAA;gBAChC,CAAC;gBACD,KAAK,CAAC,MAAM,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,CAAA;gBACvB,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,KAAK,gBAAgB,CAAC,KAAK,CAAC,GAAG,EAAE,CAAA;YACnE,CAAC;YAED,KAAK,iBAAiB,CAAC,CAAC,CAAC;gBACvB,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;oBACtB,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,UAAU,CAAA;oBAChC,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,KAAK,EAAE,gBAAgB,EAAE,CAAC,CAAA;oBACxD,KAAK,CAAC,MAAM,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,CAAA;gBACzB,CAAC;gBACD,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAA;gBACzC,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,kBAAkB,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAA;YACvF,CAAC;YAED,KAAK,eAAe,CAAC,CAAC,CAAC;gBACrB,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;oBACtB,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,UAAU,CAAA;oBAChC,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,KAAK,EAAE,gBAAgB,EAAE,CAAC,CAAA;oBACxD,KAAK,CAAC,MAAM,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,CAAA;gBACzB,CAAC;gBACD,MAAM,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAA;gBACvC,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAA;YACrF,CAAC;YAED,KAAK,MAAM,CAAC,CAAC,CAAC;gBACZ,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;gBACrC,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,CAAA;YACvE,CAAC;YAED,KAAK,KAAK,CAAC,CAAC,CAAC;gBACX,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,cAAc,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAA;gBACtD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,MAAM,CAAC,IAAI,EAAE,EAAE,CAAA;YACzD,CAAC;YAED,KAAK,UAAU,CAAC,CAAC,CAAC;gBAChB,MAAM,GAAG,GAAG,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;gBACrC,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;gBAC7B,IAAI,CAAC;oBACH,MAAM,KAAK,CAAC,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAA;gBACrC,CAAC;wBAAS,CAAC;oBACT,MAAM,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,GAAG,CAAC,CAAA;gBAC7B,CAAC;gBACD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,MAAM,CAAC,IAAI,QAAQ,MAAM,CAAC,QAAQ,GAAG,EAAE,CAAA;YAC9E,CAAC;YAED,KAAK,QAAQ,CAAC,CAAC,CAAC;gBACd,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,UAAU,CAAA;gBAChC,MAAM,SAAS,GAAG,cAAc,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;gBAC7C,MAAM,SAAS,CAAC,IAAI,EAAE,SAAS,CAAC,CAAA;gBAChC,IAAI,CAAC;oBACH,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,KAAK,EAAE,gBAAgB,EAAE,CAAC,CAAA;oBACxD,MAAM,EAAE,GAAG,MAAM,CAAC,aAAa,GAAG,uBAAuB,CAAA;oBACzD,IAAI,EAAE,GAAG,CAAC,CAAA;oBACV,IAAI,EAAE,GAAG,CAAC,CAAA;oBACV,QAAQ,MAAM,CAAC,gBAAgB,EAAE,CAAC;wBAChC,KAAK,IAAI;4BAAK,EAAE,GAAG,CAAC,EAAE,CAAC;4BAAC,MAAK;wBAC7B,KAAK,MAAM;4BAAG,EAAE,GAAG,EAAE,CAAC;4BAAE,MAAK;wBAC7B,KAAK,MAAM;4BAAG,EAAE,GAAG,CAAC,EAAE,CAAC;4BAAC,MAAK;wBAC7B,KAAK,OAAO;4BAAE,EAAE,GAAG,EAAE,CAAC;4BAAE,MAAK;oBAC/B,CAAC;oBACD,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,CAAC,CAAA;gBAChC,CAAC;wBAAS,CAAC;oBACT,MAAM,OAAO,CAAC,IAAI,EAAE,SAAS,CAAC,CAAA;gBAChC,CAAC;gBACD,KAAK,CAAC,MAAM,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,CAAA;gBACvB,OAAO;oBACL,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,YAAY,MAAM,CAAC,gBAAgB,IAAI,MAAM,CAAC,aAAa,eAAe,CAAC,KAAK,CAAC,GAAG;iBAC3F,CAAA;YACH,CAAC;YAED,OAAO,CAAC,CAAC,CAAC;gBACR,iEAAiE;gBACjE,kDAAkD;gBAClD,MAAM,WAAW,GAAU,MAAM,CAAA;gBACjC,MAAM,IAAI,KAAK,CAAC,4BAA4B,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,EAAE,CAAC,CAAA;YAC5E,CAAC;QACH,CAAC;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,IAAI,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;QAC7D,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,CAAA;IAChC,CAAC;AACH,CAAC;AAED,6DAA6D;AAE7D,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAA;AAC1D,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,IAAwB;IACrD,IAAI,CAAC,IAAI;QAAE,OAAO,EAAE,CAAA;IACpB,OAAO,IAAI;SACR,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;SAClC,MAAM,CAAC,OAAO,CAAC,CAAA;AACpB,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,YAAY,CAAC,GAAW;IACtC,QAAQ,GAAG,CAAC,WAAW,EAAE,EAAE,CAAC;QAC1B,KAAK,MAAM,CAAC;QACZ,KAAK,SAAS,CAAC,CAAG,OAAO,SAAS,CAAA;QAClC,KAAK,OAAO,CAAC,CAAK,OAAO,OAAO,CAAA;QAChC,KAAK,KAAK,CAAC,CAAO,OAAO,KAAK,CAAA;QAC9B,KAAK,KAAK,CAAC;QACX,KAAK,OAAO,CAAC;QACb,KAAK,MAAM,CAAC,CAAM,OAAO,MAAM,CAAA;QAC/B,KAAK,QAAQ,CAAC;QACd,KAAK,OAAO,CAAC,CAAK,OAAO,OAAO,CAAA;QAChC,KAAK,KAAK,CAAC,CAAO,OAAO,KAAK,CAAA;QAC9B,KAAK,OAAO,CAAC,CAAK,OAAO,OAAO,CAAA;QAChC,KAAK,QAAQ,CAAC;QACd,KAAK,KAAK,CAAC,CAAO,OAAO,QAAQ,CAAA;QACjC,KAAK,WAAW,CAAC,CAAC,OAAO,WAAW,CAAA;QACpC,KAAK,QAAQ,CAAC;QACd,KAAK,KAAK,CAAC,CAAO,OAAO,QAAQ,CAAA;QACjC,KAAK,IAAI,CAAC,CAAQ,OAAO,SAAS,CAAA;QAClC,KAAK,MAAM,CAAC,CAAM,OAAO,WAAW,CAAA;QACpC,KAAK,MAAM,CAAC,CAAM,OAAO,WAAW,CAAA;QACpC,KAAK,OAAO,CAAC,CAAK,OAAO,YAAY,CAAA;QACrC,KAAK,SAAS,CAAC;QACf,KAAK,QAAQ,CAAC,CAAI,OAAO,QAAQ,CAAA;QACjC,KAAK,WAAW,CAAC;QACjB,KAAK,UAAU,CAAC,CAAE,OAAO,UAAU,CAAA;QACnC,KAAK,MAAM,CAAC,CAAM,OAAO,MAAM,CAAA;QAC/B,KAAK,KAAK,CAAC,CAAO,OAAO,KAAK,CAAA;QAC9B,OAAO,CAAC,CAAU,OAAO,GAAG,CAAA;IAC9B,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAAC,KAAa;IAC1C,OAAO,KAAK;SACT,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;SAClC,IAAI,CAAC,GAAG,CAAC,CAAA;AACd,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,IAAc,EAAE,IAAc;IACrD,KAAK,MAAM,CAAC,IAAI,IAAI;QAAE,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;AACnD,CAAC;AAED,KAAK,UAAU,OAAO,CAAC,IAAc,EAAE,IAAc;IACnD,KAAK,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,EAAE;QAAE,MAAM,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;AAChE,CAAC"}
|
|
@@ -1,154 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* `computerUseTool({ page })` — the agent-tool factory for #A7 Phase 2.
|
|
3
|
-
*
|
|
4
|
-
* Wraps the phase-1 {@link executeComputerAction} executor as a tool the
|
|
5
|
-
* agent loop can invoke. The tool is tagged so the Anthropic provider
|
|
6
|
-
* adapter substitutes the standard function-call schema with Anthropic's
|
|
7
|
-
* native `computer_20250124` tool block at the API level — Claude is
|
|
8
|
-
* fine-tuned on that exact tool, so quality is dramatically better than
|
|
9
|
-
* a generic function-call wrapper.
|
|
10
|
-
*
|
|
11
|
-
* # Anthropic-only in v1
|
|
12
|
-
*
|
|
13
|
-
* Pass `model` to fail loud at agent-construction time when the agent's
|
|
14
|
-
* model isn't Anthropic-family — see {@link ComputerUseProviderError}.
|
|
15
|
-
* Without `model`, validation is deferred (the Anthropic adapter is the
|
|
16
|
-
* only one that recognizes the provider hint, so non-Anthropic models
|
|
17
|
-
* silently see a no-arg generic tool — degraded but not catastrophic).
|
|
18
|
-
*
|
|
19
|
-
* # Wiring
|
|
20
|
-
*
|
|
21
|
-
* ```ts
|
|
22
|
-
* import { Agent } from '@rudderjs/ai'
|
|
23
|
-
* import { computerUseTool } from '@rudderjs/ai/computer-use'
|
|
24
|
-
* import { chromium } from 'playwright'
|
|
25
|
-
*
|
|
26
|
-
* const browser = await chromium.launch()
|
|
27
|
-
* const page = await browser.newPage()
|
|
28
|
-
* await page.setViewportSize({ width: 1280, height: 800 })
|
|
29
|
-
*
|
|
30
|
-
* class BrowserAgent extends Agent {
|
|
31
|
-
* model() { return 'anthropic/claude-opus-4-7' }
|
|
32
|
-
*
|
|
33
|
-
* tools() {
|
|
34
|
-
* return [
|
|
35
|
-
* computerUseTool({
|
|
36
|
-
* page,
|
|
37
|
-
* viewport: { width: 1280, height: 800 },
|
|
38
|
-
* model: this.model(), // upfront provider check
|
|
39
|
-
* }),
|
|
40
|
-
* ]
|
|
41
|
-
* }
|
|
42
|
-
* }
|
|
43
|
-
* ```
|
|
44
|
-
*
|
|
45
|
-
* # State
|
|
46
|
-
*
|
|
47
|
-
* Each `computerUseTool({...})` call captures a fresh
|
|
48
|
-
* {@link ComputerExecutorState} in its closure. Passing the same tool
|
|
49
|
-
* instance through multiple agent runs SHARES cursor state across them
|
|
50
|
-
* — usually fine, but call the factory inside `tools()` (which Agent
|
|
51
|
-
* runs per request) for clean per-run state.
|
|
52
|
-
*
|
|
53
|
-
* The same closure carries the action counter for {@link maxActions}.
|
|
54
|
-
*
|
|
55
|
-
* # Image results
|
|
56
|
-
*
|
|
57
|
-
* `screenshot` actions return PNG bytes. The tool's execute base64-
|
|
58
|
-
* encodes them and returns a `ContentPart[]` array with one image
|
|
59
|
-
* block — the Anthropic adapter's `toAnthropicMessages` handles array
|
|
60
|
-
* tool-message content directly (a generic enhancement, not
|
|
61
|
-
* computer-use-specific). Other providers see a JSON-stringified
|
|
62
|
-
* fallback; in practice they never get here because the tool throws at
|
|
63
|
-
* construction when bound to a non-Anthropic model.
|
|
64
|
-
*/
|
|
65
|
-
import type { ContentPart, Tool, ToolCallContext, ToolDefinitionOptions, ToolDefinitionSchema } from '../types.js';
|
|
66
|
-
import { ComputerUseLimitError, ComputerUseProviderError, isAnthropicLikeModel } from './errors.js';
|
|
67
|
-
import { type ComputerAction, type ComputerExecutorState, type PageLike } from './actions.js';
|
|
68
|
-
/**
|
|
69
|
-
* Symbol-tagged marker identifying a computer-use tool. Looked up via
|
|
70
|
-
* `Symbol.for(...)` so cross-bundle / cross-realm checks succeed even
|
|
71
|
-
* when `@rudderjs/ai` is loaded twice (rare, but possible in monorepo +
|
|
72
|
-
* linked setups). Mirrors the `HANDOFF_MARKER` pattern.
|
|
73
|
-
*/
|
|
74
|
-
export declare const COMPUTER_USE_MARKER: unique symbol;
|
|
75
|
-
/**
|
|
76
|
-
* The fixed tool name. Anthropic's native `computer_20250124` tool
|
|
77
|
-
* expects calls to land on a tool literally named `computer` — the
|
|
78
|
-
* model is trained on that name. Apps don't override it.
|
|
79
|
-
*/
|
|
80
|
-
export declare const COMPUTER_USE_TOOL_NAME = "computer";
|
|
81
|
-
/** Options for {@link computerUseTool}. */
|
|
82
|
-
export interface ComputerUseToolOptions {
|
|
83
|
-
/**
|
|
84
|
-
* Playwright `Page` (or any object structurally matching {@link PageLike}).
|
|
85
|
-
* Caller owns the lifecycle — launch, set viewport, navigate, close.
|
|
86
|
-
*/
|
|
87
|
-
page: PageLike;
|
|
88
|
-
/**
|
|
89
|
-
* Display dimensions reported to the model in the native
|
|
90
|
-
* `computer_20250124` block. Defaults to `1280×800` (Anthropic's
|
|
91
|
-
* recommended training-distribution size). Must match what
|
|
92
|
-
* `page.setViewportSize(...)` was called with — Claude grounds clicks
|
|
93
|
-
* in this coordinate space.
|
|
94
|
-
*/
|
|
95
|
-
viewport?: {
|
|
96
|
-
width: number;
|
|
97
|
-
height: number;
|
|
98
|
-
};
|
|
99
|
-
/**
|
|
100
|
-
* Optional agent model id. When provided, the factory fails loud at
|
|
101
|
-
* construction time if the model isn't Anthropic-family — see
|
|
102
|
-
* {@link ComputerUseProviderError}. Pass `this.model()` from inside
|
|
103
|
-
* `Agent.tools()` to get the check.
|
|
104
|
-
*/
|
|
105
|
-
model?: string;
|
|
106
|
-
/**
|
|
107
|
-
* Per-action approval gate. `true` (default) routes every action
|
|
108
|
-
* through the framework's approval middleware before execution.
|
|
109
|
-
* `false` opts out entirely. Function form decides per-action — useful
|
|
110
|
-
* for letting cheap actions (`screenshot`, `mouse_move`) run
|
|
111
|
-
* unattended while gating destructive ones.
|
|
112
|
-
*
|
|
113
|
-
* Wired via {@link ToolDefinitionOptions.requireApproval} — same
|
|
114
|
-
* channel the rest of `@rudderjs/ai`'s approval-resume machinery uses.
|
|
115
|
-
*/
|
|
116
|
-
needsApproval?: boolean | ((action: ComputerAction) => boolean);
|
|
117
|
-
/**
|
|
118
|
-
* Maximum number of actions per agent run before
|
|
119
|
-
* {@link ComputerUseLimitError} is thrown. Default `50`. Bounds
|
|
120
|
-
* runaway loops where the model keeps trying the same broken UI step.
|
|
121
|
-
*/
|
|
122
|
-
maxActions?: number;
|
|
123
|
-
/**
|
|
124
|
-
* Override the per-run cursor-tracking state. Rarely needed — the
|
|
125
|
-
* factory creates a fresh state by default. Provide one if you want
|
|
126
|
-
* to seed the cursor (e.g. resuming a paused session).
|
|
127
|
-
*/
|
|
128
|
-
state?: ComputerExecutorState;
|
|
129
|
-
}
|
|
130
|
-
/**
|
|
131
|
-
* The tool returned by {@link computerUseTool}. Implements the
|
|
132
|
-
* {@link Tool} interface with `execute` (so the agent loop runs it
|
|
133
|
-
* directly), and carries the {@link COMPUTER_USE_MARKER} so adapters
|
|
134
|
-
* and observers can detect it without coupling to a class.
|
|
135
|
-
*/
|
|
136
|
-
export interface ComputerUseTool extends Tool<ComputerAction, ContentPart[] | string> {
|
|
137
|
-
readonly [COMPUTER_USE_MARKER]: true;
|
|
138
|
-
readonly definition: ToolDefinitionOptions;
|
|
139
|
-
execute(input: ComputerAction, ctx?: ToolCallContext): Promise<ContentPart[] | string>;
|
|
140
|
-
toSchema(): ToolDefinitionSchema;
|
|
141
|
-
}
|
|
142
|
-
/**
|
|
143
|
-
* Build the agent tool. See module JSDoc for usage.
|
|
144
|
-
*/
|
|
145
|
-
export declare function computerUseTool(opts: ComputerUseToolOptions): ComputerUseTool;
|
|
146
|
-
/**
|
|
147
|
-
* Structural typeguard. Mirrors {@link isHandoffTool} — handoff /
|
|
148
|
-
* computer-use tools are plain objects tagged with their respective
|
|
149
|
-
* `Symbol.for(...)` markers, so the loop and adapters can detect them
|
|
150
|
-
* without coupling to a class hierarchy.
|
|
151
|
-
*/
|
|
152
|
-
export declare function isComputerUseTool(t: unknown): t is ComputerUseTool;
|
|
153
|
-
export { ComputerUseLimitError, ComputerUseProviderError, isAnthropicLikeModel };
|
|
154
|
-
//# sourceMappingURL=tool.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"tool.d.ts","sourceRoot":"","sources":["../../src/computer-use/tool.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+DG;AAIH,OAAO,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE,eAAe,EAAE,qBAAqB,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAA;AAElH,OAAO,EACL,qBAAqB,EACrB,wBAAwB,EACxB,oBAAoB,EACrB,MAAM,aAAa,CAAA;AACpB,OAAO,EAEL,KAAK,cAAc,EAEnB,KAAK,qBAAqB,EAC1B,KAAK,QAAQ,EACd,MAAM,cAAc,CAAA;AAGrB;;;;;GAKG;AACH,eAAO,MAAM,mBAAmB,EAAE,OAAO,MAA+C,CAAA;AAExF;;;;GAIG;AACH,eAAO,MAAM,sBAAsB,aAAa,CAAA;AAMhD,2CAA2C;AAC3C,MAAM,WAAW,sBAAsB;IACrC;;;OAGG;IACH,IAAI,EAAE,QAAQ,CAAA;IACd;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAA;IAC5C;;;;;OAKG;IACH,KAAK,CAAC,EAAE,MAAM,CAAA;IACd;;;;;;;;;OASG;IACH,aAAa,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,MAAM,EAAE,cAAc,KAAK,OAAO,CAAC,CAAA;IAC/D;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB;;;;OAIG;IACH,KAAK,CAAC,EAAE,qBAAqB,CAAA;CAC9B;AAED;;;;;GAKG;AACH,MAAM,WAAW,eAAgB,SAAQ,IAAI,CAAC,cAAc,EAAE,WAAW,EAAE,GAAG,MAAM,CAAC;IACnF,QAAQ,CAAC,CAAC,mBAAmB,CAAC,EAAE,IAAI,CAAA;IACpC,QAAQ,CAAC,UAAU,EAAE,qBAAqB,CAAA;IAC1C,OAAO,CAAC,KAAK,EAAE,cAAc,EAAE,GAAG,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,WAAW,EAAE,GAAG,MAAM,CAAC,CAAA;IACtF,QAAQ,IAAI,oBAAoB,CAAA;CACjC;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,sBAAsB,GAAG,eAAe,CAsE7E;AAiDD;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,IAAI,eAAe,CAIlE;AAID,OAAO,EAAE,qBAAqB,EAAE,wBAAwB,EAAE,oBAAoB,EAAE,CAAA"}
|
|
@@ -1,210 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* `computerUseTool({ page })` — the agent-tool factory for #A7 Phase 2.
|
|
3
|
-
*
|
|
4
|
-
* Wraps the phase-1 {@link executeComputerAction} executor as a tool the
|
|
5
|
-
* agent loop can invoke. The tool is tagged so the Anthropic provider
|
|
6
|
-
* adapter substitutes the standard function-call schema with Anthropic's
|
|
7
|
-
* native `computer_20250124` tool block at the API level — Claude is
|
|
8
|
-
* fine-tuned on that exact tool, so quality is dramatically better than
|
|
9
|
-
* a generic function-call wrapper.
|
|
10
|
-
*
|
|
11
|
-
* # Anthropic-only in v1
|
|
12
|
-
*
|
|
13
|
-
* Pass `model` to fail loud at agent-construction time when the agent's
|
|
14
|
-
* model isn't Anthropic-family — see {@link ComputerUseProviderError}.
|
|
15
|
-
* Without `model`, validation is deferred (the Anthropic adapter is the
|
|
16
|
-
* only one that recognizes the provider hint, so non-Anthropic models
|
|
17
|
-
* silently see a no-arg generic tool — degraded but not catastrophic).
|
|
18
|
-
*
|
|
19
|
-
* # Wiring
|
|
20
|
-
*
|
|
21
|
-
* ```ts
|
|
22
|
-
* import { Agent } from '@rudderjs/ai'
|
|
23
|
-
* import { computerUseTool } from '@rudderjs/ai/computer-use'
|
|
24
|
-
* import { chromium } from 'playwright'
|
|
25
|
-
*
|
|
26
|
-
* const browser = await chromium.launch()
|
|
27
|
-
* const page = await browser.newPage()
|
|
28
|
-
* await page.setViewportSize({ width: 1280, height: 800 })
|
|
29
|
-
*
|
|
30
|
-
* class BrowserAgent extends Agent {
|
|
31
|
-
* model() { return 'anthropic/claude-opus-4-7' }
|
|
32
|
-
*
|
|
33
|
-
* tools() {
|
|
34
|
-
* return [
|
|
35
|
-
* computerUseTool({
|
|
36
|
-
* page,
|
|
37
|
-
* viewport: { width: 1280, height: 800 },
|
|
38
|
-
* model: this.model(), // upfront provider check
|
|
39
|
-
* }),
|
|
40
|
-
* ]
|
|
41
|
-
* }
|
|
42
|
-
* }
|
|
43
|
-
* ```
|
|
44
|
-
*
|
|
45
|
-
* # State
|
|
46
|
-
*
|
|
47
|
-
* Each `computerUseTool({...})` call captures a fresh
|
|
48
|
-
* {@link ComputerExecutorState} in its closure. Passing the same tool
|
|
49
|
-
* instance through multiple agent runs SHARES cursor state across them
|
|
50
|
-
* — usually fine, but call the factory inside `tools()` (which Agent
|
|
51
|
-
* runs per request) for clean per-run state.
|
|
52
|
-
*
|
|
53
|
-
* The same closure carries the action counter for {@link maxActions}.
|
|
54
|
-
*
|
|
55
|
-
* # Image results
|
|
56
|
-
*
|
|
57
|
-
* `screenshot` actions return PNG bytes. The tool's execute base64-
|
|
58
|
-
* encodes them and returns a `ContentPart[]` array with one image
|
|
59
|
-
* block — the Anthropic adapter's `toAnthropicMessages` handles array
|
|
60
|
-
* tool-message content directly (a generic enhancement, not
|
|
61
|
-
* computer-use-specific). Other providers see a JSON-stringified
|
|
62
|
-
* fallback; in practice they never get here because the tool throws at
|
|
63
|
-
* construction when bound to a non-Anthropic model.
|
|
64
|
-
*/
|
|
65
|
-
import { z } from 'zod';
|
|
66
|
-
import { ComputerUseLimitError, ComputerUseProviderError, isAnthropicLikeModel, } from './errors.js';
|
|
67
|
-
import { makeExecutorState, } from './actions.js';
|
|
68
|
-
import { executeComputerAction } from './playwright.js';
|
|
69
|
-
/**
|
|
70
|
-
* Symbol-tagged marker identifying a computer-use tool. Looked up via
|
|
71
|
-
* `Symbol.for(...)` so cross-bundle / cross-realm checks succeed even
|
|
72
|
-
* when `@rudderjs/ai` is loaded twice (rare, but possible in monorepo +
|
|
73
|
-
* linked setups). Mirrors the `HANDOFF_MARKER` pattern.
|
|
74
|
-
*/
|
|
75
|
-
export const COMPUTER_USE_MARKER = Symbol.for('rudderjs.ai.computer-use');
|
|
76
|
-
/**
|
|
77
|
-
* The fixed tool name. Anthropic's native `computer_20250124` tool
|
|
78
|
-
* expects calls to land on a tool literally named `computer` — the
|
|
79
|
-
* model is trained on that name. Apps don't override it.
|
|
80
|
-
*/
|
|
81
|
-
export const COMPUTER_USE_TOOL_NAME = 'computer';
|
|
82
|
-
const DEFAULT_VIEWPORT = { width: 1280, height: 800 };
|
|
83
|
-
const DEFAULT_MAX_ACTIONS = 50;
|
|
84
|
-
const DEFAULT_NEEDS_APPROVAL = true;
|
|
85
|
-
/**
|
|
86
|
-
* Build the agent tool. See module JSDoc for usage.
|
|
87
|
-
*/
|
|
88
|
-
export function computerUseTool(opts) {
|
|
89
|
-
// Upfront provider check — fail loud at agent construction.
|
|
90
|
-
if (opts.model !== undefined && !isAnthropicLikeModel(opts.model)) {
|
|
91
|
-
throw new ComputerUseProviderError(opts.model);
|
|
92
|
-
}
|
|
93
|
-
const viewport = opts.viewport ?? DEFAULT_VIEWPORT;
|
|
94
|
-
const maxActions = opts.maxActions ?? DEFAULT_MAX_ACTIONS;
|
|
95
|
-
const needsApproval = opts.needsApproval ?? DEFAULT_NEEDS_APPROVAL;
|
|
96
|
-
const state = opts.state ?? makeExecutorState();
|
|
97
|
-
const page = opts.page;
|
|
98
|
-
// Per-tool-instance counter. Closure-private so multiple tools (rare)
|
|
99
|
-
// don't collide.
|
|
100
|
-
const counter = { value: 0 };
|
|
101
|
-
// Build the needs-approval shape the framework's tool runner reads.
|
|
102
|
-
// ToolDefinitionOptions.needsApproval is `boolean | (input) => boolean | Promise<boolean>`.
|
|
103
|
-
const needsApprovalForDefinition = typeof needsApproval === 'function'
|
|
104
|
-
? (input) => needsApproval(input)
|
|
105
|
-
: needsApproval;
|
|
106
|
-
const definition = {
|
|
107
|
-
name: COMPUTER_USE_TOOL_NAME,
|
|
108
|
-
description: 'Take screenshots, click, type, and otherwise drive a desktop / browser. ' +
|
|
109
|
-
'Use to interact with on-screen UI you cannot reach via plain HTTP.',
|
|
110
|
-
// Anthropic's native tool block carries an implicit schema (the model
|
|
111
|
-
// is trained on it). The standard `parameters` we emit is irrelevant
|
|
112
|
-
// for Anthropic — the providerHint substitution drops it. `z.any()`
|
|
113
|
-
// is the conservative default for any non-Anthropic serialization
|
|
114
|
-
// that still tries to read the schema.
|
|
115
|
-
inputSchema: z.any(),
|
|
116
|
-
needsApproval: needsApprovalForDefinition,
|
|
117
|
-
// Carried through by `toolToSchema` → `toAnthropicTools` recognizes
|
|
118
|
-
// `providerHint.type === 'computer-use'` and emits the native
|
|
119
|
-
// `computer_20250124` block instead of a generic function-call shape.
|
|
120
|
-
providerHint: {
|
|
121
|
-
type: 'computer-use',
|
|
122
|
-
tool: 'computer_20250124',
|
|
123
|
-
display_width_px: viewport.width,
|
|
124
|
-
display_height_px: viewport.height,
|
|
125
|
-
},
|
|
126
|
-
};
|
|
127
|
-
const tool = {
|
|
128
|
-
[COMPUTER_USE_MARKER]: true,
|
|
129
|
-
definition,
|
|
130
|
-
async execute(input) {
|
|
131
|
-
counter.value++;
|
|
132
|
-
if (counter.value > maxActions) {
|
|
133
|
-
throw new ComputerUseLimitError(maxActions);
|
|
134
|
-
}
|
|
135
|
-
const result = await executeComputerAction(page, input, state);
|
|
136
|
-
return formatActionResult(result);
|
|
137
|
-
},
|
|
138
|
-
toSchema() {
|
|
139
|
-
return {
|
|
140
|
-
name: COMPUTER_USE_TOOL_NAME,
|
|
141
|
-
description: definition.description,
|
|
142
|
-
// Empty object — see comment in `definition.inputSchema` above.
|
|
143
|
-
parameters: { type: 'object', properties: {}, additionalProperties: true },
|
|
144
|
-
providerHint: definition.providerHint,
|
|
145
|
-
};
|
|
146
|
-
},
|
|
147
|
-
};
|
|
148
|
-
return tool;
|
|
149
|
-
}
|
|
150
|
-
/**
|
|
151
|
-
* Convert a {@link ComputerActionResult} into the message-content shape
|
|
152
|
-
* the agent loop stores on the tool message and passes to the provider.
|
|
153
|
-
*
|
|
154
|
-
* - `image` → `ContentPart[]` with one image block. The Anthropic
|
|
155
|
-
* adapter's tool-message handler emits this as Anthropic's `content:
|
|
156
|
-
* [{ type: 'image', source: { ... } }]` shape.
|
|
157
|
-
* - `text` → plain string (current adapter path: `content: <string>`).
|
|
158
|
-
* - `error` → throw. The agent loop's error path wraps the throw into a
|
|
159
|
-
* tool-result with `is_error: true` and the error message — exactly
|
|
160
|
-
* the Anthropic semantics we want for "the action failed; let the
|
|
161
|
-
* model retry."
|
|
162
|
-
*/
|
|
163
|
-
function formatActionResult(result) {
|
|
164
|
-
if (result.type === 'image') {
|
|
165
|
-
const data = bytesToBase64(result.data);
|
|
166
|
-
return [
|
|
167
|
-
{
|
|
168
|
-
type: 'image',
|
|
169
|
-
mimeType: result.media_type,
|
|
170
|
-
data,
|
|
171
|
-
},
|
|
172
|
-
];
|
|
173
|
-
}
|
|
174
|
-
if (result.type === 'text') {
|
|
175
|
-
return result.text;
|
|
176
|
-
}
|
|
177
|
-
// result.type === 'error' — throw so the agent loop wraps as is_error.
|
|
178
|
-
throw new Error(result.text);
|
|
179
|
-
}
|
|
180
|
-
/**
|
|
181
|
-
* Encode raw bytes as a base64 string. Uses `Buffer` when available
|
|
182
|
-
* (Node) and falls back to a synchronous browser-safe path otherwise.
|
|
183
|
-
* Computer-use only runs in Node (Playwright requires it), but the
|
|
184
|
-
* fallback keeps the module importable from runtime-agnostic tests.
|
|
185
|
-
*/
|
|
186
|
-
function bytesToBase64(data) {
|
|
187
|
-
if (typeof Buffer !== 'undefined') {
|
|
188
|
-
return Buffer.from(data).toString('base64');
|
|
189
|
-
}
|
|
190
|
-
// Browser fallback — slow but correct.
|
|
191
|
-
let binary = '';
|
|
192
|
-
for (let i = 0; i < data.length; i++)
|
|
193
|
-
binary += String.fromCharCode(data[i]);
|
|
194
|
-
return btoa(binary);
|
|
195
|
-
}
|
|
196
|
-
/**
|
|
197
|
-
* Structural typeguard. Mirrors {@link isHandoffTool} — handoff /
|
|
198
|
-
* computer-use tools are plain objects tagged with their respective
|
|
199
|
-
* `Symbol.for(...)` markers, so the loop and adapters can detect them
|
|
200
|
-
* without coupling to a class hierarchy.
|
|
201
|
-
*/
|
|
202
|
-
export function isComputerUseTool(t) {
|
|
203
|
-
if (t === null || typeof t !== 'object')
|
|
204
|
-
return false;
|
|
205
|
-
const marker = t[COMPUTER_USE_MARKER];
|
|
206
|
-
return marker === true;
|
|
207
|
-
}
|
|
208
|
-
// ─── Re-export error classes + helper ─────────────────────
|
|
209
|
-
export { ComputerUseLimitError, ComputerUseProviderError, isAnthropicLikeModel };
|
|
210
|
-
//# sourceMappingURL=tool.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"tool.js","sourceRoot":"","sources":["../../src/computer-use/tool.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+DG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAIvB,OAAO,EACL,qBAAqB,EACrB,wBAAwB,EACxB,oBAAoB,GACrB,MAAM,aAAa,CAAA;AACpB,OAAO,EACL,iBAAiB,GAKlB,MAAM,cAAc,CAAA;AACrB,OAAO,EAAE,qBAAqB,EAAE,MAAM,iBAAiB,CAAA;AAEvD;;;;;GAKG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAkB,MAAM,CAAC,GAAG,CAAC,0BAA0B,CAAC,CAAA;AAExF;;;;GAIG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,UAAU,CAAA;AAEhD,MAAM,gBAAgB,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAW,CAAA;AAC9D,MAAM,mBAAmB,GAAG,EAAE,CAAA;AAC9B,MAAM,sBAAsB,GAAoD,IAAI,CAAA;AA8DpF;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,IAA4B;IAC1D,4DAA4D;IAC5D,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAClE,MAAM,IAAI,wBAAwB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IAChD,CAAC;IAED,MAAM,QAAQ,GAAS,IAAI,CAAC,QAAQ,IAAS,gBAAgB,CAAA;IAC7D,MAAM,UAAU,GAAO,IAAI,CAAC,UAAU,IAAO,mBAAmB,CAAA;IAChE,MAAM,aAAa,GAAI,IAAI,CAAC,aAAa,IAAI,sBAAsB,CAAA;IACnE,MAAM,KAAK,GAAY,IAAI,CAAC,KAAK,IAAY,iBAAiB,EAAE,CAAA;IAChE,MAAM,IAAI,GAAa,IAAI,CAAC,IAAI,CAAA;IAEhC,sEAAsE;IACtE,iBAAiB;IACjB,MAAM,OAAO,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,CAAA;IAE5B,oEAAoE;IACpE,4FAA4F;IAC5F,MAAM,0BAA0B,GAC9B,OAAO,aAAa,KAAK,UAAU;QACjC,CAAC,CAAC,CAAC,KAAK,EAAE,EAAE,CAAE,aAAgD,CAAC,KAAK,CAAC;QACrE,CAAC,CAAC,aAAa,CAAA;IAEnB,MAAM,UAAU,GAA0B;QACxC,IAAI,EAAS,sBAAsB;QACnC,WAAW,EACT,0EAA0E;YAC1E,oEAAoE;QACtE,sEAAsE;QACtE,qEAAqE;QACrE,oEAAoE;QACpE,kEAAkE;QAClE,uCAAuC;QACvC,WAAW,EAAI,CAAC,CAAC,GAAG,EAAE;QACtB,aAAa,EAAE,0BAAoE;QACnF,oEAAoE;QACpE,8DAA8D;QAC9D,sEAAsE;QACtE,YAAY,EAAE;YACZ,IAAI,EAAe,cAAc;YACjC,IAAI,EAAe,mBAAmB;YACtC,gBAAgB,EAAG,QAAQ,CAAC,KAAK;YACjC,iBAAiB,EAAE,QAAQ,CAAC,MAAM;SACnC;KACF,CAAA;IAED,MAAM,IAAI,GAAoB;QAC5B,CAAC,mBAAmB,CAAC,EAAE,IAAI;QAC3B,UAAU;QACV,KAAK,CAAC,OAAO,CAAC,KAAqB;YACjC,OAAO,CAAC,KAAK,EAAE,CAAA;YACf,IAAI,OAAO,CAAC,KAAK,GAAG,UAAU,EAAE,CAAC;gBAC/B,MAAM,IAAI,qBAAqB,CAAC,UAAU,CAAC,CAAA;YAC7C,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,qBAAqB,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,CAAA;YAC9D,OAAO,kBAAkB,CAAC,MAAM,CAAC,CAAA;QACnC,CAAC;QACD,QAAQ;YACN,OAAO;gBACL,IAAI,EAAS,sBAAsB;gBACnC,WAAW,EAAE,UAAU,CAAC,WAAW;gBACnC,gEAAgE;gBAChE,UAAU,EAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,EAAE,EAAE,oBAAoB,EAAE,IAAI,EAAE;gBAC3E,YAAY,EAAE,UAAU,CAAC,YAAa;aACvC,CAAA;QACH,CAAC;KACF,CAAA;IAED,OAAO,IAAI,CAAA;AACb,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,SAAS,kBAAkB,CAAC,MAA4B;IACtD,IAAI,MAAM,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;QACvC,OAAO;YACL;gBACE,IAAI,EAAM,OAAO;gBACjB,QAAQ,EAAE,MAAM,CAAC,UAAU;gBAC3B,IAAI;aACL;SACF,CAAA;IACH,CAAC;IACD,IAAI,MAAM,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QAC3B,OAAO,MAAM,CAAC,IAAI,CAAA;IACpB,CAAC;IACD,uEAAuE;IACvE,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;AAC9B,CAAC;AAED;;;;;GAKG;AACH,SAAS,aAAa,CAAC,IAAgB;IACrC,IAAI,OAAO,MAAM,KAAK,WAAW,EAAE,CAAC;QAClC,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAA;IAC7C,CAAC;IACD,uCAAuC;IACvC,IAAI,MAAM,GAAG,EAAE,CAAA;IACf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE;QAAE,MAAM,IAAI,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAW,CAAC,CAAA;IACtF,OAAO,IAAI,CAAC,MAAM,CAAC,CAAA;AACrB,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,iBAAiB,CAAC,CAAU;IAC1C,IAAI,CAAC,KAAK,IAAI,IAAI,OAAO,CAAC,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAA;IACrD,MAAM,MAAM,GAAI,CAAsC,CAAC,mBAAmB,CAAC,CAAA;IAC3E,OAAO,MAAM,KAAK,IAAI,CAAA;AACxB,CAAC;AAED,6DAA6D;AAE7D,OAAO,EAAE,qBAAqB,EAAE,wBAAwB,EAAE,oBAAoB,EAAE,CAAA"}
|