@rudderjs/ai 1.5.0 → 1.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +482 -4
- package/boost/guidelines.md +60 -0
- package/boost/skills/ai-agents/SKILL.md +7 -0
- package/boost/skills/ai-tools/SKILL.md +7 -0
- package/dist/agent.d.ts +35 -1
- package/dist/agent.d.ts.map +1 -1
- package/dist/agent.js +118 -16
- package/dist/agent.js.map +1 -1
- package/dist/budget/pricing.d.ts +124 -0
- package/dist/budget/pricing.d.ts.map +1 -0
- package/dist/budget/pricing.js +175 -0
- package/dist/budget/pricing.js.map +1 -0
- package/dist/budget/storage.d.ts +104 -0
- package/dist/budget/storage.d.ts.map +1 -0
- package/dist/budget/storage.js +0 -0
- package/dist/budget/storage.js.map +1 -0
- package/dist/budget/with-budget.d.ts +119 -0
- package/dist/budget/with-budget.d.ts.map +1 -0
- package/dist/budget/with-budget.js +175 -0
- package/dist/budget/with-budget.js.map +1 -0
- package/dist/budget-orm/index.d.ts +96 -0
- package/dist/budget-orm/index.d.ts.map +1 -0
- package/dist/budget-orm/index.js +177 -0
- package/dist/budget-orm/index.js.map +1 -0
- package/dist/commands/ai-eval.d.ts +93 -0
- package/dist/commands/ai-eval.d.ts.map +1 -0
- package/dist/commands/ai-eval.js +378 -0
- package/dist/commands/ai-eval.js.map +1 -0
- package/dist/computer-use/actions.d.ts +214 -0
- package/dist/computer-use/actions.d.ts.map +1 -0
- package/dist/computer-use/actions.js +48 -0
- package/dist/computer-use/actions.js.map +1 -0
- package/dist/computer-use/errors.d.ts +57 -0
- package/dist/computer-use/errors.d.ts.map +1 -0
- package/dist/computer-use/errors.js +76 -0
- package/dist/computer-use/errors.js.map +1 -0
- package/dist/computer-use/index.d.ts +53 -0
- package/dist/computer-use/index.d.ts.map +1 -0
- package/dist/computer-use/index.js +51 -0
- package/dist/computer-use/index.js.map +1 -0
- package/dist/computer-use/playwright.d.ts +76 -0
- package/dist/computer-use/playwright.d.ts.map +1 -0
- package/dist/computer-use/playwright.js +270 -0
- package/dist/computer-use/playwright.js.map +1 -0
- package/dist/computer-use/tool.d.ts +154 -0
- package/dist/computer-use/tool.d.ts.map +1 -0
- package/dist/computer-use/tool.js +210 -0
- package/dist/computer-use/tool.js.map +1 -0
- package/dist/eval/fixtures.d.ts +65 -0
- package/dist/eval/fixtures.d.ts.map +1 -0
- package/dist/eval/fixtures.js +110 -0
- package/dist/eval/fixtures.js.map +1 -0
- package/dist/eval/html-reporter.d.ts +25 -0
- package/dist/eval/html-reporter.d.ts.map +1 -0
- package/dist/eval/html-reporter.js +209 -0
- package/dist/eval/html-reporter.js.map +1 -0
- package/dist/eval/index.d.ts +271 -0
- package/dist/eval/index.d.ts.map +1 -0
- package/dist/eval/index.js +510 -0
- package/dist/eval/index.js.map +1 -0
- package/dist/eval/json-reporter.d.ts +43 -0
- package/dist/eval/json-reporter.d.ts.map +1 -0
- package/dist/eval/json-reporter.js +40 -0
- package/dist/eval/json-reporter.js.map +1 -0
- package/dist/fake.d.ts +36 -1
- package/dist/fake.d.ts.map +1 -1
- package/dist/fake.js +49 -2
- package/dist/fake.js.map +1 -1
- package/dist/file-search.d.ts +168 -0
- package/dist/file-search.d.ts.map +1 -0
- package/dist/file-search.js +158 -0
- package/dist/file-search.js.map +1 -0
- package/dist/index.d.ts +22 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +17 -1
- package/dist/index.js.map +1 -1
- package/dist/mcp/client-tools.d.ts +39 -0
- package/dist/mcp/client-tools.d.ts.map +1 -0
- package/dist/mcp/client-tools.js +147 -0
- package/dist/mcp/client-tools.js.map +1 -0
- package/dist/mcp/index.d.ts +16 -0
- package/dist/mcp/index.d.ts.map +1 -0
- package/dist/mcp/index.js +15 -0
- package/dist/mcp/index.js.map +1 -0
- package/dist/mcp/server-from-agent.d.ts +24 -0
- package/dist/mcp/server-from-agent.d.ts.map +1 -0
- package/dist/mcp/server-from-agent.js +113 -0
- package/dist/mcp/server-from-agent.js.map +1 -0
- package/dist/mcp/types.d.ts +64 -0
- package/dist/mcp/types.d.ts.map +1 -0
- package/dist/mcp/types.js +6 -0
- package/dist/mcp/types.js.map +1 -0
- package/dist/memory-embedding/index.d.ts +121 -0
- package/dist/memory-embedding/index.d.ts.map +1 -0
- package/dist/memory-embedding/index.js +229 -0
- package/dist/memory-embedding/index.js.map +1 -0
- package/dist/memory-extract.d.ts +60 -0
- package/dist/memory-extract.d.ts.map +1 -0
- package/dist/memory-extract.js +163 -0
- package/dist/memory-extract.js.map +1 -0
- package/dist/memory-inject.d.ts +39 -0
- package/dist/memory-inject.d.ts.map +1 -0
- package/dist/memory-inject.js +135 -0
- package/dist/memory-inject.js.map +1 -0
- package/dist/memory-orm/index.d.ts +118 -0
- package/dist/memory-orm/index.d.ts.map +1 -0
- package/dist/memory-orm/index.js +187 -0
- package/dist/memory-orm/index.js.map +1 -0
- package/dist/memory.d.ts +55 -0
- package/dist/memory.d.ts.map +1 -0
- package/dist/memory.js +132 -0
- package/dist/memory.js.map +1 -0
- package/dist/observers.d.ts +22 -0
- package/dist/observers.d.ts.map +1 -1
- package/dist/observers.js.map +1 -1
- package/dist/provider-tools.d.ts +15 -1
- package/dist/provider-tools.d.ts.map +1 -1
- package/dist/provider-tools.js +21 -1
- package/dist/provider-tools.js.map +1 -1
- package/dist/providers/anthropic.d.ts.map +1 -1
- package/dist/providers/anthropic.js +61 -6
- package/dist/providers/anthropic.js.map +1 -1
- package/dist/providers/elevenlabs.d.ts +98 -0
- package/dist/providers/elevenlabs.d.ts.map +1 -0
- package/dist/providers/elevenlabs.js +229 -0
- package/dist/providers/elevenlabs.js.map +1 -0
- package/dist/providers/google.d.ts +83 -1
- package/dist/providers/google.d.ts.map +1 -1
- package/dist/providers/google.js +491 -8
- package/dist/providers/google.js.map +1 -1
- package/dist/providers/openai.d.ts +3 -1
- package/dist/providers/openai.d.ts.map +1 -1
- package/dist/providers/openai.js +209 -5
- package/dist/providers/openai.js.map +1 -1
- package/dist/providers/voyage.d.ts +91 -0
- package/dist/providers/voyage.d.ts.map +1 -0
- package/dist/providers/voyage.js +166 -0
- package/dist/providers/voyage.js.map +1 -0
- package/dist/queue-job.d.ts +69 -4
- package/dist/queue-job.d.ts.map +1 -1
- package/dist/queue-job.js +114 -11
- package/dist/queue-job.js.map +1 -1
- package/dist/registry.d.ts +3 -1
- package/dist/registry.d.ts.map +1 -1
- package/dist/registry.js +10 -0
- package/dist/registry.js.map +1 -1
- package/dist/server/provider.d.ts.map +1 -1
- package/dist/server/provider.js +23 -1
- package/dist/server/provider.js.map +1 -1
- package/dist/similarity-search.d.ts +163 -0
- package/dist/similarity-search.d.ts.map +1 -0
- package/dist/similarity-search.js +147 -0
- package/dist/similarity-search.js.map +1 -0
- package/dist/tool.d.ts.map +1 -1
- package/dist/tool.js +13 -4
- package/dist/tool.js.map +1 -1
- package/dist/types.d.ts +246 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/vector-stores/index.d.ts +96 -0
- package/dist/vector-stores/index.d.ts.map +1 -0
- package/dist/vector-stores/index.js +153 -0
- package/dist/vector-stores/index.js.map +1 -0
- package/package.json +41 -3
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Error classes for `@rudderjs/ai/computer-use` (#A7 Phase 2).
|
|
3
|
+
*
|
|
4
|
+
* Both extend native `Error` so apps can `instanceof`-check from
|
|
5
|
+
* `try/catch`, error middleware, observers, etc.
|
|
6
|
+
*/
|
|
7
|
+
/**
|
|
8
|
+
* Thrown by {@link computerUseTool} at construction time when the
|
|
9
|
+
* caller supplies a `model` that isn't an Anthropic-family model
|
|
10
|
+
* (matches `anthropic/*` or `bedrock/<region.>?anthropic.*`).
|
|
11
|
+
*
|
|
12
|
+
* Computer-use is Anthropic-only in v1 — see plan
|
|
13
|
+
* `docs/plans/2026-05-10-ai-computer-use.md`. Other providers either
|
|
14
|
+
* lack native computer-use entirely (Google, Cohere, …) or only have
|
|
15
|
+
* preview-quality versions that aren't worth shipping yet (OpenAI's
|
|
16
|
+
* `computer_use_preview`).
|
|
17
|
+
*
|
|
18
|
+
* Apps that wire `computerUseTool({ page, model: this.model() })`
|
|
19
|
+
* inside `Agent.tools()` get this error at agent boot — fail loud
|
|
20
|
+
* before the model gets a chance to hallucinate tool calls against a
|
|
21
|
+
* provider that can't execute them.
|
|
22
|
+
*/
|
|
23
|
+
export declare class ComputerUseProviderError extends Error {
|
|
24
|
+
readonly code: "COMPUTER_USE_PROVIDER_MISMATCH";
|
|
25
|
+
readonly model: string;
|
|
26
|
+
constructor(model: string);
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Thrown by {@link computerUseTool}'s execute when the per-run action
|
|
30
|
+
* counter exceeds {@link ComputerUseToolOptions.maxActions}.
|
|
31
|
+
*
|
|
32
|
+
* Bounds runaway agent loops (e.g. a model that keeps clicking the
|
|
33
|
+
* same broken button forever). Default cap is 50 — most real
|
|
34
|
+
* computer-use tasks finish well under that. Override via
|
|
35
|
+
* `computerUseTool({ page, maxActions: 100 })`.
|
|
36
|
+
*/
|
|
37
|
+
export declare class ComputerUseLimitError extends Error {
|
|
38
|
+
readonly code: "COMPUTER_USE_LIMIT_EXCEEDED";
|
|
39
|
+
readonly maxActions: number;
|
|
40
|
+
constructor(maxActions: number);
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Returns true when `model` is an Anthropic-family model id —
|
|
44
|
+
* `anthropic/*` or `bedrock/<region.>?anthropic.*` (covers cross-region
|
|
45
|
+
* inference profiles like `us.anthropic.*`, `eu.anthropic.*`,
|
|
46
|
+
* `apac.anthropic.*`).
|
|
47
|
+
*
|
|
48
|
+
* Used by {@link computerUseTool}'s upfront `model` check. Exported so
|
|
49
|
+
* apps can guard their own code path symmetrically.
|
|
50
|
+
*
|
|
51
|
+
* **Excludes** OpenRouter-routed Anthropic models
|
|
52
|
+
* (`openrouter/anthropic/*`) — OpenRouter goes through the OpenAI SDK
|
|
53
|
+
* with a different base URL, so the request never hits Anthropic's
|
|
54
|
+
* native API and the native computer-use tool block can't be sent.
|
|
55
|
+
*/
|
|
56
|
+
export declare function isAnthropicLikeModel(model: string): boolean;
|
|
57
|
+
//# sourceMappingURL=errors.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../../src/computer-use/errors.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH;;;;;;;;;;;;;;;GAeG;AACH,qBAAa,wBAAyB,SAAQ,KAAK;IACjD,QAAQ,CAAC,IAAI,EAAG,gCAAgC,CAAS;IACzD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;gBAEV,KAAK,EAAE,MAAM;CAQ1B;AAED;;;;;;;;GAQG;AACH,qBAAa,qBAAsB,SAAQ,KAAK;IAC9C,QAAQ,CAAC,IAAI,EAAG,6BAA6B,CAAS;IACtD,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;gBAEf,UAAU,EAAE,MAAM;CAQ/B;AAID;;;;;;;;;;;;;GAaG;AACH,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAM3D"}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Error classes for `@rudderjs/ai/computer-use` (#A7 Phase 2).
|
|
3
|
+
*
|
|
4
|
+
* Both extend native `Error` so apps can `instanceof`-check from
|
|
5
|
+
* `try/catch`, error middleware, observers, etc.
|
|
6
|
+
*/
|
|
7
|
+
/**
|
|
8
|
+
* Thrown by {@link computerUseTool} at construction time when the
|
|
9
|
+
* caller supplies a `model` that isn't an Anthropic-family model
|
|
10
|
+
* (matches `anthropic/*` or `bedrock/<region.>?anthropic.*`).
|
|
11
|
+
*
|
|
12
|
+
* Computer-use is Anthropic-only in v1 — see plan
|
|
13
|
+
* `docs/plans/2026-05-10-ai-computer-use.md`. Other providers either
|
|
14
|
+
* lack native computer-use entirely (Google, Cohere, …) or only have
|
|
15
|
+
* preview-quality versions that aren't worth shipping yet (OpenAI's
|
|
16
|
+
* `computer_use_preview`).
|
|
17
|
+
*
|
|
18
|
+
* Apps that wire `computerUseTool({ page, model: this.model() })`
|
|
19
|
+
* inside `Agent.tools()` get this error at agent boot — fail loud
|
|
20
|
+
* before the model gets a chance to hallucinate tool calls against a
|
|
21
|
+
* provider that can't execute them.
|
|
22
|
+
*/
|
|
23
|
+
export class ComputerUseProviderError extends Error {
|
|
24
|
+
code = 'COMPUTER_USE_PROVIDER_MISMATCH';
|
|
25
|
+
model;
|
|
26
|
+
constructor(model) {
|
|
27
|
+
super(`[RudderJS AI] computerUseTool is Anthropic-only in v1; got model "${model}". ` +
|
|
28
|
+
`Use an "anthropic/*" or "bedrock/<region.>?anthropic.*" model, or remove the tool.`);
|
|
29
|
+
this.name = 'ComputerUseProviderError';
|
|
30
|
+
this.model = model;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Thrown by {@link computerUseTool}'s execute when the per-run action
|
|
35
|
+
* counter exceeds {@link ComputerUseToolOptions.maxActions}.
|
|
36
|
+
*
|
|
37
|
+
* Bounds runaway agent loops (e.g. a model that keeps clicking the
|
|
38
|
+
* same broken button forever). Default cap is 50 — most real
|
|
39
|
+
* computer-use tasks finish well under that. Override via
|
|
40
|
+
* `computerUseTool({ page, maxActions: 100 })`.
|
|
41
|
+
*/
|
|
42
|
+
export class ComputerUseLimitError extends Error {
|
|
43
|
+
code = 'COMPUTER_USE_LIMIT_EXCEEDED';
|
|
44
|
+
maxActions;
|
|
45
|
+
constructor(maxActions) {
|
|
46
|
+
super(`[RudderJS AI] computerUseTool exceeded maxActions cap of ${maxActions}. ` +
|
|
47
|
+
`Bump the cap with computerUseTool({ page, maxActions: <n> }) if your agent legitimately needs more steps.`);
|
|
48
|
+
this.name = 'ComputerUseLimitError';
|
|
49
|
+
this.maxActions = maxActions;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
// ─── Model classification ─────────────────────────────────
|
|
53
|
+
/**
|
|
54
|
+
* Returns true when `model` is an Anthropic-family model id —
|
|
55
|
+
* `anthropic/*` or `bedrock/<region.>?anthropic.*` (covers cross-region
|
|
56
|
+
* inference profiles like `us.anthropic.*`, `eu.anthropic.*`,
|
|
57
|
+
* `apac.anthropic.*`).
|
|
58
|
+
*
|
|
59
|
+
* Used by {@link computerUseTool}'s upfront `model` check. Exported so
|
|
60
|
+
* apps can guard their own code path symmetrically.
|
|
61
|
+
*
|
|
62
|
+
* **Excludes** OpenRouter-routed Anthropic models
|
|
63
|
+
* (`openrouter/anthropic/*`) — OpenRouter goes through the OpenAI SDK
|
|
64
|
+
* with a different base URL, so the request never hits Anthropic's
|
|
65
|
+
* native API and the native computer-use tool block can't be sent.
|
|
66
|
+
*/
|
|
67
|
+
export function isAnthropicLikeModel(model) {
|
|
68
|
+
if (model.startsWith('anthropic/'))
|
|
69
|
+
return true;
|
|
70
|
+
// Bedrock: bedrock/anthropic.* OR bedrock/<region>.anthropic.*
|
|
71
|
+
// (region prefixes us./eu./apac./...)
|
|
72
|
+
if (/^bedrock\/(?:[a-z]{2,4}\.)?anthropic\./.test(model))
|
|
73
|
+
return true;
|
|
74
|
+
return false;
|
|
75
|
+
}
|
|
76
|
+
//# sourceMappingURL=errors.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"errors.js","sourceRoot":"","sources":["../../src/computer-use/errors.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH;;;;;;;;;;;;;;;GAeG;AACH,MAAM,OAAO,wBAAyB,SAAQ,KAAK;IACxC,IAAI,GAAG,gCAAyC,CAAA;IAChD,KAAK,CAAQ;IAEtB,YAAY,KAAa;QACvB,KAAK,CACH,qEAAqE,KAAK,KAAK;YAC/E,oFAAoF,CACrF,CAAA;QACD,IAAI,CAAC,IAAI,GAAG,0BAA0B,CAAA;QACtC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAA;IACpB,CAAC;CACF;AAED;;;;;;;;GAQG;AACH,MAAM,OAAO,qBAAsB,SAAQ,KAAK;IACrC,IAAI,GAAG,6BAAsC,CAAA;IAC7C,UAAU,CAAQ;IAE3B,YAAY,UAAkB;QAC5B,KAAK,CACH,4DAA4D,UAAU,IAAI;YAC1E,2GAA2G,CAC5G,CAAA;QACD,IAAI,CAAC,IAAI,GAAG,uBAAuB,CAAA;QACnC,IAAI,CAAC,UAAU,GAAG,UAAU,CAAA;IAC9B,CAAC;CACF;AAED,6DAA6D;AAE7D;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,oBAAoB,CAAC,KAAa;IAChD,IAAI,KAAK,CAAC,UAAU,CAAC,YAAY,CAAC;QAAE,OAAO,IAAI,CAAA;IAC/C,+DAA+D;IAC/D,sCAAsC;IACtC,IAAI,wCAAwC,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAA;IACrE,OAAO,KAAK,CAAA;AACd,CAAC"}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `@rudderjs/ai/computer-use` — Anthropic computer-use abstraction (#A7).
|
|
3
|
+
*
|
|
4
|
+
* Phase 1 (this entry) ships the action vocabulary + Playwright
|
|
5
|
+
* executor. Phase 2 will add `computerUseTool({ page })` — the agent
|
|
6
|
+
* tool factory that maps to Anthropic's native `computer_20250124`
|
|
7
|
+
* tool block and routes execution through the executor here.
|
|
8
|
+
*
|
|
9
|
+
* # Quick example (manual / phase-1 surface)
|
|
10
|
+
*
|
|
11
|
+
* ```ts
|
|
12
|
+
* import { chromium } from 'playwright'
|
|
13
|
+
* import { executeComputerAction, makeExecutorState } from '@rudderjs/ai/computer-use'
|
|
14
|
+
*
|
|
15
|
+
* const browser = await chromium.launch()
|
|
16
|
+
* const page = await browser.newPage()
|
|
17
|
+
* await page.setViewportSize({ width: 1280, height: 800 })
|
|
18
|
+
* await page.goto('https://example.com')
|
|
19
|
+
*
|
|
20
|
+
* const state = makeExecutorState()
|
|
21
|
+
* const screen = await executeComputerAction(page, { action: 'screenshot' }, state)
|
|
22
|
+
* if (screen.type === 'image') {
|
|
23
|
+
* // screen.data is a PNG Uint8Array
|
|
24
|
+
* }
|
|
25
|
+
*
|
|
26
|
+
* await executeComputerAction(page, { action: 'left_click', coordinate: [400, 200] }, state)
|
|
27
|
+
* ```
|
|
28
|
+
*
|
|
29
|
+
* Phase 2 will collapse this to:
|
|
30
|
+
*
|
|
31
|
+
* ```ts
|
|
32
|
+
* import { computerUseTool } from '@rudderjs/ai/computer-use'
|
|
33
|
+
*
|
|
34
|
+
* class BrowserAgent extends Agent {
|
|
35
|
+
* model = 'anthropic/claude-opus-4-7'
|
|
36
|
+
* tools() { return [computerUseTool({ page })] }
|
|
37
|
+
* }
|
|
38
|
+
* ```
|
|
39
|
+
*
|
|
40
|
+
* # Anthropic-only in v1
|
|
41
|
+
*
|
|
42
|
+
* The action vocabulary mirrors Anthropic's `computer_20250124` schema
|
|
43
|
+
* verbatim. Phase 2's tool factory throws `ComputerUseProviderError` at
|
|
44
|
+
* agent boot for non-Anthropic models — see plan
|
|
45
|
+
* `docs/plans/2026-05-10-ai-computer-use.md`.
|
|
46
|
+
*/
|
|
47
|
+
export type { ComputerAction, ComputerActionResult, ComputerExecutorState, Coordinate, PageLike, PageMouseLike, PageKeyboardLike, } from './actions.js';
|
|
48
|
+
export { makeExecutorState } from './actions.js';
|
|
49
|
+
export { executeComputerAction, parseModifiers, normalizeKey, normalizeChord, SCROLL_PIXELS_PER_CLICK, MOUSE_MOVE_STEPS, } from './playwright.js';
|
|
50
|
+
export type { ComputerUseTool, ComputerUseToolOptions, } from './tool.js';
|
|
51
|
+
export { computerUseTool, isComputerUseTool, COMPUTER_USE_MARKER, COMPUTER_USE_TOOL_NAME, } from './tool.js';
|
|
52
|
+
export { ComputerUseLimitError, ComputerUseProviderError, isAnthropicLikeModel, } from './errors.js';
|
|
53
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/computer-use/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6CG;AAEH,YAAY,EACV,cAAc,EACd,oBAAoB,EACpB,qBAAqB,EACrB,UAAU,EACV,QAAQ,EACR,aAAa,EACb,gBAAgB,GACjB,MAAM,cAAc,CAAA;AAErB,OAAO,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAA;AAEhD,OAAO,EACL,qBAAqB,EACrB,cAAc,EACd,YAAY,EACZ,cAAc,EACd,uBAAuB,EACvB,gBAAgB,GACjB,MAAM,iBAAiB,CAAA;AAIxB,YAAY,EACV,eAAe,EACf,sBAAsB,GACvB,MAAM,WAAW,CAAA;AAElB,OAAO,EACL,eAAe,EACf,iBAAiB,EACjB,mBAAmB,EACnB,sBAAsB,GACvB,MAAM,WAAW,CAAA;AAElB,OAAO,EACL,qBAAqB,EACrB,wBAAwB,EACxB,oBAAoB,GACrB,MAAM,aAAa,CAAA"}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `@rudderjs/ai/computer-use` — Anthropic computer-use abstraction (#A7).
|
|
3
|
+
*
|
|
4
|
+
* Phase 1 (this entry) ships the action vocabulary + Playwright
|
|
5
|
+
* executor. Phase 2 will add `computerUseTool({ page })` — the agent
|
|
6
|
+
* tool factory that maps to Anthropic's native `computer_20250124`
|
|
7
|
+
* tool block and routes execution through the executor here.
|
|
8
|
+
*
|
|
9
|
+
* # Quick example (manual / phase-1 surface)
|
|
10
|
+
*
|
|
11
|
+
* ```ts
|
|
12
|
+
* import { chromium } from 'playwright'
|
|
13
|
+
* import { executeComputerAction, makeExecutorState } from '@rudderjs/ai/computer-use'
|
|
14
|
+
*
|
|
15
|
+
* const browser = await chromium.launch()
|
|
16
|
+
* const page = await browser.newPage()
|
|
17
|
+
* await page.setViewportSize({ width: 1280, height: 800 })
|
|
18
|
+
* await page.goto('https://example.com')
|
|
19
|
+
*
|
|
20
|
+
* const state = makeExecutorState()
|
|
21
|
+
* const screen = await executeComputerAction(page, { action: 'screenshot' }, state)
|
|
22
|
+
* if (screen.type === 'image') {
|
|
23
|
+
* // screen.data is a PNG Uint8Array
|
|
24
|
+
* }
|
|
25
|
+
*
|
|
26
|
+
* await executeComputerAction(page, { action: 'left_click', coordinate: [400, 200] }, state)
|
|
27
|
+
* ```
|
|
28
|
+
*
|
|
29
|
+
* Phase 2 will collapse this to:
|
|
30
|
+
*
|
|
31
|
+
* ```ts
|
|
32
|
+
* import { computerUseTool } from '@rudderjs/ai/computer-use'
|
|
33
|
+
*
|
|
34
|
+
* class BrowserAgent extends Agent {
|
|
35
|
+
* model = 'anthropic/claude-opus-4-7'
|
|
36
|
+
* tools() { return [computerUseTool({ page })] }
|
|
37
|
+
* }
|
|
38
|
+
* ```
|
|
39
|
+
*
|
|
40
|
+
* # Anthropic-only in v1
|
|
41
|
+
*
|
|
42
|
+
* The action vocabulary mirrors Anthropic's `computer_20250124` schema
|
|
43
|
+
* verbatim. Phase 2's tool factory throws `ComputerUseProviderError` at
|
|
44
|
+
* agent boot for non-Anthropic models — see plan
|
|
45
|
+
* `docs/plans/2026-05-10-ai-computer-use.md`.
|
|
46
|
+
*/
|
|
47
|
+
export { makeExecutorState } from './actions.js';
|
|
48
|
+
export { executeComputerAction, parseModifiers, normalizeKey, normalizeChord, SCROLL_PIXELS_PER_CLICK, MOUSE_MOVE_STEPS, } from './playwright.js';
|
|
49
|
+
export { computerUseTool, isComputerUseTool, COMPUTER_USE_MARKER, COMPUTER_USE_TOOL_NAME, } from './tool.js';
|
|
50
|
+
export { ComputerUseLimitError, ComputerUseProviderError, isAnthropicLikeModel, } from './errors.js';
|
|
51
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/computer-use/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6CG;AAYH,OAAO,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAA;AAEhD,OAAO,EACL,qBAAqB,EACrB,cAAc,EACd,YAAY,EACZ,cAAc,EACd,uBAAuB,EACvB,gBAAgB,GACjB,MAAM,iBAAiB,CAAA;AASxB,OAAO,EACL,eAAe,EACf,iBAAiB,EACjB,mBAAmB,EACnB,sBAAsB,GACvB,MAAM,WAAW,CAAA;AAElB,OAAO,EACL,qBAAqB,EACrB,wBAAwB,EACxB,oBAAoB,GACrB,MAAM,aAAa,CAAA"}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Playwright executor for {@link ComputerAction} (#A7 Phase 1).
|
|
3
|
+
*
|
|
4
|
+
* Translates Anthropic's `computer_20250124` action vocabulary into
|
|
5
|
+
* Playwright `Page` calls. Stateless apart from {@link ComputerExecutorState},
|
|
6
|
+
* which the caller threads through so `cursor_position` can answer.
|
|
7
|
+
*
|
|
8
|
+
* # Errors
|
|
9
|
+
*
|
|
10
|
+
* Playwright throws on failures (selector misses, timeouts, navigation
|
|
11
|
+
* during action). The executor catches and returns
|
|
12
|
+
* `{ type: 'error', text: <message> }` so the agent loop can hand the
|
|
13
|
+
* failure back to the model as a tool-result with `is_error: true`. The
|
|
14
|
+
* model decides whether to retry, recover, or give up — failing the
|
|
15
|
+
* whole agent run on a single missed click is too brittle.
|
|
16
|
+
*
|
|
17
|
+
* # Modifier keys & key chords
|
|
18
|
+
*
|
|
19
|
+
* Anthropic uses xdotool naming (`ctrl`, `cmd`, `Return`); Playwright
|
|
20
|
+
* uses its own (`Control`, `Meta`, `Enter`). {@link normalizeKey} maps
|
|
21
|
+
* the common ones; unmapped keys pass through verbatim and Playwright
|
|
22
|
+
* either accepts them (single chars, function keys) or throws — which
|
|
23
|
+
* surfaces as the standard error path above.
|
|
24
|
+
*
|
|
25
|
+
* # Scroll units
|
|
26
|
+
*
|
|
27
|
+
* Anthropic's `scroll_amount` is in mouse-wheel "clicks". A typical
|
|
28
|
+
* desktop wheel click is ~100px; the executor multiplies through
|
|
29
|
+
* {@link SCROLL_PIXELS_PER_CLICK}. Tunable later if customer feedback
|
|
30
|
+
* shows drift.
|
|
31
|
+
*/
|
|
32
|
+
import type { ComputerAction, ComputerActionResult, ComputerExecutorState, PageLike } from './actions.js';
|
|
33
|
+
/**
|
|
34
|
+
* Pixels per `scroll_amount` unit. Roughly matches a desktop mouse
|
|
35
|
+
* wheel click on most platforms.
|
|
36
|
+
*/
|
|
37
|
+
export declare const SCROLL_PIXELS_PER_CLICK = 100;
|
|
38
|
+
/**
|
|
39
|
+
* Number of intermediate move steps Playwright interpolates when the
|
|
40
|
+
* executor moves the mouse to a target. Higher = smoother (more hover
|
|
41
|
+
* events fire), but slower. 5 is a reasonable default — instant enough
|
|
42
|
+
* for most pages, slow enough that hover-driven UI (tooltips, menus)
|
|
43
|
+
* has a chance to react.
|
|
44
|
+
*/
|
|
45
|
+
export declare const MOUSE_MOVE_STEPS = 5;
|
|
46
|
+
/**
|
|
47
|
+
* Dispatch a single {@link ComputerAction} against a Playwright `Page`.
|
|
48
|
+
*
|
|
49
|
+
* Updates `state.cursor` after every action that targets a coordinate
|
|
50
|
+
* (move / click / drag / scroll). Caller owns `state` and threads it
|
|
51
|
+
* through every call within an agent run.
|
|
52
|
+
*
|
|
53
|
+
* Returns a {@link ComputerActionResult} suitable for forwarding to the
|
|
54
|
+
* model as a tool-result. Never throws — Playwright failures surface as
|
|
55
|
+
* `{ type: 'error', text }`.
|
|
56
|
+
*/
|
|
57
|
+
export declare function executeComputerAction(page: PageLike, action: ComputerAction, state: ComputerExecutorState): Promise<ComputerActionResult>;
|
|
58
|
+
/**
|
|
59
|
+
* Parse Anthropic's `+`-separated modifier text into Playwright key
|
|
60
|
+
* names. Empty / undefined input → empty array (no modifiers).
|
|
61
|
+
*/
|
|
62
|
+
export declare function parseModifiers(text: string | undefined): string[];
|
|
63
|
+
/**
|
|
64
|
+
* Map Anthropic / xdotool key names to Playwright key names.
|
|
65
|
+
* Unmapped keys (single chars, F-keys, anything we haven't covered)
|
|
66
|
+
* pass through verbatim — Playwright either accepts them or throws,
|
|
67
|
+
* which is the right failure mode.
|
|
68
|
+
*/
|
|
69
|
+
export declare function normalizeKey(key: string): string;
|
|
70
|
+
/**
|
|
71
|
+
* Normalize a key chord (`+`-separated). Playwright's
|
|
72
|
+
* `keyboard.press()` parses chords natively, so we just normalize each
|
|
73
|
+
* segment and rejoin.
|
|
74
|
+
*/
|
|
75
|
+
export declare function normalizeChord(chord: string): string;
|
|
76
|
+
//# sourceMappingURL=playwright.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"playwright.d.ts","sourceRoot":"","sources":["../../src/computer-use/playwright.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAEH,OAAO,KAAK,EACV,cAAc,EACd,oBAAoB,EACpB,qBAAqB,EACrB,QAAQ,EACT,MAAM,cAAc,CAAA;AAErB;;;GAGG;AACH,eAAO,MAAM,uBAAuB,MAAM,CAAA;AAE1C;;;;;;GAMG;AACH,eAAO,MAAM,gBAAgB,IAAI,CAAA;AAEjC;;;;;;;;;;GAUG;AACH,wBAAsB,qBAAqB,CACzC,IAAI,EAAE,QAAQ,EACd,MAAM,EAAE,cAAc,EACtB,KAAK,EAAE,qBAAqB,GAC3B,OAAO,CAAC,oBAAoB,CAAC,CA4I/B;AAQD;;;GAGG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,EAAE,CAMjE;AAED;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CA8BhD;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAKpD"}
|
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Playwright executor for {@link ComputerAction} (#A7 Phase 1).
|
|
3
|
+
*
|
|
4
|
+
* Translates Anthropic's `computer_20250124` action vocabulary into
|
|
5
|
+
* Playwright `Page` calls. Stateless apart from {@link ComputerExecutorState},
|
|
6
|
+
* which the caller threads through so `cursor_position` can answer.
|
|
7
|
+
*
|
|
8
|
+
* # Errors
|
|
9
|
+
*
|
|
10
|
+
* Playwright throws on failures (selector misses, timeouts, navigation
|
|
11
|
+
* during action). The executor catches and returns
|
|
12
|
+
* `{ type: 'error', text: <message> }` so the agent loop can hand the
|
|
13
|
+
* failure back to the model as a tool-result with `is_error: true`. The
|
|
14
|
+
* model decides whether to retry, recover, or give up — failing the
|
|
15
|
+
* whole agent run on a single missed click is too brittle.
|
|
16
|
+
*
|
|
17
|
+
* # Modifier keys & key chords
|
|
18
|
+
*
|
|
19
|
+
* Anthropic uses xdotool naming (`ctrl`, `cmd`, `Return`); Playwright
|
|
20
|
+
* uses its own (`Control`, `Meta`, `Enter`). {@link normalizeKey} maps
|
|
21
|
+
* the common ones; unmapped keys pass through verbatim and Playwright
|
|
22
|
+
* either accepts them (single chars, function keys) or throws — which
|
|
23
|
+
* surfaces as the standard error path above.
|
|
24
|
+
*
|
|
25
|
+
* # Scroll units
|
|
26
|
+
*
|
|
27
|
+
* Anthropic's `scroll_amount` is in mouse-wheel "clicks". A typical
|
|
28
|
+
* desktop wheel click is ~100px; the executor multiplies through
|
|
29
|
+
* {@link SCROLL_PIXELS_PER_CLICK}. Tunable later if customer feedback
|
|
30
|
+
* shows drift.
|
|
31
|
+
*/
|
|
32
|
+
/**
|
|
33
|
+
* Pixels per `scroll_amount` unit. Roughly matches a desktop mouse
|
|
34
|
+
* wheel click on most platforms.
|
|
35
|
+
*/
|
|
36
|
+
export const SCROLL_PIXELS_PER_CLICK = 100;
|
|
37
|
+
/**
|
|
38
|
+
* Number of intermediate move steps Playwright interpolates when the
|
|
39
|
+
* executor moves the mouse to a target. Higher = smoother (more hover
|
|
40
|
+
* events fire), but slower. 5 is a reasonable default — instant enough
|
|
41
|
+
* for most pages, slow enough that hover-driven UI (tooltips, menus)
|
|
42
|
+
* has a chance to react.
|
|
43
|
+
*/
|
|
44
|
+
export const MOUSE_MOVE_STEPS = 5;
|
|
45
|
+
/**
|
|
46
|
+
* Dispatch a single {@link ComputerAction} against a Playwright `Page`.
|
|
47
|
+
*
|
|
48
|
+
* Updates `state.cursor` after every action that targets a coordinate
|
|
49
|
+
* (move / click / drag / scroll). Caller owns `state` and threads it
|
|
50
|
+
* through every call within an agent run.
|
|
51
|
+
*
|
|
52
|
+
* Returns a {@link ComputerActionResult} suitable for forwarding to the
|
|
53
|
+
* model as a tool-result. Never throws — Playwright failures surface as
|
|
54
|
+
* `{ type: 'error', text }`.
|
|
55
|
+
*/
|
|
56
|
+
export async function executeComputerAction(page, action, state) {
|
|
57
|
+
try {
|
|
58
|
+
switch (action.action) {
|
|
59
|
+
case 'screenshot': {
|
|
60
|
+
const data = await page.screenshot({ type: 'png' });
|
|
61
|
+
return { type: 'image', media_type: 'image/png', data };
|
|
62
|
+
}
|
|
63
|
+
case 'cursor_position': {
|
|
64
|
+
return { type: 'text', text: `X=${state.cursor.x}, Y=${state.cursor.y}` };
|
|
65
|
+
}
|
|
66
|
+
case 'wait': {
|
|
67
|
+
await sleep(action.duration * 1000);
|
|
68
|
+
return { type: 'text', text: `waited ${action.duration}s` };
|
|
69
|
+
}
|
|
70
|
+
case 'mouse_move': {
|
|
71
|
+
const [x, y] = action.coordinate;
|
|
72
|
+
await page.mouse.move(x, y, { steps: MOUSE_MOVE_STEPS });
|
|
73
|
+
state.cursor = { x, y };
|
|
74
|
+
return { type: 'text', text: `moved to (${x}, ${y})` };
|
|
75
|
+
}
|
|
76
|
+
case 'left_click':
|
|
77
|
+
case 'right_click':
|
|
78
|
+
case 'middle_click': {
|
|
79
|
+
const button = action.action === 'left_click'
|
|
80
|
+
? 'left'
|
|
81
|
+
: action.action === 'right_click'
|
|
82
|
+
? 'right'
|
|
83
|
+
: 'middle';
|
|
84
|
+
const [x, y] = action.coordinate;
|
|
85
|
+
const modifiers = parseModifiers(action.text);
|
|
86
|
+
await pressDown(page, modifiers);
|
|
87
|
+
try {
|
|
88
|
+
await page.mouse.move(x, y, { steps: MOUSE_MOVE_STEPS });
|
|
89
|
+
await page.mouse.click(x, y, { button });
|
|
90
|
+
}
|
|
91
|
+
finally {
|
|
92
|
+
await pressUp(page, modifiers);
|
|
93
|
+
}
|
|
94
|
+
state.cursor = { x, y };
|
|
95
|
+
return { type: 'text', text: `${button}-clicked at (${x}, ${y})` };
|
|
96
|
+
}
|
|
97
|
+
case 'double_click':
|
|
98
|
+
case 'triple_click': {
|
|
99
|
+
const clickCount = action.action === 'double_click' ? 2 : 3;
|
|
100
|
+
const label = action.action === 'double_click' ? 'double' : 'triple';
|
|
101
|
+
const [x, y] = action.coordinate;
|
|
102
|
+
const modifiers = parseModifiers(action.text);
|
|
103
|
+
await pressDown(page, modifiers);
|
|
104
|
+
try {
|
|
105
|
+
await page.mouse.move(x, y, { steps: MOUSE_MOVE_STEPS });
|
|
106
|
+
await page.mouse.click(x, y, { button: 'left', clickCount });
|
|
107
|
+
}
|
|
108
|
+
finally {
|
|
109
|
+
await pressUp(page, modifiers);
|
|
110
|
+
}
|
|
111
|
+
state.cursor = { x, y };
|
|
112
|
+
return { type: 'text', text: `${label}-clicked at (${x}, ${y})` };
|
|
113
|
+
}
|
|
114
|
+
case 'left_mouse_down': {
|
|
115
|
+
if (action.coordinate) {
|
|
116
|
+
const [x, y] = action.coordinate;
|
|
117
|
+
await page.mouse.move(x, y, { steps: MOUSE_MOVE_STEPS });
|
|
118
|
+
state.cursor = { x, y };
|
|
119
|
+
}
|
|
120
|
+
await page.mouse.down({ button: 'left' });
|
|
121
|
+
return { type: 'text', text: `mouse down at (${state.cursor.x}, ${state.cursor.y})` };
|
|
122
|
+
}
|
|
123
|
+
case 'left_mouse_up': {
|
|
124
|
+
if (action.coordinate) {
|
|
125
|
+
const [x, y] = action.coordinate;
|
|
126
|
+
await page.mouse.move(x, y, { steps: MOUSE_MOVE_STEPS });
|
|
127
|
+
state.cursor = { x, y };
|
|
128
|
+
}
|
|
129
|
+
await page.mouse.up({ button: 'left' });
|
|
130
|
+
return { type: 'text', text: `mouse up at (${state.cursor.x}, ${state.cursor.y})` };
|
|
131
|
+
}
|
|
132
|
+
case 'type': {
|
|
133
|
+
await page.keyboard.type(action.text);
|
|
134
|
+
return { type: 'text', text: `typed ${JSON.stringify(action.text)}` };
|
|
135
|
+
}
|
|
136
|
+
case 'key': {
|
|
137
|
+
await page.keyboard.press(normalizeChord(action.text));
|
|
138
|
+
return { type: 'text', text: `pressed ${action.text}` };
|
|
139
|
+
}
|
|
140
|
+
case 'hold_key': {
|
|
141
|
+
const key = normalizeKey(action.text);
|
|
142
|
+
await page.keyboard.down(key);
|
|
143
|
+
try {
|
|
144
|
+
await sleep(action.duration * 1000);
|
|
145
|
+
}
|
|
146
|
+
finally {
|
|
147
|
+
await page.keyboard.up(key);
|
|
148
|
+
}
|
|
149
|
+
return { type: 'text', text: `held ${action.text} for ${action.duration}s` };
|
|
150
|
+
}
|
|
151
|
+
case 'scroll': {
|
|
152
|
+
const [x, y] = action.coordinate;
|
|
153
|
+
const modifiers = parseModifiers(action.text);
|
|
154
|
+
await pressDown(page, modifiers);
|
|
155
|
+
try {
|
|
156
|
+
await page.mouse.move(x, y, { steps: MOUSE_MOVE_STEPS });
|
|
157
|
+
const px = action.scroll_amount * SCROLL_PIXELS_PER_CLICK;
|
|
158
|
+
let dx = 0;
|
|
159
|
+
let dy = 0;
|
|
160
|
+
switch (action.scroll_direction) {
|
|
161
|
+
case 'up':
|
|
162
|
+
dy = -px;
|
|
163
|
+
break;
|
|
164
|
+
case 'down':
|
|
165
|
+
dy = px;
|
|
166
|
+
break;
|
|
167
|
+
case 'left':
|
|
168
|
+
dx = -px;
|
|
169
|
+
break;
|
|
170
|
+
case 'right':
|
|
171
|
+
dx = px;
|
|
172
|
+
break;
|
|
173
|
+
}
|
|
174
|
+
await page.mouse.wheel(dx, dy);
|
|
175
|
+
}
|
|
176
|
+
finally {
|
|
177
|
+
await pressUp(page, modifiers);
|
|
178
|
+
}
|
|
179
|
+
state.cursor = { x, y };
|
|
180
|
+
return {
|
|
181
|
+
type: 'text',
|
|
182
|
+
text: `scrolled ${action.scroll_direction} ${action.scroll_amount} clicks at (${x}, ${y})`,
|
|
183
|
+
};
|
|
184
|
+
}
|
|
185
|
+
default: {
|
|
186
|
+
// Exhaustiveness guard — TS errors if a future action variant is
|
|
187
|
+
// added to ComputerAction without a handler here.
|
|
188
|
+
const _exhaustive = action;
|
|
189
|
+
throw new Error(`Unknown computer action: ${JSON.stringify(_exhaustive)}`);
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
catch (err) {
|
|
194
|
+
const text = err instanceof Error ? err.message : String(err);
|
|
195
|
+
return { type: 'error', text };
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
// ─── Helpers ──────────────────────────────────────────────
|
|
199
|
+
function sleep(ms) {
|
|
200
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
201
|
+
}
|
|
202
|
+
/**
|
|
203
|
+
* Parse Anthropic's `+`-separated modifier text into Playwright key
|
|
204
|
+
* names. Empty / undefined input → empty array (no modifiers).
|
|
205
|
+
*/
|
|
206
|
+
export function parseModifiers(text) {
|
|
207
|
+
if (!text)
|
|
208
|
+
return [];
|
|
209
|
+
return text
|
|
210
|
+
.split('+')
|
|
211
|
+
.map((s) => normalizeKey(s.trim()))
|
|
212
|
+
.filter(Boolean);
|
|
213
|
+
}
|
|
214
|
+
/**
|
|
215
|
+
* Map Anthropic / xdotool key names to Playwright key names.
|
|
216
|
+
* Unmapped keys (single chars, F-keys, anything we haven't covered)
|
|
217
|
+
* pass through verbatim — Playwright either accepts them or throws,
|
|
218
|
+
* which is the right failure mode.
|
|
219
|
+
*/
|
|
220
|
+
export function normalizeKey(key) {
|
|
221
|
+
switch (key.toLowerCase()) {
|
|
222
|
+
case 'ctrl':
|
|
223
|
+
case 'control': return 'Control';
|
|
224
|
+
case 'shift': return 'Shift';
|
|
225
|
+
case 'alt': return 'Alt';
|
|
226
|
+
case 'cmd':
|
|
227
|
+
case 'super':
|
|
228
|
+
case 'meta': return 'Meta';
|
|
229
|
+
case 'return':
|
|
230
|
+
case 'enter': return 'Enter';
|
|
231
|
+
case 'tab': return 'Tab';
|
|
232
|
+
case 'space': return 'Space';
|
|
233
|
+
case 'escape':
|
|
234
|
+
case 'esc': return 'Escape';
|
|
235
|
+
case 'backspace': return 'Backspace';
|
|
236
|
+
case 'delete':
|
|
237
|
+
case 'del': return 'Delete';
|
|
238
|
+
case 'up': return 'ArrowUp';
|
|
239
|
+
case 'down': return 'ArrowDown';
|
|
240
|
+
case 'left': return 'ArrowLeft';
|
|
241
|
+
case 'right': return 'ArrowRight';
|
|
242
|
+
case 'page_up':
|
|
243
|
+
case 'pageup': return 'PageUp';
|
|
244
|
+
case 'page_down':
|
|
245
|
+
case 'pagedown': return 'PageDown';
|
|
246
|
+
case 'home': return 'Home';
|
|
247
|
+
case 'end': return 'End';
|
|
248
|
+
default: return key;
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
/**
|
|
252
|
+
* Normalize a key chord (`+`-separated). Playwright's
|
|
253
|
+
* `keyboard.press()` parses chords natively, so we just normalize each
|
|
254
|
+
* segment and rejoin.
|
|
255
|
+
*/
|
|
256
|
+
export function normalizeChord(chord) {
|
|
257
|
+
return chord
|
|
258
|
+
.split('+')
|
|
259
|
+
.map((s) => normalizeKey(s.trim()))
|
|
260
|
+
.join('+');
|
|
261
|
+
}
|
|
262
|
+
async function pressDown(page, keys) {
|
|
263
|
+
for (const k of keys)
|
|
264
|
+
await page.keyboard.down(k);
|
|
265
|
+
}
|
|
266
|
+
async function pressUp(page, keys) {
|
|
267
|
+
for (const k of [...keys].reverse())
|
|
268
|
+
await page.keyboard.up(k);
|
|
269
|
+
}
|
|
270
|
+
//# sourceMappingURL=playwright.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"playwright.js","sourceRoot":"","sources":["../../src/computer-use/playwright.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AASH;;;GAGG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,GAAG,CAAA;AAE1C;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAA;AAEjC;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,IAAc,EACd,MAAsB,EACtB,KAA4B;IAE5B,IAAI,CAAC;QACH,QAAQ,MAAM,CAAC,MAAM,EAAE,CAAC;YACtB,KAAK,YAAY,CAAC,CAAC,CAAC;gBAClB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAA;gBACnD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,IAAI,EAAE,CAAA;YACzD,CAAC;YAED,KAAK,iBAAiB,CAAC,CAAC,CAAC;gBACvB,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,KAAK,CAAC,MAAM,CAAC,CAAC,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,CAAA;YAC3E,CAAC;YAED,KAAK,MAAM,CAAC,CAAC,CAAC;gBACZ,MAAM,KAAK,CAAC,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAA;gBACnC,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,MAAM,CAAC,QAAQ,GAAG,EAAE,CAAA;YAC7D,CAAC;YAED,KAAK,YAAY,CAAC,CAAC,CAAC;gBAClB,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,UAAU,CAAA;gBAChC,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,KAAK,EAAE,gBAAgB,EAAE,CAAC,CAAA;gBACxD,KAAK,CAAC,MAAM,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,CAAA;gBACvB,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,CAAC,KAAK,CAAC,GAAG,EAAE,CAAA;YACxD,CAAC;YAED,KAAK,YAAY,CAAC;YAClB,KAAK,aAAa,CAAC;YACnB,KAAK,cAAc,CAAC,CAAC,CAAC;gBACpB,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,KAAK,YAAY;oBAC3C,CAAC,CAAC,MAAM;oBACR,CAAC,CAAC,MAAM,CAAC,MAAM,KAAK,aAAa;wBACjC,CAAC,CAAC,OAAO;wBACT,CAAC,CAAC,QAAQ,CAAA;gBACZ,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,UAAU,CAAA;gBAChC,MAAM,SAAS,GAAG,cAAc,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;gBAC7C,MAAM,SAAS,CAAC,IAAI,EAAE,SAAS,CAAC,CAAA;gBAChC,IAAI,CAAC;oBACH,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,KAAK,EAAE,gBAAgB,EAAE,CAAC,CAAA;oBACxD,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,CAAA;gBAC1C,CAAC;wBAAS,CAAC;oBACT,MAAM,OAAO,CAAC,IAAI,EAAE,SAAS,CAAC,CAAA;gBAChC,CAAC;gBACD,KAAK,CAAC,MAAM,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,CAAA;gBACvB,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,gBAAgB,CAAC,KAAK,CAAC,GAAG,EAAE,CAAA;YACpE,CAAC;YAED,KAAK,cAAc,CAAC;YACpB,KAAK,cAAc,CAAC,CAAC,CAAC;gBACpB,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,KAAK,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;gBAC3D,MAAM,KAAK,GAAO,MAAM,CAAC,MAAM,KAAK,cAAc,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAA;gBACxE,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,UAAU,CAAA;gBAChC,MAAM,SAAS,GAAG,cAAc,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;gBAC7C,MAAM,SAAS,CAAC,IAAI,EAAE,SAAS,CAAC,CAAA;gBAChC,IAAI,CAAC;oBACH,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,KAAK,EAAE,gBAAgB,EAAE,CAAC,CAAA;oBACxD,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAA;gBAC9D,CAAC;wBAAS,CAAC;oBACT,MAAM,OAAO,CAAC,IAAI,EAAE,SAAS,CAAC,CAAA;gBAChC,CAAC;gBACD,KAAK,CAAC,MAAM,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,CAAA;gBACvB,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,KAAK,gBAAgB,CAAC,KAAK,CAAC,GAAG,EAAE,CAAA;YACnE,CAAC;YAED,KAAK,iBAAiB,CAAC,CAAC,CAAC;gBACvB,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;oBACtB,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,UAAU,CAAA;oBAChC,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,KAAK,EAAE,gBAAgB,EAAE,CAAC,CAAA;oBACxD,KAAK,CAAC,MAAM,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,CAAA;gBACzB,CAAC;gBACD,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAA;gBACzC,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,kBAAkB,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAA;YACvF,CAAC;YAED,KAAK,eAAe,CAAC,CAAC,CAAC;gBACrB,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;oBACtB,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,UAAU,CAAA;oBAChC,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,KAAK,EAAE,gBAAgB,EAAE,CAAC,CAAA;oBACxD,KAAK,CAAC,MAAM,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,CAAA;gBACzB,CAAC;gBACD,MAAM,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAA;gBACvC,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAA;YACrF,CAAC;YAED,KAAK,MAAM,CAAC,CAAC,CAAC;gBACZ,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;gBACrC,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,CAAA;YACvE,CAAC;YAED,KAAK,KAAK,CAAC,CAAC,CAAC;gBACX,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,cAAc,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAA;gBACtD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,MAAM,CAAC,IAAI,EAAE,EAAE,CAAA;YACzD,CAAC;YAED,KAAK,UAAU,CAAC,CAAC,CAAC;gBAChB,MAAM,GAAG,GAAG,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;gBACrC,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;gBAC7B,IAAI,CAAC;oBACH,MAAM,KAAK,CAAC,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAA;gBACrC,CAAC;wBAAS,CAAC;oBACT,MAAM,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,GAAG,CAAC,CAAA;gBAC7B,CAAC;gBACD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,MAAM,CAAC,IAAI,QAAQ,MAAM,CAAC,QAAQ,GAAG,EAAE,CAAA;YAC9E,CAAC;YAED,KAAK,QAAQ,CAAC,CAAC,CAAC;gBACd,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,UAAU,CAAA;gBAChC,MAAM,SAAS,GAAG,cAAc,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;gBAC7C,MAAM,SAAS,CAAC,IAAI,EAAE,SAAS,CAAC,CAAA;gBAChC,IAAI,CAAC;oBACH,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,KAAK,EAAE,gBAAgB,EAAE,CAAC,CAAA;oBACxD,MAAM,EAAE,GAAG,MAAM,CAAC,aAAa,GAAG,uBAAuB,CAAA;oBACzD,IAAI,EAAE,GAAG,CAAC,CAAA;oBACV,IAAI,EAAE,GAAG,CAAC,CAAA;oBACV,QAAQ,MAAM,CAAC,gBAAgB,EAAE,CAAC;wBAChC,KAAK,IAAI;4BAAK,EAAE,GAAG,CAAC,EAAE,CAAC;4BAAC,MAAK;wBAC7B,KAAK,MAAM;4BAAG,EAAE,GAAG,EAAE,CAAC;4BAAE,MAAK;wBAC7B,KAAK,MAAM;4BAAG,EAAE,GAAG,CAAC,EAAE,CAAC;4BAAC,MAAK;wBAC7B,KAAK,OAAO;4BAAE,EAAE,GAAG,EAAE,CAAC;4BAAE,MAAK;oBAC/B,CAAC;oBACD,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,CAAC,CAAA;gBAChC,CAAC;wBAAS,CAAC;oBACT,MAAM,OAAO,CAAC,IAAI,EAAE,SAAS,CAAC,CAAA;gBAChC,CAAC;gBACD,KAAK,CAAC,MAAM,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,CAAA;gBACvB,OAAO;oBACL,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,YAAY,MAAM,CAAC,gBAAgB,IAAI,MAAM,CAAC,aAAa,eAAe,CAAC,KAAK,CAAC,GAAG;iBAC3F,CAAA;YACH,CAAC;YAED,OAAO,CAAC,CAAC,CAAC;gBACR,iEAAiE;gBACjE,kDAAkD;gBAClD,MAAM,WAAW,GAAU,MAAM,CAAA;gBACjC,MAAM,IAAI,KAAK,CAAC,4BAA4B,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,EAAE,CAAC,CAAA;YAC5E,CAAC;QACH,CAAC;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,IAAI,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;QAC7D,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,CAAA;IAChC,CAAC;AACH,CAAC;AAED,6DAA6D;AAE7D,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAA;AAC1D,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,IAAwB;IACrD,IAAI,CAAC,IAAI;QAAE,OAAO,EAAE,CAAA;IACpB,OAAO,IAAI;SACR,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;SAClC,MAAM,CAAC,OAAO,CAAC,CAAA;AACpB,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,YAAY,CAAC,GAAW;IACtC,QAAQ,GAAG,CAAC,WAAW,EAAE,EAAE,CAAC;QAC1B,KAAK,MAAM,CAAC;QACZ,KAAK,SAAS,CAAC,CAAG,OAAO,SAAS,CAAA;QAClC,KAAK,OAAO,CAAC,CAAK,OAAO,OAAO,CAAA;QAChC,KAAK,KAAK,CAAC,CAAO,OAAO,KAAK,CAAA;QAC9B,KAAK,KAAK,CAAC;QACX,KAAK,OAAO,CAAC;QACb,KAAK,MAAM,CAAC,CAAM,OAAO,MAAM,CAAA;QAC/B,KAAK,QAAQ,CAAC;QACd,KAAK,OAAO,CAAC,CAAK,OAAO,OAAO,CAAA;QAChC,KAAK,KAAK,CAAC,CAAO,OAAO,KAAK,CAAA;QAC9B,KAAK,OAAO,CAAC,CAAK,OAAO,OAAO,CAAA;QAChC,KAAK,QAAQ,CAAC;QACd,KAAK,KAAK,CAAC,CAAO,OAAO,QAAQ,CAAA;QACjC,KAAK,WAAW,CAAC,CAAC,OAAO,WAAW,CAAA;QACpC,KAAK,QAAQ,CAAC;QACd,KAAK,KAAK,CAAC,CAAO,OAAO,QAAQ,CAAA;QACjC,KAAK,IAAI,CAAC,CAAQ,OAAO,SAAS,CAAA;QAClC,KAAK,MAAM,CAAC,CAAM,OAAO,WAAW,CAAA;QACpC,KAAK,MAAM,CAAC,CAAM,OAAO,WAAW,CAAA;QACpC,KAAK,OAAO,CAAC,CAAK,OAAO,YAAY,CAAA;QACrC,KAAK,SAAS,CAAC;QACf,KAAK,QAAQ,CAAC,CAAI,OAAO,QAAQ,CAAA;QACjC,KAAK,WAAW,CAAC;QACjB,KAAK,UAAU,CAAC,CAAE,OAAO,UAAU,CAAA;QACnC,KAAK,MAAM,CAAC,CAAM,OAAO,MAAM,CAAA;QAC/B,KAAK,KAAK,CAAC,CAAO,OAAO,KAAK,CAAA;QAC9B,OAAO,CAAC,CAAU,OAAO,GAAG,CAAA;IAC9B,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAAC,KAAa;IAC1C,OAAO,KAAK;SACT,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;SAClC,IAAI,CAAC,GAAG,CAAC,CAAA;AACd,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,IAAc,EAAE,IAAc;IACrD,KAAK,MAAM,CAAC,IAAI,IAAI;QAAE,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;AACnD,CAAC;AAED,KAAK,UAAU,OAAO,CAAC,IAAc,EAAE,IAAc;IACnD,KAAK,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,EAAE;QAAE,MAAM,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;AAChE,CAAC"}
|