appium-desktop-driver 1.6.0 → 1.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +77 -0
- package/build/lib/commands/index.d.ts +1 -1
- package/build/lib/commands/index.d.ts.map +1 -1
- package/build/lib/commands/vision.d.ts +1 -1
- package/build/lib/commands/vision.d.ts.map +1 -1
- package/build/lib/commands/vision.js +11 -2
- package/build/lib/commands/vision.js.map +1 -1
- package/build/lib/mcp/session.d.ts.map +1 -1
- package/build/lib/mcp/session.js +1 -0
- package/build/lib/mcp/session.js.map +1 -1
- package/build/lib/mcp/tools/vision.d.ts.map +1 -1
- package/build/lib/mcp/tools/vision.js +17 -6
- package/build/lib/mcp/tools/vision.js.map +1 -1
- package/build/lib/vision-utils.d.ts +4 -4
- package/build/lib/vision-utils.d.ts.map +1 -1
- package/build/lib/vision-utils.js +43 -6
- package/build/lib/vision-utils.js.map +1 -1
- package/build/tsconfig.tsbuildinfo +1 -1
- package/package.json +2 -1
package/README.md
CHANGED
|
@@ -633,6 +633,83 @@ if secondary:
|
|
|
633
633
|
driver.execute_script('windows: click', {'x': cx, 'y': cy})
|
|
634
634
|
```
|
|
635
635
|
|
|
636
|
+
### windows: findByVision
|
|
637
|
+
|
|
638
|
+
Uses a vision-capable LLM to locate a UI element by natural language
|
|
639
|
+
description. Takes a screenshot of the current window, sends it to the
|
|
640
|
+
configured LLM, and returns the screen coordinates of the described element.
|
|
641
|
+
|
|
642
|
+
The `model` argument controls which LLM provider is used. The corresponding
|
|
643
|
+
API key must be set as an environment variable before starting the Appium
|
|
644
|
+
server.
|
|
645
|
+
|
|
646
|
+
#### Supported Models
|
|
647
|
+
|
|
648
|
+
| Provider | Model prefix | Environment variable(s) | Example models |
|
|
649
|
+
| --- | --- | --- | --- |
|
|
650
|
+
| Anthropic | `claude-` | `ANTHROPIC_API_KEY` | `claude-opus-4-6`, `claude-sonnet-4-6` |
|
|
651
|
+
| OpenAI | `gpt-`, `o1`, `o3`, `o4` | `OPENAI_API_KEY` | `gpt-4o`, `o3` |
|
|
652
|
+
| Google | `gemini-` | `GEMINI_API_KEY` | `gemini-1.5-pro`, `gemini-2.0-flash` |
|
|
653
|
+
| Amazon (Bedrock) | `amazon.nova-`, `us.amazon.nova-`, `eu.amazon.nova-`, `ap.amazon.nova-` | `AWS_ACCESS_KEY_ID` + `AWS_SECRET_ACCESS_KEY` | `amazon.nova-pro-v1:0`, `amazon.nova-lite-v1:0` |
|
|
654
|
+
|
|
655
|
+
For Amazon Bedrock, set `AWS_REGION` (or `AWS_DEFAULT_REGION`) to select the deployment region (defaults to `us-east-1`). IAM roles and AWS credential profiles are also supported via the standard AWS credential chain.
|
|
656
|
+
|
|
657
|
+
#### Arguments
|
|
658
|
+
|
|
659
|
+
| Name | Type | Required | Description | Example |
|
|
660
|
+
| --- | --- | --- | --- | --- |
|
|
661
|
+
| prompt | string | yes | Natural language description of the element to find | `"Submit button"` |
|
|
662
|
+
| model | string | yes | LLM model identifier | `"claude-opus-4-6"`, `"gpt-4o"`, `"amazon.nova-pro-v1:0"` |
|
|
663
|
+
|
|
664
|
+
#### Returns
|
|
665
|
+
|
|
666
|
+
| Name | Type | Description |
|
|
667
|
+
| --- | --- | --- |
|
|
668
|
+
| x | number | Horizontal screen coordinate of the element center |
|
|
669
|
+
| y | number | Vertical screen coordinate of the element center |
|
|
670
|
+
| label | string | Brief description of the element as identified by the LLM |
|
|
671
|
+
|
|
672
|
+
#### Example
|
|
673
|
+
|
|
674
|
+
```javascript
|
|
675
|
+
// WebdriverIO — locate and click an element using Claude
|
|
676
|
+
const result = await driver.executeScript('windows: findByVision', [{
|
|
677
|
+
prompt: 'Save button in the toolbar',
|
|
678
|
+
model: 'claude-opus-4-6',
|
|
679
|
+
}]);
|
|
680
|
+
await driver.executeScript('windows: click', [{ x: result.x, y: result.y }]);
|
|
681
|
+
```
|
|
682
|
+
|
|
683
|
+
```javascript
|
|
684
|
+
// WebdriverIO — use GPT-4o instead
|
|
685
|
+
const result = await driver.executeScript('windows: findByVision', [{
|
|
686
|
+
prompt: 'Save button in the toolbar',
|
|
687
|
+
model: 'gpt-4o',
|
|
688
|
+
}]);
|
|
689
|
+
await driver.executeScript('windows: click', [{ x: result.x, y: result.y }]);
|
|
690
|
+
```
|
|
691
|
+
|
|
692
|
+
```javascript
|
|
693
|
+
// WebdriverIO — use Amazon Nova via Bedrock
|
|
694
|
+
const result = await driver.executeScript('windows: findByVision', [{
|
|
695
|
+
prompt: 'Save button in the toolbar',
|
|
696
|
+
model: 'amazon.nova-pro-v1:0',
|
|
697
|
+
}]);
|
|
698
|
+
await driver.executeScript('windows: click', [{ x: result.x, y: result.y }]);
|
|
699
|
+
```
|
|
700
|
+
|
|
701
|
+
```python
|
|
702
|
+
# Python — locate and click using Gemini
|
|
703
|
+
result = driver.execute_script('windows: findByVision', {
|
|
704
|
+
'prompt': 'Save button in the toolbar',
|
|
705
|
+
'model': 'gemini-1.5-pro',
|
|
706
|
+
})
|
|
707
|
+
driver.execute_script('windows: click', {
|
|
708
|
+
'x': result['x'],
|
|
709
|
+
'y': result['y'],
|
|
710
|
+
})
|
|
711
|
+
```
|
|
712
|
+
|
|
636
713
|
## Development
|
|
637
714
|
|
|
638
715
|
it is recommended to use Matt Bierner's [Comment tagged templates](https://marketplace.visualstudio.com/items?itemName=bierner.comment-tagged-templates)
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../lib/commands/index.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,OAAO,MAAM,WAAW,CAAC;AAOrC,QAAA,MAAM,QAAQ
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../lib/commands/index.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,OAAO,MAAM,WAAW,CAAC;AAOrC,QAAA,MAAM,QAAQ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;mBAoBg7W,CAAC;;;mBAAyqB,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;iBAAmiH,CAAC;SAAe,CAAC;SAAe,CAAC;cAAoB,CAAC;oBAA6B,CAAC;kBAA8G,CAAC;aAAmB,CAAC;yBAA+B,CAAC;;;sBAAmtG,CAAC;cAAoB,CAAC;cAAoB,CAAC;oBAA0B,CAAC;YAAkB,CAAC;YAAkB,CAAC;oBAA0B,CAAC;kBAA2F,CAAC;;;iBAA2yH,CAAC;SAAe,CAAC;SAAe,CAAC;cAAoB,CAAC;cAAoB,CAAC;oBAA0B,CAAC;;;kBAA8qF,CAAC;iBAAuB,CAAC;gBAAsB,CAAC;mBAAyB,CAAC;cAAoB,CAAC;qBAA2B,CAAC;qBAA4B,CAAC;kBAAyB,CAAC;oBAA0B,CAAC;;;;;;;;iBAAo5F,CAAC;;;sBAAouB,CAAC;cAAoB,CAAC;cAAoB,CAAC;oBAA0B,CAAC;YAAkB,CAAC;YAAkB,CAAC;oBAA0B,CAAC;kBAA2F,CAAC;cAAoB,CAAC;;;cAAu1I,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;0FAArlgC,wDAAiC;mFAAwtB,sDAA+B;;;;CAVh9F,CAAC;AAEF,KAAK,QAAQ,GAAG;KACX,GAAG,IAAI,MAAM,OAAO,QAAQ,GAAG,OAAO,QAAQ,CAAC,GAAG,CAAC;CACvD,CAAC;AAEF,OAAO,QAAQ,WAAW,CAAC;IACvB,UAAU,mBAAoB,SAAQ,QAAQ;KAAG;CACpD;AAED,eAAe,QAAQ,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vision.d.ts","sourceRoot":"","sources":["../../../lib/commands/vision.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,WAAW,CAAC;AA4ChD,wBAAsB,mBAAmB,CACrC,IAAI,EAAE,mBAAmB,EACzB,IAAI,EAAE;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,KAAK,
|
|
1
|
+
{"version":3,"file":"vision.d.ts","sourceRoot":"","sources":["../../../lib/commands/vision.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,WAAW,CAAC;AA4ChD,wBAAsB,mBAAmB,CACrC,IAAI,EAAE,mBAAmB,EACzB,IAAI,EAAE;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GACxC,OAAO,CAAC;IAAE,CAAC,EAAE,MAAM,CAAC;IAAC,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAgClD"}
|
|
@@ -15,12 +15,21 @@ async function buildCoordMapping(driver, ssW, ssH) {
|
|
|
15
15
|
return (0, vision_utils_1.computeCoordMapping)(false, rect.x, rect.y, rect.width, rect.height, (0, user32_1.getResolutionScalingFactor)(), ssW, ssH);
|
|
16
16
|
}
|
|
17
17
|
async function executeFindByVision(args) {
|
|
18
|
-
|
|
19
|
-
|
|
18
|
+
if (!args.model) {
|
|
19
|
+
throw new Error('windows: findByVision requires a "model" argument. ' +
|
|
20
|
+
'Supported prefixes: claude-* (ANTHROPIC_API_KEY), gpt-*/o-series (OPENAI_API_KEY), ' +
|
|
21
|
+
'gemini-* (GEMINI_API_KEY), amazon.nova-*/us.amazon.nova-*/eu.amazon.nova-*/ap.amazon.nova-* (AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY).');
|
|
22
|
+
}
|
|
23
|
+
const model = args.model;
|
|
24
|
+
const provider = (0, vision_utils_1.getProviderForModel)(model);
|
|
25
|
+
const envVar = (0, vision_utils_1.getApiKeyEnvVar)(provider);
|
|
20
26
|
const apiKey = process.env[envVar];
|
|
21
27
|
if (!apiKey) {
|
|
22
28
|
throw new Error(`${envVar} environment variable is required for windows: findByVision (model: ${model})`);
|
|
23
29
|
}
|
|
30
|
+
if (provider === 'amazon' && !process.env.AWS_SECRET_ACCESS_KEY) {
|
|
31
|
+
throw new Error('AWS_SECRET_ACCESS_KEY environment variable is required for Amazon Bedrock models');
|
|
32
|
+
}
|
|
24
33
|
const base64 = await this.getScreenshot();
|
|
25
34
|
const { width: ssW, height: ssH } = (0, util_1.getPngDimensions)(base64);
|
|
26
35
|
const raw = await (0, vision_utils_1.callVisionLLM)(base64, (0, vision_utils_1.buildVisionPrompt)(args.prompt, ssW, ssH), model, apiKey);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vision.js","sourceRoot":"","sources":["../../../lib/commands/vision.ts"],"names":[],"mappings":";;AA4CA,
|
|
1
|
+
{"version":3,"file":"vision.js","sourceRoot":"","sources":["../../../lib/commands/vision.ts"],"names":[],"mappings":";;AA4CA,kDAmCC;AA9ED,kCAA2C;AAC3C,6CAA8D;AAC9D,kDASyB;AAEzB,KAAK,UAAU,iBAAiB,CAC5B,MAA2B,EAC3B,GAAW,EACX,GAAW;IAEX,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,aAAa,EAAE,CAAC;IAC1C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IAElC,IAAI,MAAM,EAAE,CAAC;QACT,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,kBAAkB,EAG9C,CAAC;QACH,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,QAAQ,CAAC,CAAC,CAAC,CAAC;QAC/D,OAAO,IAAA,kCAAmB,EACtB,IAAI,EACJ,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,EACvC,CAAC,EAAE,GAAG,EAAE,GAAG,EACX,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,CAClD,CAAC;IACN,CAAC;IAED,OAAO,IAAA,kCAAmB,EACtB,KAAK,EACL,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,EACvC,IAAA,mCAA0B,GAAE,EAC5B,GAAG,EAAE,GAAG,CACX,CAAC;AACN,CAAC;AAEM,KAAK,UAAU,mBAAmB,CAErC,IAAuC;IAEvC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;QACd,MAAM,IAAI,KAAK,CACX,qDAAqD;YACrD,qFAAqF;YACrF,0IAA0I,CAC7I,CAAC;IACN,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;IACzB,MAAM,QAAQ,GAAG,IAAA,kCAAmB,EAAC,KAAK,CAAC,CAAC;IAC5C,MAAM,MAAM,GAAG,IAAA,8BAAe,EAAC,QAAQ,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACnC,IAAI,CAAC,MAAM,EAAE,CAAC;QACV,MAAM,IAAI,KAAK,CACX,GAAG,MAAM,uEAAuE,KAAK,GAAG,CAC3F,CAAC;IACN,CAAC;IACD,IAAI,QAAQ,KAAK,QAAQ,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,qBAAqB,EAAE,CAAC;QAC9D,MAAM,IAAI,KAAK,CAAC,kFAAkF,CAAC,CAAC;IACxG,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;IAC1C,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,IAAA,uBAAgB,EAAC,MAAM,CAAC,CAAC;IAE7D,MAAM,GAAG,GAAG,MAAM,IAAA,4BAAa,EAAC,MAAM,EAAE,IAAA,gCAAiB,EAAC,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;IACjG,MAAM,MAAM,GAAG,IAAA,gCAAiB,EAAC,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IAEnD,MAAM,OAAO,GAAG,MAAM,iBAAiB,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;IACxD,OAAO;QACH,GAAG,IAAA,gCAAiB,EAAC,OAAO,EAAE,MAAM,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;QACjD,KAAK,EAAE,MAAM,CAAC,KAAK;KACtB,CAAC;AACN,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../../lib/mcp/session.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAE7C,4EAA4E;AAC5E,MAAM,WAAW,aAAa;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED,qBAAa,aAAa;IAGV,OAAO,CAAC,QAAQ,CAAC,YAAY;IAFzC,OAAO,CAAC,MAAM,CAAwB;gBAET,YAAY,EAAE,SAAS;IAE9C,MAAM,CAAC,MAAM,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../../lib/mcp/session.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAE7C,4EAA4E;AAC5E,MAAM,WAAW,aAAa;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED,qBAAa,aAAa;IAGV,OAAO,CAAC,QAAQ,CAAC,YAAY;IAFzC,OAAO,CAAC,MAAM,CAAwB;gBAET,YAAY,EAAE,SAAS;IAE9C,MAAM,CAAC,MAAM,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IAiC5C,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAY7B,QAAQ,IAAI,OAAO;IAInB,SAAS,IAAI,OAAO;CAIvB"}
|
package/build/lib/mcp/session.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session.js","sourceRoot":"","sources":["../../../lib/mcp/session.ts"],"names":[],"mappings":";;;AAAA,6CAAqC;AAiBrC,MAAa,aAAa;IAGO;IAFrB,MAAM,GAAmB,IAAI,CAAC;IAEtC,YAA6B,YAAuB;QAAvB,iBAAY,GAAZ,YAAY,CAAW;IAAG,CAAC;IAExD,KAAK,CAAC,MAAM,CAAC,MAAqB;QAC9B,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;QAC/E,CAAC;QAED,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,0CAA0C,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC;QAE/E,MAAM,IAAI,GAA4B;YAClC,YAAY,EAAE,SAAS;YACvB,uBAAuB,EAAE,eAAe;YACxC,YAAY,EAAE,MAAM,CAAC,GAAG;SAC3B,CAAC;QAEF,IAAI,MAAM,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;YAAA,IAAI,CAAC,qBAAqB,CAAC,GAAG,MAAM,CAAC,YAAY,CAAC;QAAA,CAAC;QAC3F,IAAI,MAAM,CAAC,aAAa,KAAK,SAAS,EAAE,CAAC;YAAA,IAAI,CAAC,sBAAsB,CAAC,GAAG,MAAM,CAAC,aAAa,CAAC;QAAA,CAAC;QAC9F,IAAI,MAAM,CAAC,gBAAgB,KAAK,SAAS,EAAE,CAAC;YAAA,IAAI,CAAC,yBAAyB,CAAC,GAAG,MAAM,CAAC,gBAAgB,CAAC;QAAA,CAAC;QACvG,IAAI,MAAM,CAAC,cAAc,KAAK,SAAS,EAAE,CAAC;YAAA,IAAI,CAAC,uBAAuB,CAAC,GAAG,MAAM,CAAC,cAAc,CAAC;QAAA,CAAC;QACjG,IAAI,MAAM,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;YAAA,IAAI,CAAC,wBAAwB,CAAC,GAAG,MAAM,CAAC,eAAe,CAAC;QAAA,CAAC;QACpG,IAAI,MAAM,CAAC,gBAAgB,KAAK,SAAS,EAAE,CAAC;YAAA,IAAI,CAAC,yBAAyB,CAAC,GAAG,MAAM,CAAC,gBAAgB,CAAC;QAAA,CAAC;QACvG,IAAI,MAAM,CAAC,iBAAiB,KAAK,SAAS,EAAE,CAAC;YAAA,IAAI,CAAC,0BAA0B,CAAC,GAAG,MAAM,CAAC,iBAAiB,CAAC;QAAA,CAAC;QAE1G,IAAI,CAAC,MAAM,GAAG,MAAM,IAAA,oBAAM,EAAC;YACvB,QAAQ,EAAE,IAAI,CAAC,YAAY,CAAC,UAAU;YACtC,IAAI,EAAE,IAAI,CAAC,YAAY,CAAC,UAAU;YAClC,IAAI,EAAE,GAAG;YACT,YAAY,EAAE,IAAgC;SACjD,CAAC,CAAC;QAEH,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC;QACnE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,sCAAsC,CAAC,CAAC;IACjE,CAAC;IAED,KAAK,CAAC,MAAM;QACR,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YAAA,OAAO;QAAA,CAAC;QAC3B,IAAI,CAAC;YACD,MAAM,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,CAAC;YAClC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;QACpD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,yCAAyC,GAAG,IAAI,CAAC,CAAC;QAC3E,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACvB,CAAC;IACL,CAAC;IAED,QAAQ;QACJ,OAAO,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC;IAChC,CAAC;IAED,SAAS;QACL,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YAAA,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;QAAA,CAAC;QACrF,OAAO,IAAI,CAAC,MAAM,CAAC;IACvB,CAAC;CACJ;
|
|
1
|
+
{"version":3,"file":"session.js","sourceRoot":"","sources":["../../../lib/mcp/session.ts"],"names":[],"mappings":";;;AAAA,6CAAqC;AAiBrC,MAAa,aAAa;IAGO;IAFrB,MAAM,GAAmB,IAAI,CAAC;IAEtC,YAA6B,YAAuB;QAAvB,iBAAY,GAAZ,YAAY,CAAW;IAAG,CAAC;IAExD,KAAK,CAAC,MAAM,CAAC,MAAqB;QAC9B,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;QAC/E,CAAC;QAED,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,0CAA0C,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC;QAE/E,MAAM,IAAI,GAA4B;YAClC,YAAY,EAAE,SAAS;YACvB,uBAAuB,EAAE,eAAe;YACxC,YAAY,EAAE,MAAM,CAAC,GAAG;SAC3B,CAAC;QAEF,IAAI,MAAM,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;YAAA,IAAI,CAAC,qBAAqB,CAAC,GAAG,MAAM,CAAC,YAAY,CAAC;QAAA,CAAC;QAC3F,IAAI,MAAM,CAAC,aAAa,KAAK,SAAS,EAAE,CAAC;YAAA,IAAI,CAAC,sBAAsB,CAAC,GAAG,MAAM,CAAC,aAAa,CAAC;QAAA,CAAC;QAC9F,IAAI,MAAM,CAAC,gBAAgB,KAAK,SAAS,EAAE,CAAC;YAAA,IAAI,CAAC,yBAAyB,CAAC,GAAG,MAAM,CAAC,gBAAgB,CAAC;QAAA,CAAC;QACvG,IAAI,MAAM,CAAC,cAAc,KAAK,SAAS,EAAE,CAAC;YAAA,IAAI,CAAC,uBAAuB,CAAC,GAAG,MAAM,CAAC,cAAc,CAAC;QAAA,CAAC;QACjG,IAAI,MAAM,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;YAAA,IAAI,CAAC,wBAAwB,CAAC,GAAG,MAAM,CAAC,eAAe,CAAC;QAAA,CAAC;QACpG,IAAI,MAAM,CAAC,gBAAgB,KAAK,SAAS,EAAE,CAAC;YAAA,IAAI,CAAC,yBAAyB,CAAC,GAAG,MAAM,CAAC,gBAAgB,CAAC;QAAA,CAAC;QACvG,IAAI,MAAM,CAAC,iBAAiB,KAAK,SAAS,EAAE,CAAC;YAAA,IAAI,CAAC,0BAA0B,CAAC,GAAG,MAAM,CAAC,iBAAiB,CAAC;QAAA,CAAC;QAE1G,IAAI,CAAC,MAAM,GAAG,MAAM,IAAA,oBAAM,EAAC;YACvB,QAAQ,EAAE,IAAI,CAAC,YAAY,CAAC,UAAU;YACtC,IAAI,EAAE,IAAI,CAAC,YAAY,CAAC,UAAU;YAClC,IAAI,EAAE,GAAG;YACT,QAAQ,EAAE,QAAQ;YAClB,YAAY,EAAE,IAAgC;SACjD,CAAC,CAAC;QAEH,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC;QACnE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,sCAAsC,CAAC,CAAC;IACjE,CAAC;IAED,KAAK,CAAC,MAAM;QACR,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YAAA,OAAO;QAAA,CAAC;QAC3B,IAAI,CAAC;YACD,MAAM,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,CAAC;YAClC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;QACpD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,yCAAyC,GAAG,IAAI,CAAC,CAAC;QAC3E,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACvB,CAAC;IACL,CAAC;IAED,QAAQ;QACJ,OAAO,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC;IAChC,CAAC;IAED,SAAS;QACL,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YAAA,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;QAAA,CAAC;QACrF,OAAO,IAAI,CAAC,MAAM,CAAC;IACvB,CAAC;CACJ;AA1DD,sCA0DC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vision.d.ts","sourceRoot":"","sources":["../../../../lib/mcp/tools/vision.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AAGpE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"vision.d.ts","sourceRoot":"","sources":["../../../../lib/mcp/tools/vision.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AAGpE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AA0CnD,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,aAAa,GAAG,IAAI,CA6EnF"}
|
|
@@ -5,7 +5,6 @@ const zod_1 = require("zod");
|
|
|
5
5
|
const errors_js_1 = require("../errors.js");
|
|
6
6
|
const util_1 = require("../../util");
|
|
7
7
|
const vision_utils_1 = require("../../vision-utils");
|
|
8
|
-
const DEFAULT_MODEL = 'claude-opus-4-6';
|
|
9
8
|
async function buildCoordMapping(driver, ssW, ssH) {
|
|
10
9
|
try {
|
|
11
10
|
const rect = await driver.getWindowRect();
|
|
@@ -31,23 +30,35 @@ function registerVisionTools(server, session) {
|
|
|
31
30
|
'For "coordinates" format, locates a UI element and returns {x,y,label} with actual screen ' +
|
|
32
31
|
'coordinates (DPI-corrected) ready to pass to click tools. ' +
|
|
33
32
|
'For "text" format, answers a general question about the screen in plain text. ' +
|
|
34
|
-
'Requires ANTHROPIC_API_KEY (Claude), OPENAI_API_KEY (GPT-4o / o-series),
|
|
35
|
-
'GEMINI_API_KEY (Gemini)
|
|
33
|
+
'Requires ANTHROPIC_API_KEY (Claude), OPENAI_API_KEY (GPT-4o / o-series), ' +
|
|
34
|
+
'GEMINI_API_KEY (Gemini), or AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY (Amazon Nova via Bedrock) ' +
|
|
35
|
+
'depending on the chosen model.',
|
|
36
36
|
inputSchema: {
|
|
37
37
|
prompt: zod_1.z.string().min(1).describe('Question or instruction about the screenshot'),
|
|
38
38
|
responseFormat: zod_1.z.enum(['coordinates', 'text']).default('coordinates').describe('"coordinates" (default) locates an element and returns JSON {x,y,label} with converted screen coordinates. ' +
|
|
39
39
|
'"text" answers a general question about the screen in plain text.'),
|
|
40
|
-
model: zod_1.z.string().
|
|
40
|
+
model: zod_1.z.string().min(1).describe('Vision model to use. Determines which credentials are required: ' +
|
|
41
|
+
'claude-* → ANTHROPIC_API_KEY, gpt-*/o-series → OPENAI_API_KEY, ' +
|
|
42
|
+
'gemini-* → GEMINI_API_KEY, amazon.nova-* → AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY.'),
|
|
41
43
|
},
|
|
42
44
|
annotations: { readOnlyHint: true },
|
|
43
45
|
}, async ({ prompt, responseFormat, model }) => {
|
|
44
46
|
try {
|
|
45
|
-
|
|
46
|
-
|
|
47
|
+
if (!model) {
|
|
48
|
+
throw new Error('find_by_vision requires a "model" argument. ' +
|
|
49
|
+
'Supported prefixes: claude-* (ANTHROPIC_API_KEY), gpt-*/o-series (OPENAI_API_KEY), ' +
|
|
50
|
+
'gemini-* (GEMINI_API_KEY), amazon.nova-*/us.amazon.nova-*/eu.amazon.nova-*/ap.amazon.nova-* (AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY).');
|
|
51
|
+
}
|
|
52
|
+
const visionModel = model;
|
|
53
|
+
const provider = (0, vision_utils_1.getProviderForModel)(visionModel);
|
|
54
|
+
const envVar = (0, vision_utils_1.getApiKeyEnvVar)(provider);
|
|
47
55
|
const apiKey = process.env[envVar];
|
|
48
56
|
if (!apiKey) {
|
|
49
57
|
throw new Error(`${envVar} environment variable is required for find_by_vision (model: ${visionModel})`);
|
|
50
58
|
}
|
|
59
|
+
if (provider === 'amazon' && !process.env.AWS_SECRET_ACCESS_KEY) {
|
|
60
|
+
throw new Error('AWS_SECRET_ACCESS_KEY environment variable is required for Amazon Bedrock models');
|
|
61
|
+
}
|
|
51
62
|
const driver = session.getDriver();
|
|
52
63
|
const base64 = await driver.takeScreenshot();
|
|
53
64
|
const { width: ssW, height: ssH } = (0, util_1.getPngDimensions)(base64);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vision.js","sourceRoot":"","sources":["../../../../lib/mcp/tools/vision.ts"],"names":[],"mappings":";;
|
|
1
|
+
{"version":3,"file":"vision.js","sourceRoot":"","sources":["../../../../lib/mcp/tools/vision.ts"],"names":[],"mappings":";;AA6CA,kDA6EC;AAzHD,6BAAwB;AAGxB,4CAA2C;AAC3C,qCAA8C;AAC9C,qDAS4B;AAE5B,KAAK,UAAU,iBAAiB,CAAC,MAAe,EAAE,GAAW,EAAE,GAAW;IACtE,IAAI,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,aAAa,EAAE,CAAC;QAC1C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QAElC,IAAI,MAAM,EAAE,CAAC;YACT,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,sBAAsB,EAAE,EAAE,CAAU,CAAC;YACjF,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,QAAQ,CAAC,CAAC,CAAC,CAAC;YACpE,IAAI,CAAC,OAAO,EAAE,CAAC;gBAAC,OAAO,SAAS,CAAC;YAAC,CAAC;YACnC,OAAO,IAAA,kCAAmB,EACtB,IAAI,EACJ,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,EACvC,CAAC,EAAE,GAAG,EAAE,GAAG,EACX,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM,CAC9C,CAAC;QACN,CAAC;QAED,MAAM,QAAQ,GAAG,CAAC,MAAM,MAAM,CAAC,aAAa,CAAC,sBAAsB,EAAE,EAAE,CAAC,CAAW,CAAC;QACpF,OAAO,IAAA,kCAAmB,EACtB,KAAK,EACL,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,EACvC,QAAQ,EAAE,GAAG,EAAE,GAAG,CACrB,CAAC;IACN,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,SAAS,CAAC;IACrB,CAAC;AACL,CAAC;AAED,SAAgB,mBAAmB,CAAC,MAAiB,EAAE,OAAsB;IACzE,MAAM,CAAC,YAAY,CACf,gBAAgB,EAChB;QACI,WAAW,EACP,uFAAuF;YACvF,4FAA4F;YAC5F,4DAA4D;YAC5D,gFAAgF;YAChF,2EAA2E;YAC3E,kGAAkG;YAClG,gCAAgC;QACpC,WAAW,EAAE;YACT,MAAM,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,8CAA8C,CAAC;YAClF,cAAc,EAAE,OAAC,CAAC,IAAI,CAAC,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,QAAQ,CAC3E,6GAA6G;gBAC7G,mEAAmE,CACtE;YACD,KAAK,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAC7B,kEAAkE;gBAClE,iEAAiE;gBACjE,uFAAuF,CAC1F;SACJ;QACD,WAAW,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE;KACtC,EACD,KAAK,EAAE,EAAE,MAAM,EAAE,cAAc,EAAE,KAAK,EAAE,EAAE,EAAE;QACxC,IAAI,CAAC;YACD,IAAI,CAAC,KAAK,EAAE,CAAC;gBACT,MAAM,IAAI,KAAK,CACX,8CAA8C;oBAC9C,qFAAqF;oBACrF,0IAA0I,CAC7I,CAAC;YACN,CAAC;YACD,MAAM,WAAW,GAAG,KAAK,CAAC;YAC1B,MAAM,QAAQ,GAAG,IAAA,kCAAmB,EAAC,WAAW,CAAC,CAAC;YAClD,MAAM,MAAM,GAAG,IAAA,8BAAe,EAAC,QAAQ,CAAC,CAAC;YACzC,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACnC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACV,MAAM,IAAI,KAAK,CACX,GAAG,MAAM,gEAAgE,WAAW,GAAG,CAC1F,CAAC;YACN,CAAC;YACD,IAAI,QAAQ,KAAK,QAAQ,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,qBAAqB,EAAE,CAAC;gBAC9D,MAAM,IAAI,KAAK,CAAC,kFAAkF,CAAC,CAAC;YACxG,CAAC;YAED,MAAM,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;YACnC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,cAAc,EAAY,CAAC;YACvD,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,IAAA,uBAAgB,EAAC,MAAM,CAAC,CAAC;YAE7D,IAAI,cAAc,KAAK,MAAM,EAAE,CAAC;gBAC5B,MAAM,UAAU,GAAG,gDAAgD,MAAM,6BAA6B,CAAC;gBACvG,MAAM,IAAI,GAAG,MAAM,IAAA,4BAAa,EAAC,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;gBAChF,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;YAC1D,CAAC;YAED,sFAAsF;YACtF,MAAM,GAAG,GAAG,MAAM,IAAA,4BAAa,EAAC,MAAM,EAAE,IAAA,gCAAiB,EAAC,MAAM,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;YAClG,MAAM,MAAM,GAAG,IAAA,gCAAiB,EAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YAC9C,MAAM,OAAO,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;YAC1D,MAAM,MAAM,GAAG,OAAO;gBAClB,CAAC,CAAC,IAAA,gCAAiB,EAAC,OAAO,EAAE,MAAM,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;gBAChD,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC;YAEnC,OAAO;gBACH,OAAO,EAAE,CAAC;wBACN,IAAI,EAAE,MAAe;wBACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC;qBAC3D,CAAC;aACL,CAAC;QACN,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,IAAA,uBAAW,EAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;QAC3F,CAAC;IACL,CAAC,CACJ,CAAC;AACN,CAAC"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export type LLMProvider = 'anthropic' | 'openai' | 'google';
|
|
1
|
+
export type LLMProvider = 'anthropic' | 'openai' | 'google' | 'amazon';
|
|
2
2
|
export declare function getProviderForModel(model: string): LLMProvider;
|
|
3
3
|
/** Returns the environment variable name that holds the API key for the given provider. */
|
|
4
4
|
export declare function getApiKeyEnvVar(provider: LLMProvider): string;
|
|
@@ -30,9 +30,9 @@ export declare function parseVisionCoords(raw: string, prompt: string): {
|
|
|
30
30
|
};
|
|
31
31
|
/**
|
|
32
32
|
* Sends a base64 screenshot + text prompt to a vision model and returns the raw
|
|
33
|
-
* text response. Dispatches to Anthropic, OpenAI,
|
|
34
|
-
* model name prefix. The caller is responsible for building the prompt
|
|
35
|
-
* parsing the result.
|
|
33
|
+
* text response. Dispatches to Anthropic, OpenAI, Google Gemini, or Amazon Bedrock
|
|
34
|
+
* based on the model name prefix. The caller is responsible for building the prompt
|
|
35
|
+
* and parsing the result.
|
|
36
36
|
*/
|
|
37
37
|
export declare function callVisionLLM(base64: string, textPrompt: string, model: string, apiKey: string, maxTokens?: number): Promise<string>;
|
|
38
38
|
//# sourceMappingURL=vision-utils.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vision-utils.d.ts","sourceRoot":"","sources":["../../lib/vision-utils.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"vision-utils.d.ts","sourceRoot":"","sources":["../../lib/vision-utils.ts"],"names":[],"mappings":"AAGA,MAAM,MAAM,WAAW,GAAG,WAAW,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,CAAC;AAcvE,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW,CAkB9D;AAED,2FAA2F;AAC3F,wBAAgB,eAAe,CAAC,QAAQ,EAAE,WAAW,GAAG,MAAM,CAO7D;AAED,MAAM,WAAW,YAAY;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,gFAAgF;IAChF,MAAM,EAAE,MAAM,CAAC;IACf,wCAAwC;IACxC,MAAM,EAAE,MAAM,CAAC;CAClB;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CAC/B,MAAM,EAAE,OAAO,EACf,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,GAAG,EAAE,MAAM,EACX,GAAG,EAAE,MAAM,EACX,QAAQ,CAAC,EAAE,MAAM,EACjB,QAAQ,CAAC,EAAE,MAAM,GAClB,YAAY,CAqBd;AAED,0EAA0E;AAC1E,wBAAgB,iBAAiB,CAC7B,OAAO,EAAE,YAAY,EACrB,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,MAAM,GACb;IAAE,CAAC,EAAE,MAAM,CAAC;IAAC,CAAC,EAAE,MAAM,CAAA;CAAE,CAK1B;AAED,sEAAsE;AACtE,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAWlF;AAED,sFAAsF;AACtF,wBAAgB,iBAAiB,CAC7B,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,MAAM,GACf;IAAE,CAAC,EAAE,MAAM,CAAC;IAAC,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAUzC;AAgJD;;;;;GAKG;AACH,wBAAsB,aAAa,CAC/B,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EACd,SAAS,SAAM,GAChB,OAAO,CAAC,MAAM,CAAC,CAQjB"}
|
|
@@ -11,12 +11,17 @@ exports.buildVisionPrompt = buildVisionPrompt;
|
|
|
11
11
|
exports.parseVisionCoords = parseVisionCoords;
|
|
12
12
|
exports.callVisionLLM = callVisionLLM;
|
|
13
13
|
const sdk_1 = __importDefault(require("@anthropic-ai/sdk"));
|
|
14
|
+
const client_bedrock_runtime_1 = require("@aws-sdk/client-bedrock-runtime");
|
|
14
15
|
/** Infers the LLM provider from the model identifier. */
|
|
15
16
|
const SUPPORTED_MODELS = [
|
|
16
|
-
'claude-*
|
|
17
|
-
'gpt-*
|
|
17
|
+
'claude-* (e.g. claude-sonnet-4-6)',
|
|
18
|
+
'gpt-* (e.g. gpt-4o)',
|
|
18
19
|
'o1, o3, o4, o1-mini, o3-pro, …',
|
|
19
|
-
'gemini-*
|
|
20
|
+
'gemini-* (e.g. gemini-1.5-pro)',
|
|
21
|
+
'amazon.nova-* (e.g. amazon.nova-pro-v1:0)',
|
|
22
|
+
'us.amazon.nova-* (cross-region inference, e.g. us.amazon.nova-pro-v1:0)',
|
|
23
|
+
'eu.amazon.nova-* (cross-region inference)',
|
|
24
|
+
'ap.amazon.nova-* (cross-region inference)',
|
|
20
25
|
];
|
|
21
26
|
function getProviderForModel(model) {
|
|
22
27
|
const lower = model.toLowerCase();
|
|
@@ -29,6 +34,9 @@ function getProviderForModel(model) {
|
|
|
29
34
|
if (lower.startsWith('claude-')) {
|
|
30
35
|
return 'anthropic';
|
|
31
36
|
}
|
|
37
|
+
if (/^(us\.|eu\.|ap\.)?amazon\./.test(lower)) {
|
|
38
|
+
return 'amazon';
|
|
39
|
+
}
|
|
32
40
|
throw new Error(`Unsupported model: "${model}". ` +
|
|
33
41
|
`Supported model prefixes are:\n ${SUPPORTED_MODELS.join('\n ')}`);
|
|
34
42
|
}
|
|
@@ -37,6 +45,7 @@ function getApiKeyEnvVar(provider) {
|
|
|
37
45
|
switch (provider) {
|
|
38
46
|
case 'openai': return 'OPENAI_API_KEY';
|
|
39
47
|
case 'google': return 'GEMINI_API_KEY';
|
|
48
|
+
case 'amazon': return 'AWS_ACCESS_KEY_ID';
|
|
40
49
|
default: return 'ANTHROPIC_API_KEY';
|
|
41
50
|
}
|
|
42
51
|
}
|
|
@@ -183,17 +192,45 @@ async function callGoogleVision(base64, textPrompt, model, apiKey, maxTokens) {
|
|
|
183
192
|
}
|
|
184
193
|
return text;
|
|
185
194
|
}
|
|
195
|
+
async function callAmazonBedrockVision(base64, textPrompt, model, maxTokens) {
|
|
196
|
+
const client = new client_bedrock_runtime_1.BedrockRuntimeClient({
|
|
197
|
+
region: process.env.AWS_REGION ?? process.env.AWS_DEFAULT_REGION ?? 'us-east-1',
|
|
198
|
+
});
|
|
199
|
+
const command = new client_bedrock_runtime_1.ConverseCommand({
|
|
200
|
+
modelId: model,
|
|
201
|
+
messages: [{
|
|
202
|
+
role: 'user',
|
|
203
|
+
content: [
|
|
204
|
+
{
|
|
205
|
+
image: {
|
|
206
|
+
format: 'png',
|
|
207
|
+
source: { bytes: Buffer.from(base64, 'base64') },
|
|
208
|
+
},
|
|
209
|
+
},
|
|
210
|
+
{ text: textPrompt },
|
|
211
|
+
],
|
|
212
|
+
}],
|
|
213
|
+
inferenceConfig: { maxTokens },
|
|
214
|
+
});
|
|
215
|
+
const response = await client.send(command);
|
|
216
|
+
const text = response.output?.message?.content?.find((b) => 'text' in b && typeof b.text === 'string');
|
|
217
|
+
if (!text) {
|
|
218
|
+
throw new Error(`Unexpected response from Amazon Bedrock model "${model}": no text content in output`);
|
|
219
|
+
}
|
|
220
|
+
return text.text;
|
|
221
|
+
}
|
|
186
222
|
/**
|
|
187
223
|
* Sends a base64 screenshot + text prompt to a vision model and returns the raw
|
|
188
|
-
* text response. Dispatches to Anthropic, OpenAI,
|
|
189
|
-
* model name prefix. The caller is responsible for building the prompt
|
|
190
|
-
* parsing the result.
|
|
224
|
+
* text response. Dispatches to Anthropic, OpenAI, Google Gemini, or Amazon Bedrock
|
|
225
|
+
* based on the model name prefix. The caller is responsible for building the prompt
|
|
226
|
+
* and parsing the result.
|
|
191
227
|
*/
|
|
192
228
|
async function callVisionLLM(base64, textPrompt, model, apiKey, maxTokens = 256) {
|
|
193
229
|
const provider = getProviderForModel(model);
|
|
194
230
|
switch (provider) {
|
|
195
231
|
case 'openai': return callOpenAIVision(base64, textPrompt, model, apiKey, maxTokens);
|
|
196
232
|
case 'google': return callGoogleVision(base64, textPrompt, model, apiKey, maxTokens);
|
|
233
|
+
case 'amazon': return callAmazonBedrockVision(base64, textPrompt, model, maxTokens);
|
|
197
234
|
default: return callAnthropicVision(base64, textPrompt, model, apiKey, maxTokens);
|
|
198
235
|
}
|
|
199
236
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vision-utils.js","sourceRoot":"","sources":["../../lib/vision-utils.ts"],"names":[],"mappings":";;;;;
|
|
1
|
+
{"version":3,"file":"vision-utils.js","sourceRoot":"","sources":["../../lib/vision-utils.ts"],"names":[],"mappings":";;;;;AAiBA,kDAkBC;AAGD,0CAOC;AAeD,kDAgCC;AAGD,8CASC;AAGD,8CAWC;AAGD,8CAaC;AAsJD,sCAcC;AA1SD,4DAA0C;AAC1C,4EAAwF;AAIxF,yDAAyD;AACzD,MAAM,gBAAgB,GAAG;IACrB,2CAA2C;IAC3C,gCAAgC;IAChC,gCAAgC;IAChC,wCAAwC;IACxC,8CAA8C;IAC9C,yEAAyE;IACzE,2CAA2C;IAC3C,2CAA2C;CAC9C,CAAC;AAEF,SAAgB,mBAAmB,CAAC,KAAa;IAC7C,MAAM,KAAK,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC;IAClC,IAAI,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACjD,OAAO,QAAQ,CAAC;IACpB,CAAC;IACD,IAAI,KAAK,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC9B,OAAO,QAAQ,CAAC;IACpB,CAAC;IACD,IAAI,KAAK,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC9B,OAAO,WAAW,CAAC;IACvB,CAAC;IACD,IAAI,4BAA4B,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAC3C,OAAO,QAAQ,CAAC;IACpB,CAAC;IACD,MAAM,IAAI,KAAK,CACX,uBAAuB,KAAK,KAAK;QACjC,oCAAoC,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CACtE,CAAC;AACN,CAAC;AAED,2FAA2F;AAC3F,SAAgB,eAAe,CAAC,QAAqB;IACjD,QAAQ,QAAQ,EAAE,CAAC;QACf,KAAK,QAAQ,CAAC,CAAC,OAAO,gBAAgB,CAAC;QACvC,KAAK,QAAQ,CAAC,CAAC,OAAO,gBAAgB,CAAC;QACvC,KAAK,QAAQ,CAAC,CAAC,OAAO,mBAAmB,CAAC;QAC1C,OAAO,CAAC,CAAC,OAAO,mBAAmB,CAAC;IACxC,CAAC;AACL,CAAC;AAWD;;;GAGG;AACH,SAAgB,mBAAmB,CAC/B,MAAe,EACf,KAAa,EACb,KAAa,EACb,KAAa,EACb,KAAa,EACb,QAAgB,EAChB,GAAW,EACX,GAAW,EACX,QAAiB,EACjB,QAAiB;IAEjB,IAAI,MAAM,EAAE,CAAC;QACT,OAAO;YACH,OAAO,EAAE,CAAC;YACV,OAAO,EAAE,CAAC;YACV,MAAM,EAAE,CAAC,QAAQ,IAAI,GAAG,CAAC,GAAG,GAAG;YAC/B,MAAM,EAAE,CAAC,QAAQ,IAAI,GAAG,CAAC,GAAG,GAAG;SAClC,CAAC;IACN,CAAC;IAED,8EAA8E;IAC9E,4DAA4D;IAC5D,MAAM,SAAS,GAAG,QAAQ,GAAG,IAAI,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,QAAQ,GAAG,GAAG,CAAC,GAAG,GAAG,GAAG,IAAI,CAAC;IACnF,MAAM,KAAK,GAAG,SAAS,CAAC,CAAC,CAAC,KAAK,GAAG,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC;IACnD,MAAM,KAAK,GAAG,SAAS,CAAC,CAAC,CAAC,KAAK,GAAG,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC;IACnD,OAAO;QACH,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK;QACzD,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK;QACzD,MAAM,EAAE,KAAK,GAAG,GAAG;QACnB,MAAM,EAAE,KAAK,GAAG,GAAG;KACtB,CAAC;AACN,CAAC;AAED,0EAA0E;AAC1E,SAAgB,iBAAiB,CAC7B,OAAqB,EACrB,IAAY,EACZ,IAAY;IAEZ,OAAO;QACH,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,GAAG,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC;QACtD,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,GAAG,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC;KACzD,CAAC;AACN,CAAC;AAED,sEAAsE;AACtE,SAAgB,iBAAiB,CAAC,MAAc,EAAE,GAAW,EAAE,GAAW;IACtE,OAAO,CACH,oDAAoD,MAAM,OAAO;QACjE,gBAAgB,GAAG,IAAI,GAAG,cAAc;QACxC,oDAAoD;QACpD,0FAA0F;QAC1F,oFAAoF;QACpF,2BAA2B,GAAG,6BAA6B,GAAG,OAAO;QACrE,iDAAiD;QACjD,4CAA4C,CAC/C,CAAC;AACN,CAAC;AAED,sFAAsF;AACtF,SAAgB,iBAAiB,CAC7B,GAAW,EACX,MAAc;IAEd,MAAM,SAAS,GAAG,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IAC3C,IAAI,CAAC,SAAS,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,4BAA4B,GAAG,EAAE,CAAC,CAAC;IACvD,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAA4C,CAAC;IACnF,IAAI,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CAAC,uBAAuB,MAAM,GAAG,CAAC,CAAC;IACtD,CAAC;IACD,OAAO,MAAM,CAAC;AAClB,CAAC;AAED,KAAK,UAAU,mBAAmB,CAC9B,MAAc,EACd,UAAkB,EAClB,KAAa,EACb,MAAc,EACd,SAAiB;IAEjB,MAAM,MAAM,GAAG,IAAI,aAAS,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;IACzC,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;QAC1C,KAAK;QACL,UAAU,EAAE,SAAS;QACrB,QAAQ,EAAE,CAAC;gBACP,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE;oBACL;wBACI,IAAI,EAAE,OAAO;wBACb,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE;qBACpE;oBACD,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE;iBACrC;aACJ,CAAC;KACL,CAAC,CAAC;IACH,OAAO,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,EAAE,IAAI,IAAI,EAAE,CAAC;AACvE,CAAC;AAED,KAAK,UAAU,gBAAgB,CAC3B,MAAc,EACd,UAAkB,EAClB,KAAa,EACb,MAAc,EACd,SAAiB;IAEjB,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,4CAA4C,EAAE;QAClE,MAAM,EAAE,MAAM;QACd,OAAO,EAAE;YACL,cAAc,EAAE,kBAAkB;YAClC,eAAe,EAAE,UAAU,MAAM,EAAE;SACtC;QACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;YACjB,KAAK;YACL,UAAU,EAAE,SAAS;YACrB,QAAQ,EAAE,CAAC;oBACP,IAAI,EAAE,MAAM;oBACZ,OAAO,EAAE;wBACL,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE,yBAAyB,MAAM,EAAE,EAAE,EAAE;wBAC5E,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE;qBACrC;iBACJ,CAAC;SACL,CAAC;KACL,CAAC,CAAC;IACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;QACV,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAC9B,IAAI,OAAe,CAAC;QACpB,IAAI,CAAC;YACD,OAAO,GAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAqC,CAAC,KAAK,EAAE,OAAO,IAAI,IAAI,CAAC;QAC3F,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,GAAG,IAAI,IAAI,GAAG,CAAC,UAAU,CAAC;QACrC,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,qBAAqB,OAAO,EAAE,CAAC,CAAC;IACpD,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAA2D,CAAC;IACvF,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC;IACpD,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,0CAA0C,KAAK,kDAAkD,CAAC,CAAC;IACvH,CAAC;IACD,OAAO,OAAO,CAAC;AACnB,CAAC;AAED,KAAK,UAAU,gBAAgB,CAC3B,MAAc,EACd,UAAkB,EAClB,KAAa,EACb,MAAc,EACd,SAAiB;IAEjB,MAAM,GAAG,GAAG,2DAA2D,KAAK,kBAAkB,CAAC;IAC/F,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;QACzB,MAAM,EAAE,MAAM;QACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,EAAE;QACzE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;YACjB,QAAQ,EAAE,CAAC;oBACP,KAAK,EAAE;wBACH,EAAE,WAAW,EAAE,EAAE,SAAS,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE;wBACzD,EAAE,IAAI,EAAE,UAAU,EAAE;qBACvB;iBACJ,CAAC;YACF,gBAAgB,EAAE,EAAE,eAAe,EAAE,SAAS,EAAE;SACnD,CAAC;KACL,CAAC,CAAC;IACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;QACV,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAC9B,IAAI,OAAe,CAAC;QACpB,IAAI,CAAC;YACD,OAAO,GAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAqC,CAAC,KAAK,EAAE,OAAO,IAAI,IAAI,CAAC;QAC3F,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,GAAG,IAAI,IAAI,GAAG,CAAC,UAAU,CAAC;QACrC,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,qBAAqB,OAAO,EAAE,CAAC,CAAC;IACpD,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAE1B,CAAC;IACF,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC;IAC7D,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CAAC,0CAA0C,KAAK,mDAAmD,CAAC,CAAC;IACxH,CAAC;IACD,OAAO,IAAI,CAAC;AAChB,CAAC;AAED,KAAK,UAAU,uBAAuB,CAClC,MAAc,EACd,UAAkB,EAClB,KAAa,EACb,SAAiB;IAEjB,MAAM,MAAM,GAAG,IAAI,6CAAoB,CAAC;QACpC,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,WAAW;KAClF,CAAC,CAAC;IACH,MAAM,OAAO,GAAG,IAAI,wCAAe,CAAC;QAChC,OAAO,EAAE,KAAK;QACd,QAAQ,EAAE,CAAC;gBACP,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE;oBACL;wBACI,KAAK,EAAE;4BACH,MAAM,EAAE,KAAK;4BACb,MAAM,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAE;yBACnD;qBACJ;oBACD,EAAE,IAAI,EAAE,UAAU,EAAE;iBACvB;aACJ,CAAC;QACF,eAAe,EAAE,EAAE,SAAS,EAAE;KACjC,CAAC,CAAC;IACH,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC5C,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,IAAI,CAAC,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAiC,CAAC;IACvI,IAAI,CAAC,IAAI,EAAE,CAAC;QACR,MAAM,IAAI,KAAK,CAAC,kDAAkD,KAAK,8BAA8B,CAAC,CAAC;IAC3G,CAAC;IACD,OAAO,IAAI,CAAC,IAAI,CAAC;AACrB,CAAC;AAED;;;;;GAKG;AACI,KAAK,UAAU,aAAa,CAC/B,MAAc,EACd,UAAkB,EAClB,KAAa,EACb,MAAc,EACd,SAAS,GAAG,GAAG;IAEf,MAAM,QAAQ,GAAG,mBAAmB,CAAC,KAAK,CAAC,CAAC;IAC5C,QAAQ,QAAQ,EAAE,CAAC;QACf,KAAK,QAAQ,CAAC,CAAC,OAAO,gBAAgB,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;QACrF,KAAK,QAAQ,CAAC,CAAC,OAAO,gBAAgB,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;QACrF,KAAK,QAAQ,CAAC,CAAC,OAAO,uBAAuB,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;QACpF,OAAO,CAAC,CAAC,OAAO,mBAAmB,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;IACtF,CAAC;AACL,CAAC"}
|