appium-desktop-driver 1.5.1 → 1.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +77 -0
- package/build/lib/commands/index.d.ts +1 -1
- package/build/lib/commands/index.d.ts.map +1 -1
- package/build/lib/commands/vision.d.ts +1 -1
- package/build/lib/commands/vision.d.ts.map +1 -1
- package/build/lib/commands/vision.js +13 -3
- package/build/lib/commands/vision.js.map +1 -1
- package/build/lib/mcp/session.d.ts.map +1 -1
- package/build/lib/mcp/session.js +1 -0
- package/build/lib/mcp/session.js.map +1 -1
- package/build/lib/mcp/tools/vision.d.ts.map +1 -1
- package/build/lib/mcp/tools/vision.js +19 -6
- package/build/lib/mcp/tools/vision.js.map +1 -1
- package/build/lib/vision-utils.d.ts +8 -3
- package/build/lib/vision-utils.d.ts.map +1 -1
- package/build/lib/vision-utils.js +153 -6
- package/build/lib/vision-utils.js.map +1 -1
- package/build/tsconfig.tsbuildinfo +1 -1
- package/package.json +2 -1
package/README.md
CHANGED
|
@@ -633,6 +633,83 @@ if secondary:
|
|
|
633
633
|
driver.execute_script('windows: click', {'x': cx, 'y': cy})
|
|
634
634
|
```
|
|
635
635
|
|
|
636
|
+
### windows: findByVision
|
|
637
|
+
|
|
638
|
+
Uses a vision-capable LLM to locate a UI element by natural language
|
|
639
|
+
description. Takes a screenshot of the current window, sends it to the
|
|
640
|
+
configured LLM, and returns the screen coordinates of the described element.
|
|
641
|
+
|
|
642
|
+
The `model` argument controls which LLM provider is used. The corresponding
|
|
643
|
+
API key must be set as an environment variable before starting the Appium
|
|
644
|
+
server.
|
|
645
|
+
|
|
646
|
+
#### Supported Models
|
|
647
|
+
|
|
648
|
+
| Provider | Model prefix | Environment variable(s) | Example models |
|
|
649
|
+
| --- | --- | --- | --- |
|
|
650
|
+
| Anthropic | `claude-` | `ANTHROPIC_API_KEY` | `claude-opus-4-6`, `claude-sonnet-4-6` |
|
|
651
|
+
| OpenAI | `gpt-`, `o1`, `o3`, `o4` | `OPENAI_API_KEY` | `gpt-4o`, `o3` |
|
|
652
|
+
| Google | `gemini-` | `GEMINI_API_KEY` | `gemini-1.5-pro`, `gemini-2.0-flash` |
|
|
653
|
+
| Amazon (Bedrock) | `amazon.nova-`, `us.amazon.nova-`, `eu.amazon.nova-`, `ap.amazon.nova-` | `AWS_ACCESS_KEY_ID` + `AWS_SECRET_ACCESS_KEY` | `amazon.nova-pro-v1:0`, `amazon.nova-lite-v1:0` |
|
|
654
|
+
|
|
655
|
+
For Amazon Bedrock, set `AWS_REGION` (or `AWS_DEFAULT_REGION`) to select the deployment region (defaults to `us-east-1`). IAM roles and AWS credential profiles are also supported via the standard AWS credential chain.
|
|
656
|
+
|
|
657
|
+
#### Arguments
|
|
658
|
+
|
|
659
|
+
| Name | Type | Required | Description | Example |
|
|
660
|
+
| --- | --- | --- | --- | --- |
|
|
661
|
+
| prompt | string | yes | Natural language description of the element to find | `"Submit button"` |
|
|
662
|
+
| model | string | yes | LLM model identifier | `"claude-opus-4-6"`, `"gpt-4o"`, `"amazon.nova-pro-v1:0"` |
|
|
663
|
+
|
|
664
|
+
#### Returns
|
|
665
|
+
|
|
666
|
+
| Name | Type | Description |
|
|
667
|
+
| --- | --- | --- |
|
|
668
|
+
| x | number | Horizontal screen coordinate of the element center |
|
|
669
|
+
| y | number | Vertical screen coordinate of the element center |
|
|
670
|
+
| label | string | Brief description of the element as identified by the LLM |
|
|
671
|
+
|
|
672
|
+
#### Example
|
|
673
|
+
|
|
674
|
+
```javascript
|
|
675
|
+
// WebdriverIO — locate and click an element using Claude
|
|
676
|
+
const result = await driver.executeScript('windows: findByVision', [{
|
|
677
|
+
prompt: 'Save button in the toolbar',
|
|
678
|
+
model: 'claude-opus-4-6',
|
|
679
|
+
}]);
|
|
680
|
+
await driver.executeScript('windows: click', [{ x: result.x, y: result.y }]);
|
|
681
|
+
```
|
|
682
|
+
|
|
683
|
+
```javascript
|
|
684
|
+
// WebdriverIO — use GPT-4o instead
|
|
685
|
+
const result = await driver.executeScript('windows: findByVision', [{
|
|
686
|
+
prompt: 'Save button in the toolbar',
|
|
687
|
+
model: 'gpt-4o',
|
|
688
|
+
}]);
|
|
689
|
+
await driver.executeScript('windows: click', [{ x: result.x, y: result.y }]);
|
|
690
|
+
```
|
|
691
|
+
|
|
692
|
+
```javascript
|
|
693
|
+
// WebdriverIO — use Amazon Nova via Bedrock
|
|
694
|
+
const result = await driver.executeScript('windows: findByVision', [{
|
|
695
|
+
prompt: 'Save button in the toolbar',
|
|
696
|
+
model: 'amazon.nova-pro-v1:0',
|
|
697
|
+
}]);
|
|
698
|
+
await driver.executeScript('windows: click', [{ x: result.x, y: result.y }]);
|
|
699
|
+
```
|
|
700
|
+
|
|
701
|
+
```python
|
|
702
|
+
# Python — locate and click using Gemini
|
|
703
|
+
result = driver.execute_script('windows: findByVision', {
|
|
704
|
+
'prompt': 'Save button in the toolbar',
|
|
705
|
+
'model': 'gemini-1.5-pro',
|
|
706
|
+
})
|
|
707
|
+
driver.execute_script('windows: click', {
|
|
708
|
+
'x': result['x'],
|
|
709
|
+
'y': result['y'],
|
|
710
|
+
})
|
|
711
|
+
```
|
|
712
|
+
|
|
636
713
|
## Development
|
|
637
714
|
|
|
638
715
|
it is recommended to use Matt Bierner's [Comment tagged templates](https://marketplace.visualstudio.com/items?itemName=bierner.comment-tagged-templates)
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../lib/commands/index.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,OAAO,MAAM,WAAW,CAAC;AAOrC,QAAA,MAAM,QAAQ
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../lib/commands/index.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,OAAO,MAAM,WAAW,CAAC;AAOrC,QAAA,MAAM,QAAQ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;mBAoBg7W,CAAC;;;mBAAyqB,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;iBAAmiH,CAAC;SAAe,CAAC;SAAe,CAAC;cAAoB,CAAC;oBAA6B,CAAC;kBAA8G,CAAC;aAAmB,CAAC;yBAA+B,CAAC;;;sBAAmtG,CAAC;cAAoB,CAAC;cAAoB,CAAC;oBAA0B,CAAC;YAAkB,CAAC;YAAkB,CAAC;oBAA0B,CAAC;kBAA2F,CAAC;;;iBAA2yH,CAAC;SAAe,CAAC;SAAe,CAAC;cAAoB,CAAC;cAAoB,CAAC;oBAA0B,CAAC;;;kBAA8qF,CAAC;iBAAuB,CAAC;gBAAsB,CAAC;mBAAyB,CAAC;cAAoB,CAAC;qBAA2B,CAAC;qBAA4B,CAAC;kBAAyB,CAAC;oBAA0B,CAAC;;;;;;;;iBAAo5F,CAAC;;;sBAAouB,CAAC;cAAoB,CAAC;cAAoB,CAAC;oBAA0B,CAAC;YAAkB,CAAC;YAAkB,CAAC;oBAA0B,CAAC;kBAA2F,CAAC;cAAoB,CAAC;;;cAAu1I,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;0FAArlgC,wDAAiC;mFAAwtB,sDAA+B;;;;CAVh9F,CAAC;AAEF,KAAK,QAAQ,GAAG;KACX,GAAG,IAAI,MAAM,OAAO,QAAQ,GAAG,OAAO,QAAQ,CAAC,GAAG,CAAC;CACvD,CAAC;AAEF,OAAO,QAAQ,WAAW,CAAC;IACvB,UAAU,mBAAoB,SAAQ,QAAQ;KAAG;CACpD;AAED,eAAe,QAAQ,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vision.d.ts","sourceRoot":"","sources":["../../../lib/commands/vision.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"vision.d.ts","sourceRoot":"","sources":["../../../lib/commands/vision.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,WAAW,CAAC;AA4ChD,wBAAsB,mBAAmB,CACrC,IAAI,EAAE,mBAAmB,EACzB,IAAI,EAAE;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GACxC,OAAO,CAAC;IAAE,CAAC,EAAE,MAAM,CAAC;IAAC,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAgClD"}
|
|
@@ -15,13 +15,23 @@ async function buildCoordMapping(driver, ssW, ssH) {
|
|
|
15
15
|
return (0, vision_utils_1.computeCoordMapping)(false, rect.x, rect.y, rect.width, rect.height, (0, user32_1.getResolutionScalingFactor)(), ssW, ssH);
|
|
16
16
|
}
|
|
17
17
|
async function executeFindByVision(args) {
|
|
18
|
-
|
|
18
|
+
if (!args.model) {
|
|
19
|
+
throw new Error('windows: findByVision requires a "model" argument. ' +
|
|
20
|
+
'Supported prefixes: claude-* (ANTHROPIC_API_KEY), gpt-*/o-series (OPENAI_API_KEY), ' +
|
|
21
|
+
'gemini-* (GEMINI_API_KEY), amazon.nova-*/us.amazon.nova-*/eu.amazon.nova-*/ap.amazon.nova-* (AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY).');
|
|
22
|
+
}
|
|
23
|
+
const model = args.model;
|
|
24
|
+
const provider = (0, vision_utils_1.getProviderForModel)(model);
|
|
25
|
+
const envVar = (0, vision_utils_1.getApiKeyEnvVar)(provider);
|
|
26
|
+
const apiKey = process.env[envVar];
|
|
19
27
|
if (!apiKey) {
|
|
20
|
-
throw new Error(
|
|
28
|
+
throw new Error(`${envVar} environment variable is required for windows: findByVision (model: ${model})`);
|
|
29
|
+
}
|
|
30
|
+
if (provider === 'amazon' && !process.env.AWS_SECRET_ACCESS_KEY) {
|
|
31
|
+
throw new Error('AWS_SECRET_ACCESS_KEY environment variable is required for Amazon Bedrock models');
|
|
21
32
|
}
|
|
22
33
|
const base64 = await this.getScreenshot();
|
|
23
34
|
const { width: ssW, height: ssH } = (0, util_1.getPngDimensions)(base64);
|
|
24
|
-
const model = args.model ?? 'claude-opus-4-6';
|
|
25
35
|
const raw = await (0, vision_utils_1.callVisionLLM)(base64, (0, vision_utils_1.buildVisionPrompt)(args.prompt, ssW, ssH), model, apiKey);
|
|
26
36
|
const parsed = (0, vision_utils_1.parseVisionCoords)(raw, args.prompt);
|
|
27
37
|
const mapping = await buildCoordMapping(this, ssW, ssH);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vision.js","sourceRoot":"","sources":["../../../lib/commands/vision.ts"],"names":[],"mappings":";;
|
|
1
|
+
{"version":3,"file":"vision.js","sourceRoot":"","sources":["../../../lib/commands/vision.ts"],"names":[],"mappings":";;AA4CA,kDAmCC;AA9ED,kCAA2C;AAC3C,6CAA8D;AAC9D,kDASyB;AAEzB,KAAK,UAAU,iBAAiB,CAC5B,MAA2B,EAC3B,GAAW,EACX,GAAW;IAEX,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,aAAa,EAAE,CAAC;IAC1C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IAElC,IAAI,MAAM,EAAE,CAAC;QACT,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,kBAAkB,EAG9C,CAAC;QACH,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,QAAQ,CAAC,CAAC,CAAC,CAAC;QAC/D,OAAO,IAAA,kCAAmB,EACtB,IAAI,EACJ,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,EACvC,CAAC,EAAE,GAAG,EAAE,GAAG,EACX,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,CAClD,CAAC;IACN,CAAC;IAED,OAAO,IAAA,kCAAmB,EACtB,KAAK,EACL,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,EACvC,IAAA,mCAA0B,GAAE,EAC5B,GAAG,EAAE,GAAG,CACX,CAAC;AACN,CAAC;AAEM,KAAK,UAAU,mBAAmB,CAErC,IAAuC;IAEvC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;QACd,MAAM,IAAI,KAAK,CACX,qDAAqD;YACrD,qFAAqF;YACrF,0IAA0I,CAC7I,CAAC;IACN,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;IACzB,MAAM,QAAQ,GAAG,IAAA,kCAAmB,EAAC,KAAK,CAAC,CAAC;IAC5C,MAAM,MAAM,GAAG,IAAA,8BAAe,EAAC,QAAQ,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACnC,IAAI,CAAC,MAAM,EAAE,CAAC;QACV,MAAM,IAAI,KAAK,CACX,GAAG,MAAM,uEAAuE,KAAK,GAAG,CAC3F,CAAC;IACN,CAAC;IACD,IAAI,QAAQ,KAAK,QAAQ,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,qBAAqB,EAAE,CAAC;QAC9D,MAAM,IAAI,KAAK,CAAC,kFAAkF,CAAC,CAAC;IACxG,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;IAC1C,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,IAAA,uBAAgB,EAAC,MAAM,CAAC,CAAC;IAE7D,MAAM,GAAG,GAAG,MAAM,IAAA,4BAAa,EAAC,MAAM,EAAE,IAAA,gCAAiB,EAAC,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;IACjG,MAAM,MAAM,GAAG,IAAA,gCAAiB,EAAC,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IAEnD,MAAM,OAAO,GAAG,MAAM,iBAAiB,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;IACxD,OAAO;QACH,GAAG,IAAA,gCAAiB,EAAC,OAAO,EAAE,MAAM,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;QACjD,KAAK,EAAE,MAAM,CAAC,KAAK;KACtB,CAAC;AACN,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../../lib/mcp/session.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAE7C,4EAA4E;AAC5E,MAAM,WAAW,aAAa;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED,qBAAa,aAAa;IAGV,OAAO,CAAC,QAAQ,CAAC,YAAY;IAFzC,OAAO,CAAC,MAAM,CAAwB;gBAET,YAAY,EAAE,SAAS;IAE9C,MAAM,CAAC,MAAM,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../../lib/mcp/session.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAE7C,4EAA4E;AAC5E,MAAM,WAAW,aAAa;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED,qBAAa,aAAa;IAGV,OAAO,CAAC,QAAQ,CAAC,YAAY;IAFzC,OAAO,CAAC,MAAM,CAAwB;gBAET,YAAY,EAAE,SAAS;IAE9C,MAAM,CAAC,MAAM,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IAiC5C,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAY7B,QAAQ,IAAI,OAAO;IAInB,SAAS,IAAI,OAAO;CAIvB"}
|
package/build/lib/mcp/session.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session.js","sourceRoot":"","sources":["../../../lib/mcp/session.ts"],"names":[],"mappings":";;;AAAA,6CAAqC;AAiBrC,MAAa,aAAa;IAGO;IAFrB,MAAM,GAAmB,IAAI,CAAC;IAEtC,YAA6B,YAAuB;QAAvB,iBAAY,GAAZ,YAAY,CAAW;IAAG,CAAC;IAExD,KAAK,CAAC,MAAM,CAAC,MAAqB;QAC9B,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;QAC/E,CAAC;QAED,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,0CAA0C,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC;QAE/E,MAAM,IAAI,GAA4B;YAClC,YAAY,EAAE,SAAS;YACvB,uBAAuB,EAAE,eAAe;YACxC,YAAY,EAAE,MAAM,CAAC,GAAG;SAC3B,CAAC;QAEF,IAAI,MAAM,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;YAAA,IAAI,CAAC,qBAAqB,CAAC,GAAG,MAAM,CAAC,YAAY,CAAC;QAAA,CAAC;QAC3F,IAAI,MAAM,CAAC,aAAa,KAAK,SAAS,EAAE,CAAC;YAAA,IAAI,CAAC,sBAAsB,CAAC,GAAG,MAAM,CAAC,aAAa,CAAC;QAAA,CAAC;QAC9F,IAAI,MAAM,CAAC,gBAAgB,KAAK,SAAS,EAAE,CAAC;YAAA,IAAI,CAAC,yBAAyB,CAAC,GAAG,MAAM,CAAC,gBAAgB,CAAC;QAAA,CAAC;QACvG,IAAI,MAAM,CAAC,cAAc,KAAK,SAAS,EAAE,CAAC;YAAA,IAAI,CAAC,uBAAuB,CAAC,GAAG,MAAM,CAAC,cAAc,CAAC;QAAA,CAAC;QACjG,IAAI,MAAM,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;YAAA,IAAI,CAAC,wBAAwB,CAAC,GAAG,MAAM,CAAC,eAAe,CAAC;QAAA,CAAC;QACpG,IAAI,MAAM,CAAC,gBAAgB,KAAK,SAAS,EAAE,CAAC;YAAA,IAAI,CAAC,yBAAyB,CAAC,GAAG,MAAM,CAAC,gBAAgB,CAAC;QAAA,CAAC;QACvG,IAAI,MAAM,CAAC,iBAAiB,KAAK,SAAS,EAAE,CAAC;YAAA,IAAI,CAAC,0BAA0B,CAAC,GAAG,MAAM,CAAC,iBAAiB,CAAC;QAAA,CAAC;QAE1G,IAAI,CAAC,MAAM,GAAG,MAAM,IAAA,oBAAM,EAAC;YACvB,QAAQ,EAAE,IAAI,CAAC,YAAY,CAAC,UAAU;YACtC,IAAI,EAAE,IAAI,CAAC,YAAY,CAAC,UAAU;YAClC,IAAI,EAAE,GAAG;YACT,YAAY,EAAE,IAAgC;SACjD,CAAC,CAAC;QAEH,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC;QACnE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,sCAAsC,CAAC,CAAC;IACjE,CAAC;IAED,KAAK,CAAC,MAAM;QACR,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YAAA,OAAO;QAAA,CAAC;QAC3B,IAAI,CAAC;YACD,MAAM,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,CAAC;YAClC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;QACpD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,yCAAyC,GAAG,IAAI,CAAC,CAAC;QAC3E,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACvB,CAAC;IACL,CAAC;IAED,QAAQ;QACJ,OAAO,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC;IAChC,CAAC;IAED,SAAS;QACL,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YAAA,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;QAAA,CAAC;QACrF,OAAO,IAAI,CAAC,MAAM,CAAC;IACvB,CAAC;CACJ;
|
|
1
|
+
{"version":3,"file":"session.js","sourceRoot":"","sources":["../../../lib/mcp/session.ts"],"names":[],"mappings":";;;AAAA,6CAAqC;AAiBrC,MAAa,aAAa;IAGO;IAFrB,MAAM,GAAmB,IAAI,CAAC;IAEtC,YAA6B,YAAuB;QAAvB,iBAAY,GAAZ,YAAY,CAAW;IAAG,CAAC;IAExD,KAAK,CAAC,MAAM,CAAC,MAAqB;QAC9B,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;QAC/E,CAAC;QAED,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,0CAA0C,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC;QAE/E,MAAM,IAAI,GAA4B;YAClC,YAAY,EAAE,SAAS;YACvB,uBAAuB,EAAE,eAAe;YACxC,YAAY,EAAE,MAAM,CAAC,GAAG;SAC3B,CAAC;QAEF,IAAI,MAAM,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;YAAA,IAAI,CAAC,qBAAqB,CAAC,GAAG,MAAM,CAAC,YAAY,CAAC;QAAA,CAAC;QAC3F,IAAI,MAAM,CAAC,aAAa,KAAK,SAAS,EAAE,CAAC;YAAA,IAAI,CAAC,sBAAsB,CAAC,GAAG,MAAM,CAAC,aAAa,CAAC;QAAA,CAAC;QAC9F,IAAI,MAAM,CAAC,gBAAgB,KAAK,SAAS,EAAE,CAAC;YAAA,IAAI,CAAC,yBAAyB,CAAC,GAAG,MAAM,CAAC,gBAAgB,CAAC;QAAA,CAAC;QACvG,IAAI,MAAM,CAAC,cAAc,KAAK,SAAS,EAAE,CAAC;YAAA,IAAI,CAAC,uBAAuB,CAAC,GAAG,MAAM,CAAC,cAAc,CAAC;QAAA,CAAC;QACjG,IAAI,MAAM,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;YAAA,IAAI,CAAC,wBAAwB,CAAC,GAAG,MAAM,CAAC,eAAe,CAAC;QAAA,CAAC;QACpG,IAAI,MAAM,CAAC,gBAAgB,KAAK,SAAS,EAAE,CAAC;YAAA,IAAI,CAAC,yBAAyB,CAAC,GAAG,MAAM,CAAC,gBAAgB,CAAC;QAAA,CAAC;QACvG,IAAI,MAAM,CAAC,iBAAiB,KAAK,SAAS,EAAE,CAAC;YAAA,IAAI,CAAC,0BAA0B,CAAC,GAAG,MAAM,CAAC,iBAAiB,CAAC;QAAA,CAAC;QAE1G,IAAI,CAAC,MAAM,GAAG,MAAM,IAAA,oBAAM,EAAC;YACvB,QAAQ,EAAE,IAAI,CAAC,YAAY,CAAC,UAAU;YACtC,IAAI,EAAE,IAAI,CAAC,YAAY,CAAC,UAAU;YAClC,IAAI,EAAE,GAAG;YACT,QAAQ,EAAE,QAAQ;YAClB,YAAY,EAAE,IAAgC;SACjD,CAAC,CAAC;QAEH,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC;QACnE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,sCAAsC,CAAC,CAAC;IACjE,CAAC;IAED,KAAK,CAAC,MAAM;QACR,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YAAA,OAAO;QAAA,CAAC;QAC3B,IAAI,CAAC;YACD,MAAM,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,CAAC;YAClC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;QACpD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,yCAAyC,GAAG,IAAI,CAAC,CAAC;QAC3E,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACvB,CAAC;IACL,CAAC;IAED,QAAQ;QACJ,OAAO,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC;IAChC,CAAC;IAED,SAAS;QACL,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YAAA,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;QAAA,CAAC;QACrF,OAAO,IAAI,CAAC,MAAM,CAAC;IACvB,CAAC;CACJ;AA1DD,sCA0DC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vision.d.ts","sourceRoot":"","sources":["../../../../lib/mcp/tools/vision.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AAGpE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AA0CnD,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,aAAa,GAAG,IAAI,
|
|
1
|
+
{"version":3,"file":"vision.d.ts","sourceRoot":"","sources":["../../../../lib/mcp/tools/vision.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AAGpE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AA0CnD,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,aAAa,GAAG,IAAI,CA6EnF"}
|
|
@@ -5,7 +5,6 @@ const zod_1 = require("zod");
|
|
|
5
5
|
const errors_js_1 = require("../errors.js");
|
|
6
6
|
const util_1 = require("../../util");
|
|
7
7
|
const vision_utils_1 = require("../../vision-utils");
|
|
8
|
-
const DEFAULT_MODEL = 'claude-opus-4-6';
|
|
9
8
|
async function buildCoordMapping(driver, ssW, ssH) {
|
|
10
9
|
try {
|
|
11
10
|
const rect = await driver.getWindowRect();
|
|
@@ -31,24 +30,38 @@ function registerVisionTools(server, session) {
|
|
|
31
30
|
'For "coordinates" format, locates a UI element and returns {x,y,label} with actual screen ' +
|
|
32
31
|
'coordinates (DPI-corrected) ready to pass to click tools. ' +
|
|
33
32
|
'For "text" format, answers a general question about the screen in plain text. ' +
|
|
34
|
-
'Requires ANTHROPIC_API_KEY
|
|
33
|
+
'Requires ANTHROPIC_API_KEY (Claude), OPENAI_API_KEY (GPT-4o / o-series), ' +
|
|
34
|
+
'GEMINI_API_KEY (Gemini), or AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY (Amazon Nova via Bedrock) ' +
|
|
35
|
+
'depending on the chosen model.',
|
|
35
36
|
inputSchema: {
|
|
36
37
|
prompt: zod_1.z.string().min(1).describe('Question or instruction about the screenshot'),
|
|
37
38
|
responseFormat: zod_1.z.enum(['coordinates', 'text']).default('coordinates').describe('"coordinates" (default) locates an element and returns JSON {x,y,label} with converted screen coordinates. ' +
|
|
38
39
|
'"text" answers a general question about the screen in plain text.'),
|
|
39
|
-
model: zod_1.z.string().
|
|
40
|
+
model: zod_1.z.string().min(1).describe('Vision model to use. Determines which credentials are required: ' +
|
|
41
|
+
'claude-* → ANTHROPIC_API_KEY, gpt-*/o-series → OPENAI_API_KEY, ' +
|
|
42
|
+
'gemini-* → GEMINI_API_KEY, amazon.nova-* → AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY.'),
|
|
40
43
|
},
|
|
41
44
|
annotations: { readOnlyHint: true },
|
|
42
45
|
}, async ({ prompt, responseFormat, model }) => {
|
|
43
46
|
try {
|
|
44
|
-
|
|
47
|
+
if (!model) {
|
|
48
|
+
throw new Error('find_by_vision requires a "model" argument. ' +
|
|
49
|
+
'Supported prefixes: claude-* (ANTHROPIC_API_KEY), gpt-*/o-series (OPENAI_API_KEY), ' +
|
|
50
|
+
'gemini-* (GEMINI_API_KEY), amazon.nova-*/us.amazon.nova-*/eu.amazon.nova-*/ap.amazon.nova-* (AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY).');
|
|
51
|
+
}
|
|
52
|
+
const visionModel = model;
|
|
53
|
+
const provider = (0, vision_utils_1.getProviderForModel)(visionModel);
|
|
54
|
+
const envVar = (0, vision_utils_1.getApiKeyEnvVar)(provider);
|
|
55
|
+
const apiKey = process.env[envVar];
|
|
45
56
|
if (!apiKey) {
|
|
46
|
-
throw new Error(
|
|
57
|
+
throw new Error(`${envVar} environment variable is required for find_by_vision (model: ${visionModel})`);
|
|
58
|
+
}
|
|
59
|
+
if (provider === 'amazon' && !process.env.AWS_SECRET_ACCESS_KEY) {
|
|
60
|
+
throw new Error('AWS_SECRET_ACCESS_KEY environment variable is required for Amazon Bedrock models');
|
|
47
61
|
}
|
|
48
62
|
const driver = session.getDriver();
|
|
49
63
|
const base64 = await driver.takeScreenshot();
|
|
50
64
|
const { width: ssW, height: ssH } = (0, util_1.getPngDimensions)(base64);
|
|
51
|
-
const visionModel = model ?? DEFAULT_MODEL;
|
|
52
65
|
if (responseFormat === 'text') {
|
|
53
66
|
const textPrompt = `Answer the following about this screenshot: "${prompt}"\nRespond with plain text.`;
|
|
54
67
|
const text = await (0, vision_utils_1.callVisionLLM)(base64, textPrompt, visionModel, apiKey, 1024);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vision.js","sourceRoot":"","sources":["../../../../lib/mcp/tools/vision.ts"],"names":[],"mappings":";;AA6CA,
|
|
1
|
+
{"version":3,"file":"vision.js","sourceRoot":"","sources":["../../../../lib/mcp/tools/vision.ts"],"names":[],"mappings":";;AA6CA,kDA6EC;AAzHD,6BAAwB;AAGxB,4CAA2C;AAC3C,qCAA8C;AAC9C,qDAS4B;AAE5B,KAAK,UAAU,iBAAiB,CAAC,MAAe,EAAE,GAAW,EAAE,GAAW;IACtE,IAAI,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,aAAa,EAAE,CAAC;QAC1C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QAElC,IAAI,MAAM,EAAE,CAAC;YACT,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,sBAAsB,EAAE,EAAE,CAAU,CAAC;YACjF,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,QAAQ,CAAC,CAAC,CAAC,CAAC;YACpE,IAAI,CAAC,OAAO,EAAE,CAAC;gBAAC,OAAO,SAAS,CAAC;YAAC,CAAC;YACnC,OAAO,IAAA,kCAAmB,EACtB,IAAI,EACJ,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,EACvC,CAAC,EAAE,GAAG,EAAE,GAAG,EACX,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM,CAC9C,CAAC;QACN,CAAC;QAED,MAAM,QAAQ,GAAG,CAAC,MAAM,MAAM,CAAC,aAAa,CAAC,sBAAsB,EAAE,EAAE,CAAC,CAAW,CAAC;QACpF,OAAO,IAAA,kCAAmB,EACtB,KAAK,EACL,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,EACvC,QAAQ,EAAE,GAAG,EAAE,GAAG,CACrB,CAAC;IACN,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,SAAS,CAAC;IACrB,CAAC;AACL,CAAC;AAED,SAAgB,mBAAmB,CAAC,MAAiB,EAAE,OAAsB;IACzE,MAAM,CAAC,YAAY,CACf,gBAAgB,EAChB;QACI,WAAW,EACP,uFAAuF;YACvF,4FAA4F;YAC5F,4DAA4D;YAC5D,gFAAgF;YAChF,2EAA2E;YAC3E,kGAAkG;YAClG,gCAAgC;QACpC,WAAW,EAAE;YACT,MAAM,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,8CAA8C,CAAC;YAClF,cAAc,EAAE,OAAC,CAAC,IAAI,CAAC,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,QAAQ,CAC3E,6GAA6G;gBAC7G,mEAAmE,CACtE;YACD,KAAK,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAC7B,kEAAkE;gBAClE,iEAAiE;gBACjE,uFAAuF,CAC1F;SACJ;QACD,WAAW,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE;KACtC,EACD,KAAK,EAAE,EAAE,MAAM,EAAE,cAAc,EAAE,KAAK,EAAE,EAAE,EAAE;QACxC,IAAI,CAAC;YACD,IAAI,CAAC,KAAK,EAAE,CAAC;gBACT,MAAM,IAAI,KAAK,CACX,8CAA8C;oBAC9C,qFAAqF;oBACrF,0IAA0I,CAC7I,CAAC;YACN,CAAC;YACD,MAAM,WAAW,GAAG,KAAK,CAAC;YAC1B,MAAM,QAAQ,GAAG,IAAA,kCAAmB,EAAC,WAAW,CAAC,CAAC;YAClD,MAAM,MAAM,GAAG,IAAA,8BAAe,EAAC,QAAQ,CAAC,CAAC;YACzC,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACnC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACV,MAAM,IAAI,KAAK,CACX,GAAG,MAAM,gEAAgE,WAAW,GAAG,CAC1F,CAAC;YACN,CAAC;YACD,IAAI,QAAQ,KAAK,QAAQ,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,qBAAqB,EAAE,CAAC;gBAC9D,MAAM,IAAI,KAAK,CAAC,kFAAkF,CAAC,CAAC;YACxG,CAAC;YAED,MAAM,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;YACnC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,cAAc,EAAY,CAAC;YACvD,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,IAAA,uBAAgB,EAAC,MAAM,CAAC,CAAC;YAE7D,IAAI,cAAc,KAAK,MAAM,EAAE,CAAC;gBAC5B,MAAM,UAAU,GAAG,gDAAgD,MAAM,6BAA6B,CAAC;gBACvG,MAAM,IAAI,GAAG,MAAM,IAAA,4BAAa,EAAC,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;gBAChF,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;YAC1D,CAAC;YAED,sFAAsF;YACtF,MAAM,GAAG,GAAG,MAAM,IAAA,4BAAa,EAAC,MAAM,EAAE,IAAA,gCAAiB,EAAC,MAAM,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;YAClG,MAAM,MAAM,GAAG,IAAA,gCAAiB,EAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YAC9C,MAAM,OAAO,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;YAC1D,MAAM,MAAM,GAAG,OAAO;gBAClB,CAAC,CAAC,IAAA,gCAAiB,EAAC,OAAO,EAAE,MAAM,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;gBAChD,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC;YAEnC,OAAO;gBACH,OAAO,EAAE,CAAC;wBACN,IAAI,EAAE,MAAe;wBACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC;qBAC3D,CAAC;aACL,CAAC;QACN,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,IAAA,uBAAW,EAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;QAC3F,CAAC;IACL,CAAC,CACJ,CAAC;AACN,CAAC"}
|
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
export type LLMProvider = 'anthropic' | 'openai' | 'google' | 'amazon';
|
|
2
|
+
export declare function getProviderForModel(model: string): LLMProvider;
|
|
3
|
+
/** Returns the environment variable name that holds the API key for the given provider. */
|
|
4
|
+
export declare function getApiKeyEnvVar(provider: LLMProvider): string;
|
|
1
5
|
export interface CoordMapping {
|
|
2
6
|
offsetX: number;
|
|
3
7
|
offsetY: number;
|
|
@@ -25,9 +29,10 @@ export declare function parseVisionCoords(raw: string, prompt: string): {
|
|
|
25
29
|
label: string;
|
|
26
30
|
};
|
|
27
31
|
/**
|
|
28
|
-
* Sends a base64 screenshot + text prompt to a
|
|
29
|
-
*
|
|
30
|
-
*
|
|
32
|
+
* Sends a base64 screenshot + text prompt to a vision model and returns the raw
|
|
33
|
+
* text response. Dispatches to Anthropic, OpenAI, Google Gemini, or Amazon Bedrock
|
|
34
|
+
* based on the model name prefix. The caller is responsible for building the prompt
|
|
35
|
+
* and parsing the result.
|
|
31
36
|
*/
|
|
32
37
|
export declare function callVisionLLM(base64: string, textPrompt: string, model: string, apiKey: string, maxTokens?: number): Promise<string>;
|
|
33
38
|
//# sourceMappingURL=vision-utils.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vision-utils.d.ts","sourceRoot":"","sources":["../../lib/vision-utils.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"vision-utils.d.ts","sourceRoot":"","sources":["../../lib/vision-utils.ts"],"names":[],"mappings":"AAGA,MAAM,MAAM,WAAW,GAAG,WAAW,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,CAAC;AAcvE,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW,CAkB9D;AAED,2FAA2F;AAC3F,wBAAgB,eAAe,CAAC,QAAQ,EAAE,WAAW,GAAG,MAAM,CAO7D;AAED,MAAM,WAAW,YAAY;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,gFAAgF;IAChF,MAAM,EAAE,MAAM,CAAC;IACf,wCAAwC;IACxC,MAAM,EAAE,MAAM,CAAC;CAClB;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CAC/B,MAAM,EAAE,OAAO,EACf,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,GAAG,EAAE,MAAM,EACX,GAAG,EAAE,MAAM,EACX,QAAQ,CAAC,EAAE,MAAM,EACjB,QAAQ,CAAC,EAAE,MAAM,GAClB,YAAY,CAqBd;AAED,0EAA0E;AAC1E,wBAAgB,iBAAiB,CAC7B,OAAO,EAAE,YAAY,EACrB,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,MAAM,GACb;IAAE,CAAC,EAAE,MAAM,CAAC;IAAC,CAAC,EAAE,MAAM,CAAA;CAAE,CAK1B;AAED,sEAAsE;AACtE,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAWlF;AAED,sFAAsF;AACtF,wBAAgB,iBAAiB,CAC7B,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,MAAM,GACf;IAAE,CAAC,EAAE,MAAM,CAAC;IAAC,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAUzC;AAgJD;;;;;GAKG;AACH,wBAAsB,aAAa,CAC/B,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EACd,SAAS,SAAM,GAChB,OAAO,CAAC,MAAM,CAAC,CAQjB"}
|
|
@@ -3,12 +3,52 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.getProviderForModel = getProviderForModel;
|
|
7
|
+
exports.getApiKeyEnvVar = getApiKeyEnvVar;
|
|
6
8
|
exports.computeCoordMapping = computeCoordMapping;
|
|
7
9
|
exports.applyCoordMapping = applyCoordMapping;
|
|
8
10
|
exports.buildVisionPrompt = buildVisionPrompt;
|
|
9
11
|
exports.parseVisionCoords = parseVisionCoords;
|
|
10
12
|
exports.callVisionLLM = callVisionLLM;
|
|
11
13
|
const sdk_1 = __importDefault(require("@anthropic-ai/sdk"));
|
|
14
|
+
const client_bedrock_runtime_1 = require("@aws-sdk/client-bedrock-runtime");
|
|
15
|
+
/** Infers the LLM provider from the model identifier. */
|
|
16
|
+
const SUPPORTED_MODELS = [
|
|
17
|
+
'claude-* (e.g. claude-sonnet-4-6)',
|
|
18
|
+
'gpt-* (e.g. gpt-4o)',
|
|
19
|
+
'o1, o3, o4, o1-mini, o3-pro, …',
|
|
20
|
+
'gemini-* (e.g. gemini-1.5-pro)',
|
|
21
|
+
'amazon.nova-* (e.g. amazon.nova-pro-v1:0)',
|
|
22
|
+
'us.amazon.nova-* (cross-region inference, e.g. us.amazon.nova-pro-v1:0)',
|
|
23
|
+
'eu.amazon.nova-* (cross-region inference)',
|
|
24
|
+
'ap.amazon.nova-* (cross-region inference)',
|
|
25
|
+
];
|
|
26
|
+
function getProviderForModel(model) {
|
|
27
|
+
const lower = model.toLowerCase();
|
|
28
|
+
if (lower.startsWith('gpt-') || /^o\d/.test(lower)) {
|
|
29
|
+
return 'openai';
|
|
30
|
+
}
|
|
31
|
+
if (lower.startsWith('gemini-')) {
|
|
32
|
+
return 'google';
|
|
33
|
+
}
|
|
34
|
+
if (lower.startsWith('claude-')) {
|
|
35
|
+
return 'anthropic';
|
|
36
|
+
}
|
|
37
|
+
if (/^(us\.|eu\.|ap\.)?amazon\./.test(lower)) {
|
|
38
|
+
return 'amazon';
|
|
39
|
+
}
|
|
40
|
+
throw new Error(`Unsupported model: "${model}". ` +
|
|
41
|
+
`Supported model prefixes are:\n ${SUPPORTED_MODELS.join('\n ')}`);
|
|
42
|
+
}
|
|
43
|
+
/** Returns the environment variable name that holds the API key for the given provider. */
|
|
44
|
+
function getApiKeyEnvVar(provider) {
|
|
45
|
+
switch (provider) {
|
|
46
|
+
case 'openai': return 'OPENAI_API_KEY';
|
|
47
|
+
case 'google': return 'GEMINI_API_KEY';
|
|
48
|
+
case 'amazon': return 'AWS_ACCESS_KEY_ID';
|
|
49
|
+
default: return 'ANTHROPIC_API_KEY';
|
|
50
|
+
}
|
|
51
|
+
}
|
|
12
52
|
/**
|
|
13
53
|
* Pure computation of the mapping from screenshot pixel coordinates to actual
|
|
14
54
|
* screen coordinates. All driver-specific calls are resolved by the caller.
|
|
@@ -64,12 +104,7 @@ function parseVisionCoords(raw, prompt) {
|
|
|
64
104
|
}
|
|
65
105
|
return parsed;
|
|
66
106
|
}
|
|
67
|
-
|
|
68
|
-
* Sends a base64 screenshot + text prompt to a Claude vision model and returns
|
|
69
|
-
* the raw text response. The caller is responsible for building the prompt and
|
|
70
|
-
* parsing the result.
|
|
71
|
-
*/
|
|
72
|
-
async function callVisionLLM(base64, textPrompt, model, apiKey, maxTokens = 256) {
|
|
107
|
+
async function callAnthropicVision(base64, textPrompt, model, apiKey, maxTokens) {
|
|
73
108
|
const client = new sdk_1.default({ apiKey });
|
|
74
109
|
const response = await client.messages.create({
|
|
75
110
|
model,
|
|
@@ -87,4 +122,116 @@ async function callVisionLLM(base64, textPrompt, model, apiKey, maxTokens = 256)
|
|
|
87
122
|
});
|
|
88
123
|
return response.content.find((b) => b.type === 'text')?.text ?? '';
|
|
89
124
|
}
|
|
125
|
+
async function callOpenAIVision(base64, textPrompt, model, apiKey, maxTokens) {
|
|
126
|
+
const res = await fetch('https://api.openai.com/v1/chat/completions', {
|
|
127
|
+
method: 'POST',
|
|
128
|
+
headers: {
|
|
129
|
+
'Content-Type': 'application/json',
|
|
130
|
+
'Authorization': `Bearer ${apiKey}`,
|
|
131
|
+
},
|
|
132
|
+
body: JSON.stringify({
|
|
133
|
+
model,
|
|
134
|
+
max_tokens: maxTokens,
|
|
135
|
+
messages: [{
|
|
136
|
+
role: 'user',
|
|
137
|
+
content: [
|
|
138
|
+
{ type: 'image_url', image_url: { url: `data:image/png;base64,${base64}` } },
|
|
139
|
+
{ type: 'text', text: textPrompt },
|
|
140
|
+
],
|
|
141
|
+
}],
|
|
142
|
+
}),
|
|
143
|
+
});
|
|
144
|
+
if (!res.ok) {
|
|
145
|
+
const body = await res.text();
|
|
146
|
+
let message;
|
|
147
|
+
try {
|
|
148
|
+
message = JSON.parse(body).error?.message ?? body;
|
|
149
|
+
}
|
|
150
|
+
catch {
|
|
151
|
+
message = body || res.statusText;
|
|
152
|
+
}
|
|
153
|
+
throw new Error(`OpenAI API error: ${message}`);
|
|
154
|
+
}
|
|
155
|
+
const data = await res.json();
|
|
156
|
+
const content = data.choices?.[0]?.message?.content;
|
|
157
|
+
if (typeof content !== 'string') {
|
|
158
|
+
throw new Error(`Unexpected response from OpenAI model "${model}": no text content in choices[0].message.content`);
|
|
159
|
+
}
|
|
160
|
+
return content;
|
|
161
|
+
}
|
|
162
|
+
async function callGoogleVision(base64, textPrompt, model, apiKey, maxTokens) {
|
|
163
|
+
const url = `https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent`;
|
|
164
|
+
const res = await fetch(url, {
|
|
165
|
+
method: 'POST',
|
|
166
|
+
headers: { 'Content-Type': 'application/json', 'x-goog-api-key': apiKey },
|
|
167
|
+
body: JSON.stringify({
|
|
168
|
+
contents: [{
|
|
169
|
+
parts: [
|
|
170
|
+
{ inline_data: { mime_type: 'image/png', data: base64 } },
|
|
171
|
+
{ text: textPrompt },
|
|
172
|
+
],
|
|
173
|
+
}],
|
|
174
|
+
generationConfig: { maxOutputTokens: maxTokens },
|
|
175
|
+
}),
|
|
176
|
+
});
|
|
177
|
+
if (!res.ok) {
|
|
178
|
+
const body = await res.text();
|
|
179
|
+
let message;
|
|
180
|
+
try {
|
|
181
|
+
message = JSON.parse(body).error?.message ?? body;
|
|
182
|
+
}
|
|
183
|
+
catch {
|
|
184
|
+
message = body || res.statusText;
|
|
185
|
+
}
|
|
186
|
+
throw new Error(`Gemini API error: ${message}`);
|
|
187
|
+
}
|
|
188
|
+
const data = await res.json();
|
|
189
|
+
const text = data.candidates?.[0]?.content?.parts?.[0]?.text;
|
|
190
|
+
if (typeof text !== 'string') {
|
|
191
|
+
throw new Error(`Unexpected response from Gemini model "${model}": no text in candidates[0].content.parts[0].text`);
|
|
192
|
+
}
|
|
193
|
+
return text;
|
|
194
|
+
}
|
|
195
|
+
async function callAmazonBedrockVision(base64, textPrompt, model, maxTokens) {
|
|
196
|
+
const client = new client_bedrock_runtime_1.BedrockRuntimeClient({
|
|
197
|
+
region: process.env.AWS_REGION ?? process.env.AWS_DEFAULT_REGION ?? 'us-east-1',
|
|
198
|
+
});
|
|
199
|
+
const command = new client_bedrock_runtime_1.ConverseCommand({
|
|
200
|
+
modelId: model,
|
|
201
|
+
messages: [{
|
|
202
|
+
role: 'user',
|
|
203
|
+
content: [
|
|
204
|
+
{
|
|
205
|
+
image: {
|
|
206
|
+
format: 'png',
|
|
207
|
+
source: { bytes: Buffer.from(base64, 'base64') },
|
|
208
|
+
},
|
|
209
|
+
},
|
|
210
|
+
{ text: textPrompt },
|
|
211
|
+
],
|
|
212
|
+
}],
|
|
213
|
+
inferenceConfig: { maxTokens },
|
|
214
|
+
});
|
|
215
|
+
const response = await client.send(command);
|
|
216
|
+
const text = response.output?.message?.content?.find((b) => 'text' in b && typeof b.text === 'string');
|
|
217
|
+
if (!text) {
|
|
218
|
+
throw new Error(`Unexpected response from Amazon Bedrock model "${model}": no text content in output`);
|
|
219
|
+
}
|
|
220
|
+
return text.text;
|
|
221
|
+
}
|
|
222
|
+
/**
|
|
223
|
+
* Sends a base64 screenshot + text prompt to a vision model and returns the raw
|
|
224
|
+
* text response. Dispatches to Anthropic, OpenAI, Google Gemini, or Amazon Bedrock
|
|
225
|
+
* based on the model name prefix. The caller is responsible for building the prompt
|
|
226
|
+
* and parsing the result.
|
|
227
|
+
*/
|
|
228
|
+
async function callVisionLLM(base64, textPrompt, model, apiKey, maxTokens = 256) {
|
|
229
|
+
const provider = getProviderForModel(model);
|
|
230
|
+
switch (provider) {
|
|
231
|
+
case 'openai': return callOpenAIVision(base64, textPrompt, model, apiKey, maxTokens);
|
|
232
|
+
case 'google': return callGoogleVision(base64, textPrompt, model, apiKey, maxTokens);
|
|
233
|
+
case 'amazon': return callAmazonBedrockVision(base64, textPrompt, model, maxTokens);
|
|
234
|
+
default: return callAnthropicVision(base64, textPrompt, model, apiKey, maxTokens);
|
|
235
|
+
}
|
|
236
|
+
}
|
|
90
237
|
//# sourceMappingURL=vision-utils.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vision-utils.js","sourceRoot":"","sources":["../../lib/vision-utils.ts"],"names":[],"mappings":";;;;;
|
|
1
|
+
{"version":3,"file":"vision-utils.js","sourceRoot":"","sources":["../../lib/vision-utils.ts"],"names":[],"mappings":";;;;;AAiBA,kDAkBC;AAGD,0CAOC;AAeD,kDAgCC;AAGD,8CASC;AAGD,8CAWC;AAGD,8CAaC;AAsJD,sCAcC;AA1SD,4DAA0C;AAC1C,4EAAwF;AAIxF,yDAAyD;AACzD,MAAM,gBAAgB,GAAG;IACrB,2CAA2C;IAC3C,gCAAgC;IAChC,gCAAgC;IAChC,wCAAwC;IACxC,8CAA8C;IAC9C,yEAAyE;IACzE,2CAA2C;IAC3C,2CAA2C;CAC9C,CAAC;AAEF,SAAgB,mBAAmB,CAAC,KAAa;IAC7C,MAAM,KAAK,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC;IAClC,IAAI,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACjD,OAAO,QAAQ,CAAC;IACpB,CAAC;IACD,IAAI,KAAK,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC9B,OAAO,QAAQ,CAAC;IACpB,CAAC;IACD,IAAI,KAAK,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC9B,OAAO,WAAW,CAAC;IACvB,CAAC;IACD,IAAI,4BAA4B,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAC3C,OAAO,QAAQ,CAAC;IACpB,CAAC;IACD,MAAM,IAAI,KAAK,CACX,uBAAuB,KAAK,KAAK;QACjC,oCAAoC,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CACtE,CAAC;AACN,CAAC;AAED,2FAA2F;AAC3F,SAAgB,eAAe,CAAC,QAAqB;IACjD,QAAQ,QAAQ,EAAE,CAAC;QACf,KAAK,QAAQ,CAAC,CAAC,OAAO,gBAAgB,CAAC;QACvC,KAAK,QAAQ,CAAC,CAAC,OAAO,gBAAgB,CAAC;QACvC,KAAK,QAAQ,CAAC,CAAC,OAAO,mBAAmB,CAAC;QAC1C,OAAO,CAAC,CAAC,OAAO,mBAAmB,CAAC;IACxC,CAAC;AACL,CAAC;AAWD;;;GAGG;AACH,SAAgB,mBAAmB,CAC/B,MAAe,EACf,KAAa,EACb,KAAa,EACb,KAAa,EACb,KAAa,EACb,QAAgB,EAChB,GAAW,EACX,GAAW,EACX,QAAiB,EACjB,QAAiB;IAEjB,IAAI,MAAM,EAAE,CAAC;QACT,OAAO;YACH,OAAO,EAAE,CAAC;YACV,OAAO,EAAE,CAAC;YACV,MAAM,EAAE,CAAC,QAAQ,IAAI,GAAG,CAAC,GAAG,GAAG;YAC/B,MAAM,EAAE,CAAC,QAAQ,IAAI,GAAG,CAAC,GAAG,GAAG;SAClC,CAAC;IACN,CAAC;IAED,8EAA8E;IAC9E,4DAA4D;IAC5D,MAAM,SAAS,GAAG,QAAQ,GAAG,IAAI,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,QAAQ,GAAG,GAAG,CAAC,GAAG,GAAG,GAAG,IAAI,CAAC;IACnF,MAAM,KAAK,GAAG,SAAS,CAAC,CAAC,CAAC,KAAK,GAAG,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC;IACnD,MAAM,KAAK,GAAG,SAAS,CAAC,CAAC,CAAC,KAAK,GAAG,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC;IACnD,OAAO;QACH,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK;QACzD,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK;QACzD,MAAM,EAAE,KAAK,GAAG,GAAG;QACnB,MAAM,EAAE,KAAK,GAAG,GAAG;KACtB,CAAC;AACN,CAAC;AAED,0EAA0E;AAC1E,SAAgB,iBAAiB,CAC7B,OAAqB,EACrB,IAAY,EACZ,IAAY;IAEZ,OAAO;QACH,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,GAAG,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC;QACtD,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,GAAG,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC;KACzD,CAAC;AACN,CAAC;AAED,sEAAsE;AACtE,SAAgB,iBAAiB,CAAC,MAAc,EAAE,GAAW,EAAE,GAAW;IACtE,OAAO,CACH,oDAAoD,MAAM,OAAO;QACjE,gBAAgB,GAAG,IAAI,GAAG,cAAc;QACxC,oDAAoD;QACpD,0FAA0F;QAC1F,oFAAoF;QACpF,2BAA2B,GAAG,6BAA6B,GAAG,OAAO;QACrE,iDAAiD;QACjD,4CAA4C,CAC/C,CAAC;AACN,CAAC;AAED,sFAAsF;AACtF,SAAgB,iBAAiB,CAC7B,GAAW,EACX,MAAc;IAEd,MAAM,SAAS,GAAG,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IAC3C,IAAI,CAAC,SAAS,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,4BAA4B,GAAG,EAAE,CAAC,CAAC;IACvD,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAA4C,CAAC;IACnF,IAAI,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CAAC,uBAAuB,MAAM,GAAG,CAAC,CAAC;IACtD,CAAC;IACD,OAAO,MAAM,CAAC;AAClB,CAAC;AAED,KAAK,UAAU,mBAAmB,CAC9B,MAAc,EACd,UAAkB,EAClB,KAAa,EACb,MAAc,EACd,SAAiB;IAEjB,MAAM,MAAM,GAAG,IAAI,aAAS,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;IACzC,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;QAC1C,KAAK;QACL,UAAU,EAAE,SAAS;QACrB,QAAQ,EAAE,CAAC;gBACP,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE;oBACL;wBACI,IAAI,EAAE,OAAO;wBACb,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE;qBACpE;oBACD,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE;iBACrC;aACJ,CAAC;KACL,CAAC,CAAC;IACH,OAAO,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,EAAE,IAAI,IAAI,EAAE,CAAC;AACvE,CAAC;AAED,KAAK,UAAU,gBAAgB,CAC3B,MAAc,EACd,UAAkB,EAClB,KAAa,EACb,MAAc,EACd,SAAiB;IAEjB,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,4CAA4C,EAAE;QAClE,MAAM,EAAE,MAAM;QACd,OAAO,EAAE;YACL,cAAc,EAAE,kBAAkB;YAClC,eAAe,EAAE,UAAU,MAAM,EAAE;SACtC;QACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;YACjB,KAAK;YACL,UAAU,EAAE,SAAS;YACrB,QAAQ,EAAE,CAAC;oBACP,IAAI,EAAE,MAAM;oBACZ,OAAO,EAAE;wBACL,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE,yBAAyB,MAAM,EAAE,EAAE,EAAE;wBAC5E,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE;qBACrC;iBACJ,CAAC;SACL,CAAC;KACL,CAAC,CAAC;IACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;QACV,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAC9B,IAAI,OAAe,CAAC;QACpB,IAAI,CAAC;YACD,OAAO,GAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAqC,CAAC,KAAK,EAAE,OAAO,IAAI,IAAI,CAAC;QAC3F,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,GAAG,IAAI,IAAI,GAAG,CAAC,UAAU,CAAC;QACrC,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,qBAAqB,OAAO,EAAE,CAAC,CAAC;IACpD,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAA2D,CAAC;IACvF,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC;IACpD,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,0CAA0C,KAAK,kDAAkD,CAAC,CAAC;IACvH,CAAC;IACD,OAAO,OAAO,CAAC;AACnB,CAAC;AAED,KAAK,UAAU,gBAAgB,CAC3B,MAAc,EACd,UAAkB,EAClB,KAAa,EACb,MAAc,EACd,SAAiB;IAEjB,MAAM,GAAG,GAAG,2DAA2D,KAAK,kBAAkB,CAAC;IAC/F,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;QACzB,MAAM,EAAE,MAAM;QACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,EAAE;QACzE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;YACjB,QAAQ,EAAE,CAAC;oBACP,KAAK,EAAE;wBACH,EAAE,WAAW,EAAE,EAAE,SAAS,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE;wBACzD,EAAE,IAAI,EAAE,UAAU,EAAE;qBACvB;iBACJ,CAAC;YACF,gBAAgB,EAAE,EAAE,eAAe,EAAE,SAAS,EAAE;SACnD,CAAC;KACL,CAAC,CAAC;IACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;QACV,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAC9B,IAAI,OAAe,CAAC;QACpB,IAAI,CAAC;YACD,OAAO,GAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAqC,CAAC,KAAK,EAAE,OAAO,IAAI,IAAI,CAAC;QAC3F,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,GAAG,IAAI,IAAI,GAAG,CAAC,UAAU,CAAC;QACrC,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,qBAAqB,OAAO,EAAE,CAAC,CAAC;IACpD,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAE1B,CAAC;IACF,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC;IAC7D,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CAAC,0CAA0C,KAAK,mDAAmD,CAAC,CAAC;IACxH,CAAC;IACD,OAAO,IAAI,CAAC;AAChB,CAAC;AAED,KAAK,UAAU,uBAAuB,CAClC,MAAc,EACd,UAAkB,EAClB,KAAa,EACb,SAAiB;IAEjB,MAAM,MAAM,GAAG,IAAI,6CAAoB,CAAC;QACpC,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,WAAW;KAClF,CAAC,CAAC;IACH,MAAM,OAAO,GAAG,IAAI,wCAAe,CAAC;QAChC,OAAO,EAAE,KAAK;QACd,QAAQ,EAAE,CAAC;gBACP,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE;oBACL;wBACI,KAAK,EAAE;4BACH,MAAM,EAAE,KAAK;4BACb,MAAM,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAE;yBACnD;qBACJ;oBACD,EAAE,IAAI,EAAE,UAAU,EAAE;iBACvB;aACJ,CAAC;QACF,eAAe,EAAE,EAAE,SAAS,EAAE;KACjC,CAAC,CAAC;IACH,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC5C,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,IAAI,CAAC,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAiC,CAAC;IACvI,IAAI,CAAC,IAAI,EAAE,CAAC;QACR,MAAM,IAAI,KAAK,CAAC,kDAAkD,KAAK,8BAA8B,CAAC,CAAC;IAC3G,CAAC;IACD,OAAO,IAAI,CAAC,IAAI,CAAC;AACrB,CAAC;AAED;;;;;GAKG;AACI,KAAK,UAAU,aAAa,CAC/B,MAAc,EACd,UAAkB,EAClB,KAAa,EACb,MAAc,EACd,SAAS,GAAG,GAAG;IAEf,MAAM,QAAQ,GAAG,mBAAmB,CAAC,KAAK,CAAC,CAAC;IAC5C,QAAQ,QAAQ,EAAE,CAAC;QACf,KAAK,QAAQ,CAAC,CAAC,OAAO,gBAAgB,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;QACrF,KAAK,QAAQ,CAAC,CAAC,OAAO,gBAAgB,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;QACrF,KAAK,QAAQ,CAAC,CAAC,OAAO,uBAAuB,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;QACpF,OAAO,CAAC,CAAC,OAAO,mBAAmB,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;IACtF,CAAC;AACL,CAAC"}
|