free-computer-use 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/README.md +74 -53
- package/dist/actions/executor.d.ts +2 -1
- package/dist/actions/executor.js +151 -39
- package/dist/actions/executor.js.map +1 -1
- package/dist/actions/schema.js +4 -1
- package/dist/actions/schema.js.map +1 -1
- package/dist/adapters/generic.js +10 -0
- package/dist/adapters/generic.js.map +1 -1
- package/dist/agent/Agent.d.ts +0 -1
- package/dist/agent/Agent.js +3 -8
- package/dist/agent/Agent.js.map +1 -1
- package/dist/agent/goalCriteria.js +6 -0
- package/dist/agent/goalCriteria.js.map +1 -1
- package/dist/browser/DomExtractor.d.ts +1 -1
- package/dist/browser/DomExtractor.js +34 -14
- package/dist/browser/DomExtractor.js.map +1 -1
- package/dist/browser/Interaction.d.ts +8 -1
- package/dist/browser/Interaction.js +36 -2
- package/dist/browser/Interaction.js.map +1 -1
- package/dist/browser/Observer.js +19 -1
- package/dist/browser/Observer.js.map +1 -1
- package/dist/browser/PageCompressor.d.ts +9 -21
- package/dist/browser/PageCompressor.js +10 -6
- package/dist/browser/PageCompressor.js.map +1 -1
- package/dist/browser/SelectorEngine.js +6 -3
- package/dist/browser/SelectorEngine.js.map +1 -1
- package/dist/llm/ClaudeSubscriptionProvider.js +5 -4
- package/dist/llm/ClaudeSubscriptionProvider.js.map +1 -1
- package/dist/llm/CodexSubscriptionProvider.js +5 -4
- package/dist/llm/CodexSubscriptionProvider.js.map +1 -1
- package/dist/llm/cliEnvironment.d.ts +2 -0
- package/dist/llm/cliEnvironment.js +12 -0
- package/dist/llm/cliEnvironment.js.map +1 -1
- package/dist/llm/prompts.d.ts +1 -1
- package/dist/llm/prompts.js +1 -1
- package/dist/server/index.js +9 -1
- package/dist/server/index.js.map +1 -1
- package/dist/verification/Verifier.js +7 -5
- package/dist/verification/Verifier.js.map +1 -1
- package/docs/ADOPTION.md +14 -2
- package/docs/LOCAL_VALIDATION.md +272 -6
- package/docs/RELEASE_CHECKLIST.md +23 -22
- package/docs/THREAT_MODEL.md +2 -2
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,23 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.2.0 - 2026-09-26
|
|
4
|
+
|
|
5
|
+
### Added
|
|
6
|
+
|
|
7
|
+
- Deterministic, provider-free extraction of visible page text and links.
|
|
8
|
+
- Bounded extraction of the first 1–1000 visible links, with exact-count
|
|
9
|
+
verification.
|
|
10
|
+
|
|
11
|
+
### Changed
|
|
12
|
+
|
|
13
|
+
- Zero-timeout selector checks return immediately instead of waiting through a
|
|
14
|
+
polling interval.
|
|
15
|
+
|
|
16
|
+
### Fixed
|
|
17
|
+
|
|
18
|
+
- Element verification evaluates all matching elements, avoiding a false
|
|
19
|
+
failure when a later match satisfies the condition.
|
|
20
|
+
|
|
3
21
|
## 0.1.0
|
|
4
22
|
|
|
5
23
|
### Added
|
package/README.md
CHANGED
|
@@ -3,41 +3,84 @@
|
|
|
3
3
|

|
|
4
4
|
|
|
5
5
|
<p align="center">
|
|
6
|
+
<a href="https://www.npmjs.com/package/free-computer-use"><img alt="npm version" src="https://img.shields.io/npm/v/free-computer-use?style=flat-square&labelColor=18251f&color=c4e967"></a>
|
|
7
|
+
<a href="https://github.com/OthmaneBlial/FreeComputerUse/releases/latest"><img alt="Latest GitHub release" src="https://img.shields.io/github/v/release/OthmaneBlial/FreeComputerUse?style=flat-square&labelColor=18251f&color=c4e967"></a>
|
|
6
8
|
<a href="LICENSE"><img alt="MIT license" src="https://img.shields.io/badge/license-MIT-c4e967?style=flat-square&labelColor=18251f"></a>
|
|
9
|
+
<img alt="Node.js 22.13 or newer" src="https://img.shields.io/badge/Node.js-22.13%2B-43853d?style=flat-square&labelColor=18251f">
|
|
7
10
|
<img alt="TypeScript" src="https://img.shields.io/badge/TypeScript-3178c6?style=flat-square&labelColor=18251f">
|
|
8
11
|
<img alt="Playwright" src="https://img.shields.io/badge/Playwright-45ba4b?style=flat-square&labelColor=18251f">
|
|
9
|
-
<img alt="Local-first" src="https://img.shields.io/badge/local--first-browser_runs-c4e967?style=flat-square&labelColor=18251f">
|
|
10
12
|
</p>
|
|
11
13
|
|
|
12
14
|
<p align="center">
|
|
13
15
|
<a href="#quick-start">Quick start</a> ·
|
|
16
|
+
<a href="#no-model-key-needed">Try it without a model</a> ·
|
|
17
|
+
<a href="#watch-a-real-run">Watch the demo</a> ·
|
|
18
|
+
<a href="https://github.com/OthmaneBlial/FreeComputerUse/releases/latest">Latest release</a> ·
|
|
14
19
|
<a href="https://othmaneblial.github.io/FreeComputerUse/">Project site</a> ·
|
|
15
|
-
<a href="https://othmaneblial.github.io/FreeComputerUse/lab/index.html">Try the task lab</a> ·
|
|
16
20
|
<a href="SECURITY.md">Security</a>
|
|
17
21
|
</p>
|
|
18
22
|
|
|
19
|
-
**Your browser. Your model. Your call.**
|
|
23
|
+
**Your browser. Your model. Your call.** Give your browser a goal; the model proposes short action batches, and Playwright runs them locally. You approve website access and sensitive actions, then inspect what happened.
|
|
20
24
|
|
|
21
|
-
|
|
25
|
+
Use an API or a supported model CLI for open-ended tasks. For common page-reading jobs, skip model setup entirely: FreeComputerUse can read visible text and links with deterministic local strategies.
|
|
26
|
+
|
|
27
|
+
## No model key needed
|
|
28
|
+
|
|
29
|
+
Install the release and start the local dashboard:
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
npm install --global free-computer-use@0.2.0
|
|
33
|
+
FCU_BROWSER_CHANNEL=chrome agent doctor
|
|
34
|
+
FCU_BROWSER_CHANNEL=chrome agent ui
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
Open **http://127.0.0.1:4318**, enter a page URL and one of these goals, then approve access to that website:
|
|
38
|
+
|
|
39
|
+
| Goal | Result |
|
|
40
|
+
| --- | --- |
|
|
41
|
+
| `Read the page` | Extract visible page text. |
|
|
42
|
+
| `Extract the links` | List visible link labels and URLs. |
|
|
43
|
+
| `Extract the first 5 links` | Return a bounded list of the first five links. |
|
|
44
|
+
| `Extract the table` | Try the synthetic [practice revenue table](https://othmaneblial.github.io/FreeComputerUse/lab/reports.html). |
|
|
45
|
+
|
|
46
|
+
These narrow workflows use local strategies and make zero model calls. General tasks need a configured provider. The declared minimum is Node.js 22.13; the verified setup is macOS 26.6 on Apple Silicon, Node 25.9.0 and system Chrome 154.0.8037.57. Other OS/browser combinations and the declared Node minimum remain unverified. See the [support matrix](docs/SUPPORT_MATRIX.md).
|
|
22
47
|
|
|
23
48
|
## Watch a real run
|
|
24
49
|
|
|
25
|
-
[](assets/readme/product-demo.mp4)
|
|
51
|
+
|
|
52
|
+
[Watch the 22-second dashboard walkthrough](assets/readme/product-demo.mp4) · [Watch the short portrait cut](assets/readme/product-demo-portrait.mp4)
|
|
26
53
|
|
|
27
|
-
|
|
54
|
+
This recording uses the 0.1.0 package. It shows the npm install, local dashboard, explicit site approval and a verified synthetic table result: three rows, one browser action and zero model calls. The same practice workflow remains available. It is a deterministic demo, not a claim about arbitrary websites.
|
|
55
|
+
|
|
56
|
+
[](https://othmaneblial.github.io/FreeComputerUse/#watch)
|
|
57
|
+
|
|
58
|
+
One recorded task followed six pages to a checked incident brief: **21 successful actions, 10 model calls, one repaired failure**. The configured cost estimate was **$0.00650**. This is one synthetic run, not a general success-rate claim. [Watch the full recording](https://othmaneblial.github.io/FreeComputerUse/lab/media/incident-demo.mp4) · [Inspect the evidence](assets/readme/incident-evidence.json).
|
|
28
59
|
|
|
29
60
|
## How it works
|
|
30
61
|
|
|
31
|
-
1. **Observe** the page
|
|
62
|
+
1. **Observe** the page DOM and accessible controls.
|
|
32
63
|
2. **Plan** a bounded batch with the model you choose.
|
|
33
64
|
3. **Execute** browser actions locally with Playwright; direct navigation to a new origin asks for approval.
|
|
34
|
-
4. **Verify** results, repair only what failed, then
|
|
65
|
+
4. **Verify** results, repair only what failed, then replay compatible learned workflows later.
|
|
35
66
|
|
|
36
|
-
The model does not run shell commands or arbitrary JavaScript. Browser previews stay local; the
|
|
67
|
+
The model does not run shell commands or arbitrary JavaScript. Browser previews stay local; the task and selected page context go to your configured model provider. Screenshots are not sent.
|
|
37
68
|
|
|
38
69
|
## Quick start
|
|
39
70
|
|
|
40
|
-
Requires
|
|
71
|
+
Requires Node.js 22.13+, npm and an installed browser. The verified setup selects installed Chrome and avoids a separate Playwright browser download.
|
|
72
|
+
|
|
73
|
+
### Install from npm
|
|
74
|
+
|
|
75
|
+
```bash
|
|
76
|
+
npm install --global free-computer-use@0.2.0
|
|
77
|
+
FCU_BROWSER_CHANNEL=chrome agent doctor
|
|
78
|
+
FCU_BROWSER_CHANNEL=chrome agent ui
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
Open **http://127.0.0.1:4318**. Try a keyless goal above, or configure a provider for model-planned tasks.
|
|
82
|
+
|
|
83
|
+
### Run from source
|
|
41
84
|
|
|
42
85
|
```bash
|
|
43
86
|
git clone https://github.com/OthmaneBlial/FreeComputerUse.git
|
|
@@ -47,9 +90,7 @@ cp .env.example .env
|
|
|
47
90
|
chmod 600 .env
|
|
48
91
|
```
|
|
49
92
|
|
|
50
|
-
Set `FCU_BROWSER_CHANNEL=chrome` in `.env` to use
|
|
51
|
-
|
|
52
|
-
For model-planned tasks, configure a provider in `.env`. The browser sandbox task below works without a model key:
|
|
93
|
+
Set `FCU_BROWSER_CHANNEL=chrome` in `.env` to use installed Chrome. For model-planned tasks, add the provider details to your local `.env`; for example:
|
|
53
94
|
|
|
54
95
|
```dotenv
|
|
55
96
|
FCU_BROWSER_CHANNEL=chrome
|
|
@@ -64,21 +105,11 @@ npm run agent -- doctor
|
|
|
64
105
|
npm run dev
|
|
65
106
|
```
|
|
66
107
|
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
**Try it without an API key:** open [the practice revenue table](https://othmaneblial.github.io/FreeComputerUse/lab/reports.html) and run the goal `Extract the table`. This narrow workflow has a deterministic local strategy; general tasks need a model provider.
|
|
70
|
-
|
|
71
|
-
## Troubleshooting
|
|
72
|
-
|
|
73
|
-
- **The browser does not start:** install Chrome, set `FCU_BROWSER_CHANNEL=chrome` in `.env`, then run `npm run agent -- doctor`. See the [verified platform limits](docs/SUPPORT_MATRIX.md); other OS/browser combinations are not certified here.
|
|
74
|
-
- **The dashboard says no model is configured:** the practice revenue-table task works without a provider. For model planning, configure one route from [Pick your model](#pick-your-model). `npm run agent -- doctor --api` makes an opt-in request to the configured provider; use it only when you want that network check.
|
|
75
|
-
- **A site is blocked or asks for approval:** normal mode asks before a new origin and blocks private/reserved DNS answers. Approve only the site needed for the task. Ultra mode disables those protections; do not use it as a workaround for a blocked destination.
|
|
76
|
-
- **The run finishes without proving the goal:** inspect the result checks and use a narrower goal. Browser clicks alone do not mean the requested outcome was verified.
|
|
77
|
-
- **You need to find or remove local data:** see [local storage, export, retention and deletion](docs/LOCAL_DATA.md). Data is not encrypted and is not automatically expired.
|
|
108
|
+
Then open **http://127.0.0.1:4318**, enter a starting URL and goal, and approve site access.
|
|
78
109
|
|
|
79
110
|
## Pick your model
|
|
80
111
|
|
|
81
|
-
Use a provider API key
|
|
112
|
+
Use a provider API key or sign in through the official CLI for a supported subscription. API usage and consumer subscriptions are separate billing products.
|
|
82
113
|
|
|
83
114
|
| Provider | Configuration |
|
|
84
115
|
| --- | --- |
|
|
@@ -89,51 +120,41 @@ Use a provider API key, or sign in through the official CLI for a supported subs
|
|
|
89
120
|
| ChatGPT plan | Install Codex CLI, sign in with `codex login`, set `LLM_PROVIDER=codex-subscription` |
|
|
90
121
|
| Claude Pro/Max plan | Install Claude Code 2.1.248+, sign in with `claude auth login` (not Console), set `LLM_PROVIDER=claude-subscription` |
|
|
91
122
|
|
|
92
|
-
|
|
123
|
+
DeepSeek Flash is the measured default. Other provider routes are not all live-tested. Subscription modes use local CLI sign-in, need no API key, respect plan limits, and disable their local agent tools and MCP servers while planning. See [provider setup and limits](docs/PROVIDERS.md), the [support matrix](docs/SUPPORT_MATRIX.md) and [.env.example](.env.example).
|
|
93
124
|
|
|
94
125
|
## Connect an MCP host
|
|
95
126
|
|
|
96
|
-
Run FreeComputerUse as a local MCP server over `stdio` with `agent mcp`. It
|
|
97
|
-
provides tools to start a task, inspect the observed page, follow and verify the
|
|
98
|
-
result, and stop a task. Site and sensitive-action approvals still require a
|
|
99
|
-
human response in an MCP host that supports form elicitation. No HTTP endpoint
|
|
100
|
-
is opened. See the [MCP setup and validation limits](docs/MCP.md). The separate
|
|
101
|
-
provider CLI adapters keep their own tools and MCP disabled while planning.
|
|
127
|
+
Run FreeComputerUse as a local MCP server over `stdio` with `agent mcp`. It exposes bounded tools to start a task, inspect the page, follow and verify results, and stop a task. Site and sensitive-action approvals still need a human response in an MCP host that supports form elicitation. No HTTP endpoint is opened. See the [MCP setup and validation limits](docs/MCP.md).
|
|
102
128
|
|
|
103
|
-
##
|
|
104
|
-
|
|
105
|
-
**Can I try it without a model key?** Yes, the practice revenue-table workflow runs with a deterministic local strategy. General tasks need a configured model.
|
|
129
|
+
## Security and privacy
|
|
106
130
|
|
|
107
|
-
|
|
131
|
+
- Normal mode asks before direct navigation to a new origin and blocks private or reserved DNS answers. A loopback-only proxy connects to the vetted numeric address for browser HTTP(S) and WebSocket traffic. Ultra mode and the low-level `allowExternal` option opt out of origin and private-address checks; Ultra mode is off by default.
|
|
132
|
+
- Sensitive actions have a separate confirmation gate. Read the [security boundaries](SECURITY.md) before using personal or sensitive data.
|
|
133
|
+
- Browser execution, profiles, history and downloads stay on your machine. Selected task and page context goes to the provider you choose; screenshots are not sent.
|
|
134
|
+
- Planner inputs use aliases for local profile and file values, not their contents. Local data is **not encrypted** and does not expire automatically. See [storage, export, retention and deletion](docs/LOCAL_DATA.md).
|
|
135
|
+
- Chromium profiles disable WebRTC UDP that the proxy cannot carry. Sites that require direct UDP for voice or video may fail.
|
|
108
136
|
|
|
109
|
-
|
|
137
|
+
## Evidence and limits
|
|
110
138
|
|
|
111
|
-
**
|
|
139
|
+
A focused public suite recorded **18 September 2026** passed 14/14 first-run tasks and 14/14 compatible learned repeats with independent checks. First runs used 18 model calls; repeats used zero. These single-trial sandbox results do not predict success on arbitrary websites. [Report](artifacts/benchmark-public.json) · [Method and limitations](docs/BENCHMARKS.md).
|
|
112
140
|
|
|
113
|
-
|
|
141
|
+
Provider, operating-system, browser and accessibility evidence stays bounded to the [support matrix](docs/SUPPORT_MATRIX.md). Contract tests do not certify a live provider; builds do not certify an untested platform.
|
|
114
142
|
|
|
115
|
-
|
|
116
|
-
- Browser execution, profiles, history and downloads stay on your machine. Page context needed for a plan goes to the chosen model provider; screenshots are not sent.
|
|
117
|
-
- The planner receives aliases for local profile and file values, not their contents. Local storage is **not encrypted**.
|
|
118
|
-
- Chromium profiles disable WebRTC UDP that the proxy cannot carry. Sites needing direct UDP for voice/video may fail; one local Chrome STUN fixture confirms no direct packet reached its receiver. Other non-HTTP traffic remains unverified.
|
|
119
|
-
- Runs record actions, checks, repairs, token estimates and workflow reuse so you can inspect what happened.
|
|
143
|
+
## FAQ
|
|
120
144
|
|
|
121
|
-
Read
|
|
122
|
-
See [where local data is stored and how to inspect, export or delete it](docs/LOCAL_DATA.md).
|
|
145
|
+
**Can I try it without an API key?** Yes. Use `Read the page`, `Extract the links`, `Extract the first 5 links`, or the synthetic revenue-table task. These workflows make zero model calls.
|
|
123
146
|
|
|
124
|
-
|
|
147
|
+
**Does the model receive screenshots or browser profiles?** No. The selected task and page context go to your provider; screenshots and browser profiles stay local. Local data is not encrypted.
|
|
125
148
|
|
|
126
|
-
|
|
149
|
+
**Which platform is verified?** macOS 26.6 on Apple Silicon, Node 25.9.0 and system Chrome 154.0.8037.57. Check the [support matrix](docs/SUPPORT_MATRIX.md) for exact coverage and limits.
|
|
127
150
|
|
|
128
151
|
## Explore and contribute
|
|
129
152
|
|
|
130
153
|
- [Project site and task library](https://othmaneblial.github.io/FreeComputerUse/)
|
|
131
|
-
- [
|
|
154
|
+
- [Latest GitHub release](https://github.com/OthmaneBlial/FreeComputerUse/releases/latest)
|
|
132
155
|
- [Useful browser-task examples](docs/USEFUL_EXAMPLES.md)
|
|
133
|
-
- [
|
|
134
|
-
- [
|
|
135
|
-
- [Local validation commands](docs/LOCAL_VALIDATION.md)
|
|
136
|
-
- [Contributing](CONTRIBUTING.md) · [Changelog](CHANGELOG.md)
|
|
156
|
+
- [Architecture](docs/ARCHITECTURE.md) · [implementation notes](docs/IMPLEMENTATION.md)
|
|
157
|
+
- [Local validation commands](docs/LOCAL_VALIDATION.md) · [contributing](CONTRIBUTING.md) · [changelog](CHANGELOG.md)
|
|
137
158
|
|
|
138
159
|
Useful contributions: reproducible browser tasks, safer permission scopes and checks that make results easier to trust. Keep shared examples synthetic or read-only; remove credentials and personal data from traces.
|
|
139
160
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { type Action, type Condition } from './schema.js';
|
|
2
2
|
import { type ConfirmationPolicy } from './policy.js';
|
|
3
|
-
import type
|
|
3
|
+
import { type Browser } from '../browser/Browser.js';
|
|
4
4
|
import type { Observer } from '../browser/Observer.js';
|
|
5
5
|
import type { VariableResolver } from '../profile/VariableResolver.js';
|
|
6
6
|
import type { Control } from '../agent/Control.js';
|
|
@@ -33,6 +33,7 @@ export declare class Executor {
|
|
|
33
33
|
});
|
|
34
34
|
semantic(action: Action): Action;
|
|
35
35
|
semanticCondition(condition: Condition): Condition;
|
|
36
|
+
private readTextValue;
|
|
36
37
|
private storeDownload;
|
|
37
38
|
run(input: Action): Promise<ActionResult>;
|
|
38
39
|
}
|
package/dist/actions/executor.js
CHANGED
|
@@ -3,6 +3,8 @@ import { randomUUID } from 'node:crypto';
|
|
|
3
3
|
import { basename, resolve } from 'node:path';
|
|
4
4
|
import { ActionSchema } from './schema.js';
|
|
5
5
|
import { sensitiveReason } from './policy.js';
|
|
6
|
+
import { checkedHttpURL } from '../browser/Browser.js';
|
|
7
|
+
import { SecurityBoundaryError } from '../browser/SecurityBoundaryError.js';
|
|
6
8
|
import { Verifier } from '../verification/Verifier.js';
|
|
7
9
|
import { ActionCompiler } from './compiler.js';
|
|
8
10
|
export class Executor {
|
|
@@ -34,6 +36,9 @@ export class Executor {
|
|
|
34
36
|
semanticCondition(condition) {
|
|
35
37
|
return 'target' in condition && typeof condition.target === 'string' ? { ...condition, target: this.observer.selectors.descriptor(condition.target) } : condition;
|
|
36
38
|
}
|
|
39
|
+
readTextValue(locator) {
|
|
40
|
+
return locator.evaluate(el => 'value' in el ? String(el.value) : el instanceof HTMLElement && el.isContentEditable ? el.innerText : undefined);
|
|
41
|
+
}
|
|
37
42
|
async storeDownload(download, requestedName) {
|
|
38
43
|
const folder = resolve(this.options.downloadDir ?? '.fcu/downloads');
|
|
39
44
|
await mkdir(folder, { recursive: true, mode: 0o700 });
|
|
@@ -56,7 +61,8 @@ export class Executor {
|
|
|
56
61
|
async run(input) {
|
|
57
62
|
let action = ActionSchema.parse(input);
|
|
58
63
|
const startedAt = Date.now();
|
|
59
|
-
let strategy, executed = false, receiptAction = action;
|
|
64
|
+
let strategy, executed = false, navigationUncertain = false, receiptAction = action;
|
|
65
|
+
let pageBefore;
|
|
60
66
|
let assertApproved, disposeApproved;
|
|
61
67
|
try {
|
|
62
68
|
await this.control.checkpoint();
|
|
@@ -90,6 +96,7 @@ export class Executor {
|
|
|
90
96
|
}
|
|
91
97
|
await this.control.checkpoint();
|
|
92
98
|
const page = this.browser.page;
|
|
99
|
+
pageBefore = page;
|
|
93
100
|
const timeout = action.timeoutMs ?? 4000;
|
|
94
101
|
let data;
|
|
95
102
|
const interaction = this.browser.interaction, interactionOptions = { timeout, checkpoint: () => this.control.checkpoint(), beforeEffect: assertApproved };
|
|
@@ -108,26 +115,34 @@ export class Executor {
|
|
|
108
115
|
case 'navigate':
|
|
109
116
|
if (action.url.includes('{{profile.') || action.url.includes('{{files.'))
|
|
110
117
|
throw new Error('Local vault values cannot be embedded in navigation URLs');
|
|
118
|
+
checkedHttpURL(action.url, page.url());
|
|
119
|
+
navigationUncertain = true;
|
|
111
120
|
await this.browser.navigate(action.url);
|
|
112
121
|
break;
|
|
113
122
|
case 'openTab':
|
|
114
123
|
if (action.url.includes('{{profile.') || action.url.includes('{{files.'))
|
|
115
124
|
throw new Error('Local vault values cannot be embedded in navigation URLs');
|
|
125
|
+
checkedHttpURL(action.url, page.url());
|
|
126
|
+
navigationUncertain = true;
|
|
116
127
|
await this.browser.openTab(action.url);
|
|
117
128
|
break;
|
|
118
129
|
case 'closeTab':
|
|
130
|
+
navigationUncertain = this.browser.context.pages().length > 1;
|
|
119
131
|
await this.browser.closeTab();
|
|
120
132
|
break;
|
|
121
133
|
case 'switchTab':
|
|
122
134
|
await this.browser.switchTab(action.index, timeout);
|
|
123
135
|
break;
|
|
124
136
|
case 'back':
|
|
137
|
+
navigationUncertain = true;
|
|
125
138
|
await page.goBack({ waitUntil: 'domcontentloaded' });
|
|
126
139
|
break;
|
|
127
140
|
case 'forward':
|
|
141
|
+
navigationUncertain = true;
|
|
128
142
|
await page.goForward({ waitUntil: 'domcontentloaded' });
|
|
129
143
|
break;
|
|
130
144
|
case 'reload':
|
|
145
|
+
navigationUncertain = true;
|
|
131
146
|
await page.reload({ waitUntil: 'domcontentloaded' });
|
|
132
147
|
break;
|
|
133
148
|
case 'click': {
|
|
@@ -162,12 +177,17 @@ export class Executor {
|
|
|
162
177
|
break;
|
|
163
178
|
case 'fill':
|
|
164
179
|
await interaction.enter(locator, value, true, interactionOptions);
|
|
165
|
-
if (await
|
|
180
|
+
if (await this.readTextValue(locator) !== value)
|
|
166
181
|
throw new Error('Fill postcondition failed');
|
|
167
182
|
break;
|
|
168
|
-
case 'type':
|
|
169
|
-
await
|
|
183
|
+
case 'type': {
|
|
184
|
+
if (!await locator.isEditable())
|
|
185
|
+
throw new Error('Type target is not editable');
|
|
186
|
+
const { before, after, expected } = await interaction.enter(locator, value, false, interactionOptions);
|
|
187
|
+
if (value && (expected !== undefined ? after !== expected : before !== undefined && (after ?? await this.readTextValue(locator)) === before))
|
|
188
|
+
throw new Error('Type postcondition failed');
|
|
170
189
|
break;
|
|
190
|
+
}
|
|
171
191
|
case 'select': {
|
|
172
192
|
const options = await locator.evaluate(el => [...el.options].map(o => ({ label: o.label, value: o.value })));
|
|
173
193
|
const option = options.find(o => o.value === value) || options.find(o => o.label === value);
|
|
@@ -240,47 +260,138 @@ export class Executor {
|
|
|
240
260
|
interaction.cue('extract');
|
|
241
261
|
const root = locator ?? page.locator('body');
|
|
242
262
|
if (action.format === 'table')
|
|
243
|
-
data = await root.
|
|
263
|
+
data = await root.evaluateAll((els, { match, limit }) => {
|
|
264
|
+
let rows = [...new Set(els.flatMap(el => el.matches('tr') ? [el] : [...el.querySelectorAll('tr')]))].filter(row => { const box = row.getBoundingClientRect(); return box.width > 0 && box.height > 0 && getComputedStyle(row).visibility === 'visible'; });
|
|
265
|
+
if (!match && limit)
|
|
266
|
+
rows = rows.slice(0, limit);
|
|
267
|
+
const values = rows.map(row => [...row.querySelectorAll('th,td')].map(cell => {
|
|
268
|
+
for (let parent = cell; parent; parent = parent.parentElement) {
|
|
269
|
+
const style = getComputedStyle(parent);
|
|
270
|
+
if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0')
|
|
271
|
+
return '';
|
|
272
|
+
}
|
|
273
|
+
return cell.closest('[hidden],[inert],[aria-hidden="true"]') ? '' : cell.innerText?.trim() ?? '';
|
|
274
|
+
}));
|
|
275
|
+
return (match ? values.filter(row => JSON.stringify(row).toLowerCase().includes(match)) : values).slice(0, limit);
|
|
276
|
+
}, { match: action.match?.toLowerCase(), limit: action.limit });
|
|
244
277
|
else if (action.format === 'links')
|
|
245
|
-
data = await root.evaluateAll(els
|
|
278
|
+
data = await root.evaluateAll((els, { match, limit }) => {
|
|
279
|
+
let links = [...new Set(els.flatMap(el => el.matches('a[href]') ? [el] : [...el.querySelectorAll('a[href]')]))].filter(a => {
|
|
280
|
+
if (!a.getClientRects().length || a.closest('[hidden],[inert],[aria-hidden="true"]'))
|
|
281
|
+
return false;
|
|
282
|
+
for (let parent = a; parent; parent = parent.parentElement) {
|
|
283
|
+
const style = getComputedStyle(parent);
|
|
284
|
+
if (style.visibility !== 'visible' || style.opacity === '0')
|
|
285
|
+
return false;
|
|
286
|
+
}
|
|
287
|
+
return true;
|
|
288
|
+
});
|
|
289
|
+
if (!match && limit)
|
|
290
|
+
links = links.slice(0, limit);
|
|
291
|
+
const values = links.map(a => ({ text: a.innerText?.trim(), url: a.href }));
|
|
292
|
+
return (match ? values.filter(link => JSON.stringify(link).toLowerCase().includes(match)) : values).slice(0, limit);
|
|
293
|
+
}, { match: action.match?.toLowerCase(), limit: action.limit });
|
|
246
294
|
else if (action.format === 'records') {
|
|
247
295
|
if (!action.fields || !Object.keys(action.fields).length || Object.keys(action.fields).length > 20)
|
|
248
296
|
throw new Error('Records extraction requires 1..20 controlled CSS fields');
|
|
249
|
-
data = await root.filter({ visible: true }).evaluateAll((els, fields
|
|
250
|
-
let
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
297
|
+
data = await root.filter({ visible: true }).evaluateAll((els, { fields, match, limit }) => {
|
|
298
|
+
let rows = [...new Set(els.flatMap(el => el.matches('table,tbody') ? [...el.querySelectorAll('tr')].filter(row => row.querySelector('td')) : [el]))];
|
|
299
|
+
if (!match && limit)
|
|
300
|
+
rows = rows.slice(0, limit);
|
|
301
|
+
const records = rows.map(el => Object.fromEntries(Object.entries(fields).map(([key, field]) => {
|
|
302
|
+
let node = el.querySelector(field.css);
|
|
303
|
+
let value = '';
|
|
304
|
+
// Resolve tabular fields from actual headers, rather than guessed cell indices.
|
|
305
|
+
if (el.tagName === 'TR' && field.attribute === 'text') {
|
|
306
|
+
const headers = [...(el.closest('table')?.querySelectorAll('thead th,tr:first-child th') ?? [])];
|
|
307
|
+
const [normalize] = [(s) => (s ?? '').toLowerCase().replace(/[^a-z]/g, '')];
|
|
308
|
+
const wanted = normalize(key), indices = headers.map((h, i) => normalize(h.textContent ?? '') === wanted ? i : -1).filter(i => i >= 0);
|
|
309
|
+
const cells = [...el.querySelectorAll(':scope > td')];
|
|
310
|
+
if (indices.length === 1 && cells.length >= headers.length)
|
|
311
|
+
node = cells[indices[0] + cells.length - headers.length] ?? node;
|
|
312
|
+
}
|
|
313
|
+
if (node) {
|
|
314
|
+
let hidden = !!node.closest('[hidden],[inert],[aria-hidden="true"]');
|
|
315
|
+
for (let parent = node; parent; parent = parent.parentElement) {
|
|
316
|
+
const style = getComputedStyle(parent);
|
|
317
|
+
if (style.display === 'none' || style.visibility !== 'visible' || style.opacity === '0')
|
|
318
|
+
hidden = true;
|
|
319
|
+
}
|
|
320
|
+
const autocomplete = (node.getAttribute('autocomplete') ?? '').toLowerCase().split(/\s+/);
|
|
321
|
+
const sensitiveInput = field.attribute === 'value' && node instanceof HTMLInputElement && (node.type === 'hidden' || node.type === 'password' || autocomplete.some(token => token.startsWith('cc-') || ['current-password', 'new-password', 'one-time-code'].includes(token)));
|
|
322
|
+
if (hidden && !sensitiveInput)
|
|
323
|
+
value = '';
|
|
324
|
+
else if (field.attribute === 'text')
|
|
325
|
+
value = node.innerText?.trim() ?? node.textContent?.trim() ?? '';
|
|
326
|
+
else if (field.attribute === 'href')
|
|
327
|
+
value = node.href ?? '';
|
|
328
|
+
else if (field.attribute === 'src')
|
|
329
|
+
value = node.src ?? '';
|
|
330
|
+
else if (field.attribute === 'value')
|
|
331
|
+
value = sensitiveInput ? '[sensitive value omitted]' : node.value ?? '';
|
|
332
|
+
else
|
|
333
|
+
value = node.getAttribute(field.attribute) ?? '';
|
|
334
|
+
}
|
|
335
|
+
return [key, value];
|
|
336
|
+
})));
|
|
337
|
+
return (match ? records.filter(record => JSON.stringify(record).toLowerCase().includes(match)) : records).slice(0, limit);
|
|
338
|
+
}, { fields: action.fields, match: action.match?.toLowerCase(), limit: action.limit });
|
|
275
339
|
}
|
|
276
340
|
else
|
|
277
|
-
data =
|
|
278
|
-
|
|
279
|
-
|
|
341
|
+
data = await root.filter({ visible: true }).evaluateAll(els => els.map(el => {
|
|
342
|
+
const modal = document.querySelector('dialog:modal,[role=dialog][aria-modal=true]'), modalVisible = !!modal && modal.getClientRects().length > 0;
|
|
343
|
+
const [excluded] = [(node) => {
|
|
344
|
+
if (node.matches('[hidden],[inert],[aria-hidden="true"]') || modalVisible && modal && !modal.contains(node) && modal !== node && !node.contains(modal))
|
|
345
|
+
return true;
|
|
346
|
+
const style = getComputedStyle(node);
|
|
347
|
+
return style.display === 'none' || style.visibility !== 'visible' || style.opacity === '0';
|
|
348
|
+
}];
|
|
349
|
+
let hiddenParent = false;
|
|
350
|
+
for (let parent = el.parentElement; parent; parent = parent.parentElement)
|
|
351
|
+
if (excluded(parent)) {
|
|
352
|
+
hiddenParent = true;
|
|
353
|
+
break;
|
|
354
|
+
}
|
|
355
|
+
const omitted = new Set(), dirty = new Set();
|
|
356
|
+
const [inspect] = [(node, hidden = false) => {
|
|
357
|
+
if (hidden || excluded(node)) {
|
|
358
|
+
omitted.add(node);
|
|
359
|
+
for (let parent = node; parent; parent = parent.parentElement) {
|
|
360
|
+
dirty.add(parent);
|
|
361
|
+
if (parent === el)
|
|
362
|
+
break;
|
|
363
|
+
}
|
|
364
|
+
return;
|
|
365
|
+
}
|
|
366
|
+
for (const child of node.children)
|
|
367
|
+
inspect(child);
|
|
368
|
+
}];
|
|
369
|
+
inspect(el, hiddenParent);
|
|
370
|
+
const [read] = [(node) => {
|
|
371
|
+
if (node.nodeType === Node.TEXT_NODE)
|
|
372
|
+
return node.nodeValue ?? '';
|
|
373
|
+
if (node.nodeType !== Node.ELEMENT_NODE)
|
|
374
|
+
return '';
|
|
375
|
+
const element = node;
|
|
376
|
+
if (omitted.has(element))
|
|
377
|
+
return '';
|
|
378
|
+
if (!dirty.has(element))
|
|
379
|
+
return element.innerText ?? element.textContent ?? '';
|
|
380
|
+
if (element.tagName === 'BR')
|
|
381
|
+
return '\n';
|
|
382
|
+
const content = [...element.childNodes].map(read).join('');
|
|
383
|
+
return /^(block|flex|grid|flow-root|list-item|table)/.test(getComputedStyle(element).display) ? `\n${content}\n` : content;
|
|
384
|
+
}];
|
|
385
|
+
return read(el).replace(/[ \t]*\n[ \t]*/g, '\n').replace(/\n+/g, '\n').trim();
|
|
386
|
+
}).join('\n').slice(0, 100000));
|
|
387
|
+
const { match, limit } = action;
|
|
388
|
+
if (typeof data === 'string' && (match || limit)) {
|
|
389
|
+
let lines = data.split(/\r?\n/);
|
|
280
390
|
if (match)
|
|
281
|
-
|
|
282
|
-
if (
|
|
283
|
-
|
|
391
|
+
lines = lines.filter(line => line.toLowerCase().includes(match.toLowerCase()));
|
|
392
|
+
if (limit)
|
|
393
|
+
lines = lines.slice(0, limit);
|
|
394
|
+
data = lines.join('\n');
|
|
284
395
|
}
|
|
285
396
|
this.browser.extractions.push({ key: action.key, value: data });
|
|
286
397
|
data = { [action.key]: data };
|
|
@@ -295,7 +406,8 @@ export class Executor {
|
|
|
295
406
|
return { action: receiptAction, startedAt, durationMs: Date.now() - startedAt, success: true, strategy, data };
|
|
296
407
|
}
|
|
297
408
|
catch (error) {
|
|
298
|
-
|
|
409
|
+
const blockedBeforeNavigation = error instanceof SecurityBoundaryError && ['navigate', 'openTab'].includes(action.type) && this.browser.page === pageBefore;
|
|
410
|
+
return { action: receiptAction, startedAt, durationMs: Date.now() - startedAt, success: false, strategy, error: this.variables.redact(error instanceof Error ? error.message : 'Browser action failed'), uncertain: executed && ((navigationUncertain && !blockedBeforeNavigation) || ['click', 'doubleClick', 'press', 'submit', 'download', 'type'].includes(action.type)) };
|
|
299
411
|
}
|
|
300
412
|
finally {
|
|
301
413
|
await disposeApproved?.();
|