free-computer-use 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/CHANGELOG.md +18 -0
  2. package/README.md +74 -53
  3. package/dist/actions/executor.d.ts +2 -1
  4. package/dist/actions/executor.js +151 -39
  5. package/dist/actions/executor.js.map +1 -1
  6. package/dist/actions/schema.js +4 -1
  7. package/dist/actions/schema.js.map +1 -1
  8. package/dist/adapters/generic.js +10 -0
  9. package/dist/adapters/generic.js.map +1 -1
  10. package/dist/agent/Agent.d.ts +0 -1
  11. package/dist/agent/Agent.js +3 -8
  12. package/dist/agent/Agent.js.map +1 -1
  13. package/dist/agent/goalCriteria.js +6 -0
  14. package/dist/agent/goalCriteria.js.map +1 -1
  15. package/dist/browser/DomExtractor.d.ts +1 -1
  16. package/dist/browser/DomExtractor.js +34 -14
  17. package/dist/browser/DomExtractor.js.map +1 -1
  18. package/dist/browser/Interaction.d.ts +8 -1
  19. package/dist/browser/Interaction.js +36 -2
  20. package/dist/browser/Interaction.js.map +1 -1
  21. package/dist/browser/Observer.js +19 -1
  22. package/dist/browser/Observer.js.map +1 -1
  23. package/dist/browser/PageCompressor.d.ts +9 -21
  24. package/dist/browser/PageCompressor.js +10 -6
  25. package/dist/browser/PageCompressor.js.map +1 -1
  26. package/dist/browser/SelectorEngine.js +6 -3
  27. package/dist/browser/SelectorEngine.js.map +1 -1
  28. package/dist/llm/ClaudeSubscriptionProvider.js +5 -4
  29. package/dist/llm/ClaudeSubscriptionProvider.js.map +1 -1
  30. package/dist/llm/CodexSubscriptionProvider.js +5 -4
  31. package/dist/llm/CodexSubscriptionProvider.js.map +1 -1
  32. package/dist/llm/cliEnvironment.d.ts +2 -0
  33. package/dist/llm/cliEnvironment.js +12 -0
  34. package/dist/llm/cliEnvironment.js.map +1 -1
  35. package/dist/llm/prompts.d.ts +1 -1
  36. package/dist/llm/prompts.js +1 -1
  37. package/dist/server/index.js +9 -1
  38. package/dist/server/index.js.map +1 -1
  39. package/dist/verification/Verifier.js +7 -5
  40. package/dist/verification/Verifier.js.map +1 -1
  41. package/docs/ADOPTION.md +14 -2
  42. package/docs/LOCAL_VALIDATION.md +272 -6
  43. package/docs/RELEASE_CHECKLIST.md +23 -22
  44. package/docs/THREAT_MODEL.md +2 -2
  45. package/package.json +1 -1
package/CHANGELOG.md CHANGED
@@ -1,5 +1,23 @@
1
1
  # Changelog
2
2
 
3
+ ## 0.2.0 - 2026-09-26
4
+
5
+ ### Added
6
+
7
+ - Deterministic, provider-free extraction of visible page text and links.
8
+ - Bounded extraction of the first 1–1000 visible links, with exact-count
9
+ verification.
10
+
11
+ ### Changed
12
+
13
+ - Zero-timeout selector checks return immediately instead of waiting through a
14
+ polling interval.
15
+
16
+ ### Fixed
17
+
18
+ - Element verification evaluates all matching elements, avoiding a false
19
+ failure when a later match satisfies the condition.
20
+
3
21
  ## 0.1.0
4
22
 
5
23
  ### Added
package/README.md CHANGED
@@ -3,41 +3,84 @@
3
3
  ![FreeComputerUse — a local-first browser agent that plans, acts and verifies with you in control](assets/readme/hero.svg)
4
4
 
5
5
  <p align="center">
6
+ <a href="https://www.npmjs.com/package/free-computer-use"><img alt="npm version" src="https://img.shields.io/npm/v/free-computer-use?style=flat-square&labelColor=18251f&color=c4e967"></a>
7
+ <a href="https://github.com/OthmaneBlial/FreeComputerUse/releases/latest"><img alt="Latest GitHub release" src="https://img.shields.io/github/v/release/OthmaneBlial/FreeComputerUse?style=flat-square&labelColor=18251f&color=c4e967"></a>
6
8
  <a href="LICENSE"><img alt="MIT license" src="https://img.shields.io/badge/license-MIT-c4e967?style=flat-square&labelColor=18251f"></a>
9
+ <img alt="Node.js 22.13 or newer" src="https://img.shields.io/badge/Node.js-22.13%2B-43853d?style=flat-square&labelColor=18251f">
7
10
  <img alt="TypeScript" src="https://img.shields.io/badge/TypeScript-3178c6?style=flat-square&labelColor=18251f">
8
11
  <img alt="Playwright" src="https://img.shields.io/badge/Playwright-45ba4b?style=flat-square&labelColor=18251f">
9
- <img alt="Local-first" src="https://img.shields.io/badge/local--first-browser_runs-c4e967?style=flat-square&labelColor=18251f">
10
12
  </p>
11
13
 
12
14
  <p align="center">
13
15
  <a href="#quick-start">Quick start</a> ·
16
+ <a href="#no-model-key-needed">Try it without a model</a> ·
17
+ <a href="#watch-a-real-run">Watch the demo</a> ·
18
+ <a href="https://github.com/OthmaneBlial/FreeComputerUse/releases/latest">Latest release</a> ·
14
19
  <a href="https://othmaneblial.github.io/FreeComputerUse/">Project site</a> ·
15
- <a href="https://othmaneblial.github.io/FreeComputerUse/lab/index.html">Try the task lab</a> ·
16
20
  <a href="SECURITY.md">Security</a>
17
21
  </p>
18
22
 
19
- **Your browser. Your model. Your call.** Tell it what done looks like, then stay in control of every step.
23
+ **Your browser. Your model. Your call.** Give your browser a goal; the model proposes short action batches, and Playwright runs them locally. You approve website access and sensitive actions, then inspect what happened.
20
24
 
21
- FreeComputerUse is a local-first AI browser automation agent. A model plans small action batches; TypeScript and Playwright execute and verify them in your browser. Compatible workflows can be learned once and replayed later with **zero model calls**.
25
+ Use an API or a supported model CLI for open-ended tasks. For common page-reading jobs, skip model setup entirely: FreeComputerUse can read visible text and links with deterministic local strategies.
26
+
27
+ ## No model key needed
28
+
29
+ Install the release and start the local dashboard:
30
+
31
+ ```bash
32
+ npm install --global free-computer-use@0.2.0
33
+ FCU_BROWSER_CHANNEL=chrome agent doctor
34
+ FCU_BROWSER_CHANNEL=chrome agent ui
35
+ ```
36
+
37
+ Open **http://127.0.0.1:4318**, enter a page URL and one of these goals, then approve access to that website:
38
+
39
+ | Goal | Result |
40
+ | --- | --- |
41
+ | `Read the page` | Extract visible page text. |
42
+ | `Extract the links` | List visible link labels and URLs. |
43
+ | `Extract the first 5 links` | Return a bounded list of the first five links. |
44
+ | `Extract the table` | Try the synthetic [practice revenue table](https://othmaneblial.github.io/FreeComputerUse/lab/reports.html). |
45
+
46
+ These narrow workflows use local strategies and make zero model calls. General tasks need a configured provider. The declared minimum is Node.js 22.13; the verified setup is macOS 26.6 on Apple Silicon, Node 25.9.0 and system Chrome 154.0.8037.57. Other OS/browser combinations and the declared Node minimum remain unverified. See the [support matrix](docs/SUPPORT_MATRIX.md).
22
47
 
23
48
  ## Watch a real run
24
49
 
25
- [![A real browser run: investigate a synthetic API incident and save a checked report](assets/readme/incident-demo.gif)](https://othmaneblial.github.io/FreeComputerUse/#watch)
50
+ [![FreeComputerUse: a real practice-table extraction from the published package](assets/readme/product-demo-poster.jpg)](assets/readme/product-demo.mp4)
51
+
52
+ [Watch the 22-second dashboard walkthrough](assets/readme/product-demo.mp4) · [Watch the short portrait cut](assets/readme/product-demo-portrait.mp4)
26
53
 
27
- One goal led through six pages to a verified incident brief: **21 successful actions, 10 model calls, one repaired failure**. The configured cost estimate was **$0.00650**. This is one recorded synthetic task, not a general success-rate claim. [Watch the full recording](https://othmaneblial.github.io/FreeComputerUse/lab/media/incident-demo.mp4) · [Inspect the evidence](assets/readme/incident-evidence.json).
54
+ This recording uses the 0.1.0 package. It shows the npm install, local dashboard, explicit site approval and a verified synthetic table result: three rows, one browser action and zero model calls. The same practice workflow remains available. It is a deterministic demo, not a claim about arbitrary websites.
55
+
56
+ [![Recorded browser run: investigate a synthetic API incident and save a checked report](assets/readme/incident-demo.gif)](https://othmaneblial.github.io/FreeComputerUse/#watch)
57
+
58
+ One recorded task followed six pages to a checked incident brief: **21 successful actions, 10 model calls, one repaired failure**. The configured cost estimate was **$0.00650**. This is one synthetic run, not a general success-rate claim. [Watch the full recording](https://othmaneblial.github.io/FreeComputerUse/lab/media/incident-demo.mp4) · [Inspect the evidence](assets/readme/incident-evidence.json).
28
59
 
29
60
  ## How it works
30
61
 
31
- 1. **Observe** the page’s DOM and accessible controls.
62
+ 1. **Observe** the page DOM and accessible controls.
32
63
  2. **Plan** a bounded batch with the model you choose.
33
64
  3. **Execute** browser actions locally with Playwright; direct navigation to a new origin asks for approval.
34
- 4. **Verify** results, repair only what failed, then reuse compatible learned workflows.
65
+ 4. **Verify** results, repair only what failed, then replay compatible learned workflows later.
35
66
 
36
- The model does not run shell commands or arbitrary JavaScript. Browser previews stay local; the goal and selected page context are sent to your configured model provider.
67
+ The model does not run shell commands or arbitrary JavaScript. Browser previews stay local; the task and selected page context go to your configured model provider. Screenshots are not sent.
37
68
 
38
69
  ## Quick start
39
70
 
40
- Requires **Node.js 22.13+**, npm and an installed browser. The verified setup is macOS 26.6 (Apple Silicon), Node 25.9.0 and system Chrome 154.0.8037.57; the complete serial validation passed on this combination. The declared Node minimum and other OS/browser combinations remain unverified. This setup selects installed Chrome and avoids a separate Playwright browser download; see the [support matrix](docs/SUPPORT_MATRIX.md) for exact coverage and limits.
71
+ Requires Node.js 22.13+, npm and an installed browser. The verified setup selects installed Chrome and avoids a separate Playwright browser download.
72
+
73
+ ### Install from npm
74
+
75
+ ```bash
76
+ npm install --global free-computer-use@0.2.0
77
+ FCU_BROWSER_CHANNEL=chrome agent doctor
78
+ FCU_BROWSER_CHANNEL=chrome agent ui
79
+ ```
80
+
81
+ Open **http://127.0.0.1:4318**. Try a keyless goal above, or configure a provider for model-planned tasks.
82
+
83
+ ### Run from source
41
84
 
42
85
  ```bash
43
86
  git clone https://github.com/OthmaneBlial/FreeComputerUse.git
@@ -47,9 +90,7 @@ cp .env.example .env
47
90
  chmod 600 .env
48
91
  ```
49
92
 
50
- Set `FCU_BROWSER_CHANNEL=chrome` in `.env` to use the verified installed-Chrome path without downloading a browser binary.
51
-
52
- For model-planned tasks, configure a provider in `.env`. The browser sandbox task below works without a model key:
93
+ Set `FCU_BROWSER_CHANNEL=chrome` in `.env` to use installed Chrome. For model-planned tasks, add the provider details to your local `.env`; for example:
53
94
 
54
95
  ```dotenv
55
96
  FCU_BROWSER_CHANNEL=chrome
@@ -64,21 +105,11 @@ npm run agent -- doctor
64
105
  npm run dev
65
106
  ```
66
107
 
67
- Open **http://127.0.0.1:4318**, enter a starting URL and goal, then approve site access.
68
-
69
- **Try it without an API key:** open [the practice revenue table](https://othmaneblial.github.io/FreeComputerUse/lab/reports.html) and run the goal `Extract the table`. This narrow workflow has a deterministic local strategy; general tasks need a model provider.
70
-
71
- ## Troubleshooting
72
-
73
- - **The browser does not start:** install Chrome, set `FCU_BROWSER_CHANNEL=chrome` in `.env`, then run `npm run agent -- doctor`. See the [verified platform limits](docs/SUPPORT_MATRIX.md); other OS/browser combinations are not certified here.
74
- - **The dashboard says no model is configured:** the practice revenue-table task works without a provider. For model planning, configure one route from [Pick your model](#pick-your-model). `npm run agent -- doctor --api` makes an opt-in request to the configured provider; use it only when you want that network check.
75
- - **A site is blocked or asks for approval:** normal mode asks before a new origin and blocks private/reserved DNS answers. Approve only the site needed for the task. Ultra mode disables those protections; do not use it as a workaround for a blocked destination.
76
- - **The run finishes without proving the goal:** inspect the result checks and use a narrower goal. Browser clicks alone do not mean the requested outcome was verified.
77
- - **You need to find or remove local data:** see [local storage, export, retention and deletion](docs/LOCAL_DATA.md). Data is not encrypted and is not automatically expired.
108
+ Then open **http://127.0.0.1:4318**, enter a starting URL and goal, and approve site access.
78
109
 
79
110
  ## Pick your model
80
111
 
81
- Use a provider API key, or sign in through the official CLI for a supported subscription. API usage and consumer subscriptions are separate billing products.
112
+ Use a provider API key or sign in through the official CLI for a supported subscription. API usage and consumer subscriptions are separate billing products.
82
113
 
83
114
  | Provider | Configuration |
84
115
  | --- | --- |
@@ -89,51 +120,41 @@ Use a provider API key, or sign in through the official CLI for a supported subs
89
120
  | ChatGPT plan | Install Codex CLI, sign in with `codex login`, set `LLM_PROVIDER=codex-subscription` |
90
121
  | Claude Pro/Max plan | Install Claude Code 2.1.248+, sign in with `claude auth login` (not Console), set `LLM_PROVIDER=claude-subscription` |
91
122
 
92
- The provider API routes have not all been live-tested here; DeepSeek Flash is the measured default. Subscription modes use the local CLI sign-in, need no API key and respect plan limits. Their local agent tools and MCP servers are disabled while planning. See [provider setup and limits](docs/PROVIDERS.md), the [support matrix](docs/SUPPORT_MATRIX.md) and [.env.example](.env.example) for route-specific setup, tests and live-evidence status.
123
+ DeepSeek Flash is the measured default. Other provider routes are not all live-tested. Subscription modes use local CLI sign-in, need no API key, respect plan limits, and disable their local agent tools and MCP servers while planning. See [provider setup and limits](docs/PROVIDERS.md), the [support matrix](docs/SUPPORT_MATRIX.md) and [.env.example](.env.example).
93
124
 
94
125
  ## Connect an MCP host
95
126
 
96
- Run FreeComputerUse as a local MCP server over `stdio` with `agent mcp`. It
97
- provides tools to start a task, inspect the observed page, follow and verify the
98
- result, and stop a task. Site and sensitive-action approvals still require a
99
- human response in an MCP host that supports form elicitation. No HTTP endpoint
100
- is opened. See the [MCP setup and validation limits](docs/MCP.md). The separate
101
- provider CLI adapters keep their own tools and MCP disabled while planning.
127
+ Run FreeComputerUse as a local MCP server over `stdio` with `agent mcp`. It exposes bounded tools to start a task, inspect the page, follow and verify results, and stop a task. Site and sensitive-action approvals still need a human response in an MCP host that supports form elicitation. No HTTP endpoint is opened. See the [MCP setup and validation limits](docs/MCP.md).
102
128
 
103
- ## FAQ
104
-
105
- **Can I try it without a model key?** Yes, the practice revenue-table workflow runs with a deterministic local strategy. General tasks need a configured model.
129
+ ## Security and privacy
106
130
 
107
- **Which providers have live evidence?** The current matrix records DeepSeek Flash and one synthetic plan through Codex CLI `0.156.1` with ChatGPT. Contract tests do not certify a live provider; see the [dated support matrix](docs/SUPPORT_MATRIX.md).
131
+ - Normal mode asks before direct navigation to a new origin and blocks private or reserved DNS answers. A loopback-only proxy connects to the vetted numeric address for browser HTTP(S) and WebSocket traffic. Ultra mode and the low-level `allowExternal` option opt out of origin and private-address checks; Ultra mode is off by default.
132
+ - Sensitive actions have a separate confirmation gate. Read the [security boundaries](SECURITY.md) before using personal or sensitive data.
133
+ - Browser execution, profiles, history and downloads stay on your machine. Selected task and page context goes to the provider you choose; screenshots are not sent.
134
+ - Planner inputs use aliases for local profile and file values, not their contents. Local data is **not encrypted** and does not expire automatically. See [storage, export, retention and deletion](docs/LOCAL_DATA.md).
135
+ - Chromium profiles disable WebRTC UDP that the proxy cannot carry. Sites that require direct UDP for voice or video may fail.
108
136
 
109
- **Does the model receive screenshots or browser profiles?** No screenshots are sent. The task and selected page context go to your provider; browser execution, profiles, history and downloads stay local. Local data is not encrypted.
137
+ ## Evidence and limits
110
138
 
111
- **Which platform is verified?** macOS 26.6 on Apple Silicon, Node 25.9.0 and system Chrome 154. Other OS, Node and browser combinations remain unverified in the [support matrix](docs/SUPPORT_MATRIX.md).
139
+ A focused public suite recorded **18 September 2026** passed 14/14 first-run tasks and 14/14 compatible learned repeats with independent checks. First runs used 18 model calls; repeats used zero. These single-trial sandbox results do not predict success on arbitrary websites. [Report](artifacts/benchmark-public.json) · [Method and limitations](docs/BENCHMARKS.md).
112
140
 
113
- ## Control and privacy
141
+ Provider, operating-system, browser and accessibility evidence stays bounded to the [support matrix](docs/SUPPORT_MATRIX.md). Contract tests do not certify a live provider; builds do not certify an untested platform.
114
142
 
115
- - Normal mode asks before direct navigation to a new origin and blocks hostnames resolving to private or reserved address ranges, including private IPv4 embedded in a discovered NAT64 prefix. A loopback-only proxy connects to the vetted numeric address, closing the DNS lookup-to-connection rebinding gap for browser HTTP(S) and WebSocket traffic. Explicit IP destinations require an exact origin grant. Ultra mode and the low-level `allowExternal` option opt out of origin and private-address checks. Chrome 154 tests cover redirect denial, simulated DNS rebinding, approved/blocked plain WebSockets, and approved/blocked browser-originated WSS. The WSS fixture uses a generated local certificate and ignores its certificate error only in the test; a separate opt-in smoke verifies Chrome TLS and an echo against one public WSS endpoint. Other WSS endpoints, live network-specific NAT64 discovery and other browser builds remain unverified. Sensitive actions have a separate confirmation gate. Ultra mode is explicit and off by default.
116
- - Browser execution, profiles, history and downloads stay on your machine. Page context needed for a plan goes to the chosen model provider; screenshots are not sent.
117
- - The planner receives aliases for local profile and file values, not their contents. Local storage is **not encrypted**.
118
- - Chromium profiles disable WebRTC UDP that the proxy cannot carry. Sites needing direct UDP for voice/video may fail; one local Chrome STUN fixture confirms no direct packet reached its receiver. Other non-HTTP traffic remains unverified.
119
- - Runs record actions, checks, repairs, token estimates and workflow reuse so you can inspect what happened.
143
+ ## FAQ
120
144
 
121
- Read [the security boundaries](SECURITY.md) before using personal or sensitive data.
122
- See [where local data is stored and how to inspect, export or delete it](docs/LOCAL_DATA.md).
145
+ **Can I try it without an API key?** Yes. Use `Read the page`, `Extract the links`, `Extract the first 5 links`, or the synthetic revenue-table task. These workflows make zero model calls.
123
146
 
124
- ## Evidence
147
+ **Does the model receive screenshots or browser profiles?** No. The selected task and page context go to your provider; screenshots and browser profiles stay local. Local data is not encrypted.
125
148
 
126
- In a focused public suite recorded **18 September 2026**, 14/14 first-run results and 14/14 compatible learned repeats passed independent checks. First runs used 18 model calls; repeats used zero. These single-trial sandbox results do not predict success on arbitrary websites. [Report](artifacts/benchmark-public.json) · [Method and limitations](docs/BENCHMARKS.md).
149
+ **Which platform is verified?** macOS 26.6 on Apple Silicon, Node 25.9.0 and system Chrome 154.0.8037.57. Check the [support matrix](docs/SUPPORT_MATRIX.md) for exact coverage and limits.
127
150
 
128
151
  ## Explore and contribute
129
152
 
130
153
  - [Project site and task library](https://othmaneblial.github.io/FreeComputerUse/)
131
- - [Frequently asked questions](https://othmaneblial.github.io/FreeComputerUse/#faq)
154
+ - [Latest GitHub release](https://github.com/OthmaneBlial/FreeComputerUse/releases/latest)
132
155
  - [Useful browser-task examples](docs/USEFUL_EXAMPLES.md)
133
- - [Implementation notes](docs/IMPLEMENTATION.md)
134
- - [Architecture](docs/ARCHITECTURE.md)
135
- - [Local validation commands](docs/LOCAL_VALIDATION.md)
136
- - [Contributing](CONTRIBUTING.md) · [Changelog](CHANGELOG.md)
156
+ - [Architecture](docs/ARCHITECTURE.md) · [implementation notes](docs/IMPLEMENTATION.md)
157
+ - [Local validation commands](docs/LOCAL_VALIDATION.md) · [contributing](CONTRIBUTING.md) · [changelog](CHANGELOG.md)
137
158
 
138
159
  Useful contributions: reproducible browser tasks, safer permission scopes and checks that make results easier to trust. Keep shared examples synthetic or read-only; remove credentials and personal data from traces.
139
160
 
@@ -1,6 +1,6 @@
1
1
  import { type Action, type Condition } from './schema.js';
2
2
  import { type ConfirmationPolicy } from './policy.js';
3
- import type { Browser } from '../browser/Browser.js';
3
+ import { type Browser } from '../browser/Browser.js';
4
4
  import type { Observer } from '../browser/Observer.js';
5
5
  import type { VariableResolver } from '../profile/VariableResolver.js';
6
6
  import type { Control } from '../agent/Control.js';
@@ -33,6 +33,7 @@ export declare class Executor {
33
33
  });
34
34
  semantic(action: Action): Action;
35
35
  semanticCondition(condition: Condition): Condition;
36
+ private readTextValue;
36
37
  private storeDownload;
37
38
  run(input: Action): Promise<ActionResult>;
38
39
  }
@@ -3,6 +3,8 @@ import { randomUUID } from 'node:crypto';
3
3
  import { basename, resolve } from 'node:path';
4
4
  import { ActionSchema } from './schema.js';
5
5
  import { sensitiveReason } from './policy.js';
6
+ import { checkedHttpURL } from '../browser/Browser.js';
7
+ import { SecurityBoundaryError } from '../browser/SecurityBoundaryError.js';
6
8
  import { Verifier } from '../verification/Verifier.js';
7
9
  import { ActionCompiler } from './compiler.js';
8
10
  export class Executor {
@@ -34,6 +36,9 @@ export class Executor {
34
36
  semanticCondition(condition) {
35
37
  return 'target' in condition && typeof condition.target === 'string' ? { ...condition, target: this.observer.selectors.descriptor(condition.target) } : condition;
36
38
  }
39
+ readTextValue(locator) {
40
+ return locator.evaluate(el => 'value' in el ? String(el.value) : el instanceof HTMLElement && el.isContentEditable ? el.innerText : undefined);
41
+ }
37
42
  async storeDownload(download, requestedName) {
38
43
  const folder = resolve(this.options.downloadDir ?? '.fcu/downloads');
39
44
  await mkdir(folder, { recursive: true, mode: 0o700 });
@@ -56,7 +61,8 @@ export class Executor {
56
61
  async run(input) {
57
62
  let action = ActionSchema.parse(input);
58
63
  const startedAt = Date.now();
59
- let strategy, executed = false, receiptAction = action;
64
+ let strategy, executed = false, navigationUncertain = false, receiptAction = action;
65
+ let pageBefore;
60
66
  let assertApproved, disposeApproved;
61
67
  try {
62
68
  await this.control.checkpoint();
@@ -90,6 +96,7 @@ export class Executor {
90
96
  }
91
97
  await this.control.checkpoint();
92
98
  const page = this.browser.page;
99
+ pageBefore = page;
93
100
  const timeout = action.timeoutMs ?? 4000;
94
101
  let data;
95
102
  const interaction = this.browser.interaction, interactionOptions = { timeout, checkpoint: () => this.control.checkpoint(), beforeEffect: assertApproved };
@@ -108,26 +115,34 @@ export class Executor {
108
115
  case 'navigate':
109
116
  if (action.url.includes('{{profile.') || action.url.includes('{{files.'))
110
117
  throw new Error('Local vault values cannot be embedded in navigation URLs');
118
+ checkedHttpURL(action.url, page.url());
119
+ navigationUncertain = true;
111
120
  await this.browser.navigate(action.url);
112
121
  break;
113
122
  case 'openTab':
114
123
  if (action.url.includes('{{profile.') || action.url.includes('{{files.'))
115
124
  throw new Error('Local vault values cannot be embedded in navigation URLs');
125
+ checkedHttpURL(action.url, page.url());
126
+ navigationUncertain = true;
116
127
  await this.browser.openTab(action.url);
117
128
  break;
118
129
  case 'closeTab':
130
+ navigationUncertain = this.browser.context.pages().length > 1;
119
131
  await this.browser.closeTab();
120
132
  break;
121
133
  case 'switchTab':
122
134
  await this.browser.switchTab(action.index, timeout);
123
135
  break;
124
136
  case 'back':
137
+ navigationUncertain = true;
125
138
  await page.goBack({ waitUntil: 'domcontentloaded' });
126
139
  break;
127
140
  case 'forward':
141
+ navigationUncertain = true;
128
142
  await page.goForward({ waitUntil: 'domcontentloaded' });
129
143
  break;
130
144
  case 'reload':
145
+ navigationUncertain = true;
131
146
  await page.reload({ waitUntil: 'domcontentloaded' });
132
147
  break;
133
148
  case 'click': {
@@ -162,12 +177,17 @@ export class Executor {
162
177
  break;
163
178
  case 'fill':
164
179
  await interaction.enter(locator, value, true, interactionOptions);
165
- if (await locator.inputValue() !== value)
180
+ if (await this.readTextValue(locator) !== value)
166
181
  throw new Error('Fill postcondition failed');
167
182
  break;
168
- case 'type':
169
- await interaction.enter(locator, value, false, interactionOptions);
183
+ case 'type': {
184
+ if (!await locator.isEditable())
185
+ throw new Error('Type target is not editable');
186
+ const { before, after, expected } = await interaction.enter(locator, value, false, interactionOptions);
187
+ if (value && (expected !== undefined ? after !== expected : before !== undefined && (after ?? await this.readTextValue(locator)) === before))
188
+ throw new Error('Type postcondition failed');
170
189
  break;
190
+ }
171
191
  case 'select': {
172
192
  const options = await locator.evaluate(el => [...el.options].map(o => ({ label: o.label, value: o.value })));
173
193
  const option = options.find(o => o.value === value) || options.find(o => o.label === value);
@@ -240,47 +260,138 @@ export class Executor {
240
260
  interaction.cue('extract');
241
261
  const root = locator ?? page.locator('body');
242
262
  if (action.format === 'table')
243
- data = await root.locator('tr').filter({ visible: true }).evaluateAll(rows => rows.map(row => [...row.querySelectorAll('th,td')].map(cell => cell.textContent?.trim() ?? '')));
263
+ data = await root.evaluateAll((els, { match, limit }) => {
264
+ let rows = [...new Set(els.flatMap(el => el.matches('tr') ? [el] : [...el.querySelectorAll('tr')]))].filter(row => { const box = row.getBoundingClientRect(); return box.width > 0 && box.height > 0 && getComputedStyle(row).visibility === 'visible'; });
265
+ if (!match && limit)
266
+ rows = rows.slice(0, limit);
267
+ const values = rows.map(row => [...row.querySelectorAll('th,td')].map(cell => {
268
+ for (let parent = cell; parent; parent = parent.parentElement) {
269
+ const style = getComputedStyle(parent);
270
+ if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0')
271
+ return '';
272
+ }
273
+ return cell.closest('[hidden],[inert],[aria-hidden="true"]') ? '' : cell.innerText?.trim() ?? '';
274
+ }));
275
+ return (match ? values.filter(row => JSON.stringify(row).toLowerCase().includes(match)) : values).slice(0, limit);
276
+ }, { match: action.match?.toLowerCase(), limit: action.limit });
244
277
  else if (action.format === 'links')
245
- data = await root.evaluateAll(els => els.flatMap(el => [...(el.matches('a[href]') ? [el] : el.querySelectorAll('a[href]'))].filter(a => a.getClientRects().length > 0).map(a => ({ text: a.textContent?.trim(), url: a.href }))));
278
+ data = await root.evaluateAll((els, { match, limit }) => {
279
+ let links = [...new Set(els.flatMap(el => el.matches('a[href]') ? [el] : [...el.querySelectorAll('a[href]')]))].filter(a => {
280
+ if (!a.getClientRects().length || a.closest('[hidden],[inert],[aria-hidden="true"]'))
281
+ return false;
282
+ for (let parent = a; parent; parent = parent.parentElement) {
283
+ const style = getComputedStyle(parent);
284
+ if (style.visibility !== 'visible' || style.opacity === '0')
285
+ return false;
286
+ }
287
+ return true;
288
+ });
289
+ if (!match && limit)
290
+ links = links.slice(0, limit);
291
+ const values = links.map(a => ({ text: a.innerText?.trim(), url: a.href }));
292
+ return (match ? values.filter(link => JSON.stringify(link).toLowerCase().includes(match)) : values).slice(0, limit);
293
+ }, { match: action.match?.toLowerCase(), limit: action.limit });
246
294
  else if (action.format === 'records') {
247
295
  if (!action.fields || !Object.keys(action.fields).length || Object.keys(action.fields).length > 20)
248
296
  throw new Error('Records extraction requires 1..20 controlled CSS fields');
249
- data = await root.filter({ visible: true }).evaluateAll((els, fields) => els.flatMap(el => el.matches('table,tbody') ? [...el.querySelectorAll('tr')].filter(row => row.querySelector('td')) : [el]).map(el => Object.fromEntries(Object.entries(fields).map(([key, field]) => {
250
- let node = el.querySelector(field.css);
251
- let value = '';
252
- // Resolve tabular fields from actual headers, rather than guessed cell indices.
253
- if (el.tagName === 'TR' && field.attribute === 'text') {
254
- const headers = [...(el.closest('table')?.querySelectorAll('thead th,tr:first-child th') ?? [])];
255
- const [normalize] = [(s) => (s ?? '').toLowerCase().replace(/[^a-z]/g, '')];
256
- const wanted = normalize(key), indices = headers.map((h, i) => normalize(h.textContent ?? '') === wanted ? i : -1).filter(i => i >= 0);
257
- const cells = [...el.querySelectorAll(':scope > td')];
258
- if (indices.length === 1 && cells.length >= headers.length)
259
- node = cells[indices[0] + cells.length - headers.length] ?? node;
260
- }
261
- if (node) {
262
- if (field.attribute === 'text')
263
- value = node.innerText?.trim() ?? node.textContent?.trim() ?? '';
264
- else if (field.attribute === 'href')
265
- value = node.href ?? '';
266
- else if (field.attribute === 'src')
267
- value = node.src ?? '';
268
- else if (field.attribute === 'value')
269
- value = node.matches('input[type=password],input[autocomplete=cc-number],input[autocomplete=cc-csc]') ? '[sensitive value omitted]' : node.value ?? '';
270
- else
271
- value = node.getAttribute(field.attribute) ?? '';
272
- }
273
- return [key, value];
274
- }))), action.fields);
297
+ data = await root.filter({ visible: true }).evaluateAll((els, { fields, match, limit }) => {
298
+ let rows = [...new Set(els.flatMap(el => el.matches('table,tbody') ? [...el.querySelectorAll('tr')].filter(row => row.querySelector('td')) : [el]))];
299
+ if (!match && limit)
300
+ rows = rows.slice(0, limit);
301
+ const records = rows.map(el => Object.fromEntries(Object.entries(fields).map(([key, field]) => {
302
+ let node = el.querySelector(field.css);
303
+ let value = '';
304
+ // Resolve tabular fields from actual headers, rather than guessed cell indices.
305
+ if (el.tagName === 'TR' && field.attribute === 'text') {
306
+ const headers = [...(el.closest('table')?.querySelectorAll('thead th,tr:first-child th') ?? [])];
307
+ const [normalize] = [(s) => (s ?? '').toLowerCase().replace(/[^a-z]/g, '')];
308
+ const wanted = normalize(key), indices = headers.map((h, i) => normalize(h.textContent ?? '') === wanted ? i : -1).filter(i => i >= 0);
309
+ const cells = [...el.querySelectorAll(':scope > td')];
310
+ if (indices.length === 1 && cells.length >= headers.length)
311
+ node = cells[indices[0] + cells.length - headers.length] ?? node;
312
+ }
313
+ if (node) {
314
+ let hidden = !!node.closest('[hidden],[inert],[aria-hidden="true"]');
315
+ for (let parent = node; parent; parent = parent.parentElement) {
316
+ const style = getComputedStyle(parent);
317
+ if (style.display === 'none' || style.visibility !== 'visible' || style.opacity === '0')
318
+ hidden = true;
319
+ }
320
+ const autocomplete = (node.getAttribute('autocomplete') ?? '').toLowerCase().split(/\s+/);
321
+ const sensitiveInput = field.attribute === 'value' && node instanceof HTMLInputElement && (node.type === 'hidden' || node.type === 'password' || autocomplete.some(token => token.startsWith('cc-') || ['current-password', 'new-password', 'one-time-code'].includes(token)));
322
+ if (hidden && !sensitiveInput)
323
+ value = '';
324
+ else if (field.attribute === 'text')
325
+ value = node.innerText?.trim() ?? node.textContent?.trim() ?? '';
326
+ else if (field.attribute === 'href')
327
+ value = node.href ?? '';
328
+ else if (field.attribute === 'src')
329
+ value = node.src ?? '';
330
+ else if (field.attribute === 'value')
331
+ value = sensitiveInput ? '[sensitive value omitted]' : node.value ?? '';
332
+ else
333
+ value = node.getAttribute(field.attribute) ?? '';
334
+ }
335
+ return [key, value];
336
+ })));
337
+ return (match ? records.filter(record => JSON.stringify(record).toLowerCase().includes(match)) : records).slice(0, limit);
338
+ }, { fields: action.fields, match: action.match?.toLowerCase(), limit: action.limit });
275
339
  }
276
340
  else
277
- data = (await root.filter({ visible: true }).allInnerTexts()).join('\n').slice(0, 100000);
278
- if (Array.isArray(data)) {
279
- const match = action.match;
341
+ data = await root.filter({ visible: true }).evaluateAll(els => els.map(el => {
342
+ const modal = document.querySelector('dialog:modal,[role=dialog][aria-modal=true]'), modalVisible = !!modal && modal.getClientRects().length > 0;
343
+ const [excluded] = [(node) => {
344
+ if (node.matches('[hidden],[inert],[aria-hidden="true"]') || modalVisible && modal && !modal.contains(node) && modal !== node && !node.contains(modal))
345
+ return true;
346
+ const style = getComputedStyle(node);
347
+ return style.display === 'none' || style.visibility !== 'visible' || style.opacity === '0';
348
+ }];
349
+ let hiddenParent = false;
350
+ for (let parent = el.parentElement; parent; parent = parent.parentElement)
351
+ if (excluded(parent)) {
352
+ hiddenParent = true;
353
+ break;
354
+ }
355
+ const omitted = new Set(), dirty = new Set();
356
+ const [inspect] = [(node, hidden = false) => {
357
+ if (hidden || excluded(node)) {
358
+ omitted.add(node);
359
+ for (let parent = node; parent; parent = parent.parentElement) {
360
+ dirty.add(parent);
361
+ if (parent === el)
362
+ break;
363
+ }
364
+ return;
365
+ }
366
+ for (const child of node.children)
367
+ inspect(child);
368
+ }];
369
+ inspect(el, hiddenParent);
370
+ const [read] = [(node) => {
371
+ if (node.nodeType === Node.TEXT_NODE)
372
+ return node.nodeValue ?? '';
373
+ if (node.nodeType !== Node.ELEMENT_NODE)
374
+ return '';
375
+ const element = node;
376
+ if (omitted.has(element))
377
+ return '';
378
+ if (!dirty.has(element))
379
+ return element.innerText ?? element.textContent ?? '';
380
+ if (element.tagName === 'BR')
381
+ return '\n';
382
+ const content = [...element.childNodes].map(read).join('');
383
+ return /^(block|flex|grid|flow-root|list-item|table)/.test(getComputedStyle(element).display) ? `\n${content}\n` : content;
384
+ }];
385
+ return read(el).replace(/[ \t]*\n[ \t]*/g, '\n').replace(/\n+/g, '\n').trim();
386
+ }).join('\n').slice(0, 100000));
387
+ const { match, limit } = action;
388
+ if (typeof data === 'string' && (match || limit)) {
389
+ let lines = data.split(/\r?\n/);
280
390
  if (match)
281
- data = data.filter(item => JSON.stringify(item).toLowerCase().includes(match.toLowerCase()));
282
- if (action.limit)
283
- data = data.slice(0, action.limit);
391
+ lines = lines.filter(line => line.toLowerCase().includes(match.toLowerCase()));
392
+ if (limit)
393
+ lines = lines.slice(0, limit);
394
+ data = lines.join('\n');
284
395
  }
285
396
  this.browser.extractions.push({ key: action.key, value: data });
286
397
  data = { [action.key]: data };
@@ -295,7 +406,8 @@ export class Executor {
295
406
  return { action: receiptAction, startedAt, durationMs: Date.now() - startedAt, success: true, strategy, data };
296
407
  }
297
408
  catch (error) {
298
- return { action: receiptAction, startedAt, durationMs: Date.now() - startedAt, success: false, strategy, error: this.variables.redact(error instanceof Error ? error.message : 'Browser action failed'), uncertain: executed && ['click', 'doubleClick', 'press', 'submit', 'download'].includes(action.type) };
409
+ const blockedBeforeNavigation = error instanceof SecurityBoundaryError && ['navigate', 'openTab'].includes(action.type) && this.browser.page === pageBefore;
410
+ return { action: receiptAction, startedAt, durationMs: Date.now() - startedAt, success: false, strategy, error: this.variables.redact(error instanceof Error ? error.message : 'Browser action failed'), uncertain: executed && ((navigationUncertain && !blockedBeforeNavigation) || ['click', 'doubleClick', 'press', 'submit', 'download', 'type'].includes(action.type)) };
299
411
  }
300
412
  finally {
301
413
  await disposeApproved?.();