@owlmeans/llm-reader-playwright 0.0.0-stage → 0.1.22-rc.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +65 -2
- package/agent-meta/manifest.json +16 -0
- package/agent-meta/skills/llm-reader-playwright/SKILL.md +67 -0
- package/build/consts.d.ts +14 -0
- package/build/consts.d.ts.map +1 -0
- package/build/consts.js +11 -0
- package/build/consts.js.map +1 -0
- package/build/index.d.ts +5 -0
- package/build/index.d.ts.map +1 -0
- package/build/index.js +4 -0
- package/build/index.js.map +1 -0
- package/build/plugin.d.ts +9 -0
- package/build/plugin.d.ts.map +1 -0
- package/build/plugin.js +41 -0
- package/build/plugin.js.map +1 -0
- package/build/renderer/probe.d.ts +8 -0
- package/build/renderer/probe.d.ts.map +1 -0
- package/build/renderer/probe.js +116 -0
- package/build/renderer/probe.js.map +1 -0
- package/build/renderer/types.d.ts +42 -0
- package/build/renderer/types.d.ts.map +1 -0
- package/build/renderer/types.js +2 -0
- package/build/renderer/types.js.map +1 -0
- package/build/renderer/types.local.d.ts +3 -0
- package/build/renderer/types.local.d.ts.map +1 -0
- package/build/renderer/types.local.js +2 -0
- package/build/renderer/types.local.js.map +1 -0
- package/build/renderer.d.ts +11 -0
- package/build/renderer.d.ts.map +1 -0
- package/build/renderer.js +94 -0
- package/build/renderer.js.map +1 -0
- package/package.json +53 -4
- package/src/consts.ts +14 -0
- package/src/index.ts +4 -0
- package/src/plugin.ts +43 -0
- package/src/renderer/probe.ts +103 -0
- package/src/renderer/types.local.ts +2 -0
- package/src/renderer/types.ts +39 -0
- package/src/renderer.ts +97 -0
- package/tests/bundle.spec.ts +81 -0
- package/tests/render.spec.ts +87 -0
- package/tsconfig.json +23 -0
package/README.md
CHANGED
|
@@ -1,3 +1,66 @@
|
|
|
1
|
-
#
|
|
1
|
+
# @owlmeans/llm-reader-playwright
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
A Playwright renderer for `@owlmeans/llm-reader`: pages run their scripts before they are read, and
|
|
4
|
+
the design analysis gets computed styles, custom properties, block geometry and a screenshot.
|
|
5
|
+
|
|
6
|
+
## Installation
|
|
7
|
+
|
|
8
|
+
```bash
|
|
9
|
+
bun add @owlmeans/llm-reader-playwright@^0.1.22-rc.0 playwright
|
|
10
|
+
bunx playwright install chromium
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
## Usage
|
|
14
|
+
|
|
15
|
+
```typescript
|
|
16
|
+
import { createLlmReader, createLocalBundlePlugin, createProxyPlugin } from '@owlmeans/llm-reader'
|
|
17
|
+
import { createPlaywrightRenderer, makePlaywrightFetchPlugin } from '@owlmeans/llm-reader-playwright'
|
|
18
|
+
|
|
19
|
+
const renderer = createPlaywrightRenderer()
|
|
20
|
+
const reader = createLlmReader({ renderer, plugins: [createProxyPlugin({ proxies })], models })
|
|
21
|
+
|
|
22
|
+
await reader.design('https://app.example.com') // rendered, then measured
|
|
23
|
+
await reader.page('https://spa.example.com') // an app shell is re-read rendered
|
|
24
|
+
|
|
25
|
+
// Or render every page fetch:
|
|
26
|
+
createLlmReader({ plugins: [createProxyPlugin({ proxies }), makePlaywrightFetchPlugin(renderer)] })
|
|
27
|
+
|
|
28
|
+
// A local prototype (an uploaded home.html with its css and pictures), rendered from disk:
|
|
29
|
+
const bundle = createLocalBundlePlugin({ root: '/uploads/prototype' })
|
|
30
|
+
await createLlmReader({ renderer, plugins: [bundle], models }).design(bundle.urlOf('home.html'))
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
The browser goes through the reader's proxy lease (SOCKS proxies through the reader's bridge), and
|
|
34
|
+
every request a page makes passes the reader's address guard; refused ones are listed in
|
|
35
|
+
`RenderedPage.blocked`.
|
|
36
|
+
|
|
37
|
+
## API
|
|
38
|
+
|
|
39
|
+
| Export | Purpose |
|
|
40
|
+
|---|---|
|
|
41
|
+
| `createPlaywrightRenderer(options)` | `render({ url, lease?, guard?, styles?, screenshot?, bundle? })` → `RenderedPage`; `close()`. Options: `browser`, `launch`, `viewport`, `timeout`, `waitUntil`, `settle`, `userAgent`, `bundles` (local bundles routed from disk under their synthetic origins; every other request still guarded). |
|
|
42
|
+
| `makePlaywrightFetchPlugin(renderer, { when? })` | A reader fetch plugin that renders page requests. |
|
|
43
|
+
|
|
44
|
+
## Tests
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
bun test ./tests
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
Gated on an installed Chromium; the design spec also on `ANTHROPIC_SECRET` or `OPENAI_SECRET`.
|
|
51
|
+
|
|
52
|
+
<!-- owlmeans:agent-guidance:start -->
|
|
53
|
+
## Agent guidance
|
|
54
|
+
|
|
55
|
+
This package ships embedded agent skills under `agent-meta/`. After installing your
|
|
56
|
+
`@owlmeans/*` packages, run the OwlMeans agent-skills installer to place them into
|
|
57
|
+
your project's skill store (`.agents/skills/`):
|
|
58
|
+
|
|
59
|
+
```sh
|
|
60
|
+
npx @owlmeans/agent-skills@^0.1.22-rc.0
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
The embedded files are version-matched to this package release. Do not edit them
|
|
64
|
+
directly — they are regenerated on each publish. To contribute guidance edits,
|
|
65
|
+
open a PR against the source monorepo.
|
|
66
|
+
<!-- owlmeans:agent-guidance:end -->
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schemaVersion": 3,
|
|
3
|
+
"package": "@owlmeans/llm-reader-playwright",
|
|
4
|
+
"version": "0.1.22-rc.0",
|
|
5
|
+
"generatedAt": "2026-11-10T17:39:44.206Z",
|
|
6
|
+
"canonicalRepo": "https://github.com/owlmeans/common",
|
|
7
|
+
"entries": [
|
|
8
|
+
{
|
|
9
|
+
"kind": "skill",
|
|
10
|
+
"name": "llm-reader-playwright",
|
|
11
|
+
"category": "package-specific",
|
|
12
|
+
"file": "skills/llm-reader-playwright/SKILL.md",
|
|
13
|
+
"canonicalPath": ".agents/skills/llm-reader-playwright/SKILL.md"
|
|
14
|
+
}
|
|
15
|
+
]
|
|
16
|
+
}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: llm-reader-playwright
|
|
3
|
+
description: How to use @owlmeans/llm-reader-playwright — a Playwright PageRenderer for @owlmeans/llm-reader that runs a page's scripts, samples computed styles by role, reads custom properties and block geometry, takes a screenshot, routes the browser through the reader's proxy lease (SOCKS through the bridge) and guards every request it makes, serves local bundles (uploaded html/css prototypes) under their synthetic origin; and a fetch plugin that renders inside the reader's chain. Auto-invoked when importing createPlaywrightRenderer or makePlaywrightFetchPlugin.
|
|
4
|
+
user-invocable: false
|
|
5
|
+
---
|
|
6
|
+
<!-- AUTO-GENERATED — do not edit. Regenerate via sync-agent-meta. -->
|
|
7
|
+
|
|
8
|
+
# @owlmeans/llm-reader-playwright
|
|
9
|
+
|
|
10
|
+
**Layer:** Server (drives a browser)
|
|
11
|
+
**Install:** `"@owlmeans/llm-reader-playwright": "^0.1.22-rc.0"` in `dependencies`, plus the
|
|
12
|
+
`playwright` **peer** — the consumer's installed browsers are used (`bunx playwright install chromium`)
|
|
13
|
+
|
|
14
|
+
```ts
|
|
15
|
+
import { createLlmReader, createLocalBundlePlugin, createProxyPlugin } from '@owlmeans/llm-reader'
|
|
16
|
+
import { createPlaywrightRenderer } from '@owlmeans/llm-reader-playwright'
|
|
17
|
+
|
|
18
|
+
const renderer = createPlaywrightRenderer()
|
|
19
|
+
const reader = createLlmReader({ renderer, plugins: [createProxyPlugin({ proxies })], models })
|
|
20
|
+
await reader.design('https://app.example.com') // rendered: computed styles + screenshot feed the DESIGN.md
|
|
21
|
+
await reader.page('https://spa.example.com') // an app shell is re-read rendered automatically
|
|
22
|
+
|
|
23
|
+
const bundle = createLocalBundlePlugin({ root: '/uploads/prototype' })
|
|
24
|
+
const local = createLlmReader({ renderer, plugins: [bundle], models })
|
|
25
|
+
await local.design(bundle.urlOf('home.html')) // the prototype rendered from disk: styles + screenshot
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
## Key exports
|
|
29
|
+
|
|
30
|
+
| Export | Description |
|
|
31
|
+
|---|---|
|
|
32
|
+
| `createPlaywrightRenderer({ browser?, launch?, viewport, timeout, waitUntil, settle, userAgent, bundles? })` | A `PageRenderer`: `render({ url, lease?, guard?, styles?, screenshot?, signal?, timeout?, bundle? })` → `RenderedPage` (HTML after scripts, final URL, status, `styles`, `variables`, `blocks`, JPEG `screenshot`, `blocked`); `close()` shuts down a browser it launched. |
|
|
33
|
+
| `makePlaywrightFetchPlugin(renderer, { when? })` | A reader fetch plugin that renders page requests (all, or those `when` picks). Place it after the proxy plugin so it renders through the chosen lease. |
|
|
34
|
+
| `PlaywrightRendererOptions`, `ProbeResult`, `DEFAULT_VIEWPORT`, `DEFAULT_RENDER_TIMEOUT_MS`, `DEFAULT_SETTLE_MS` | Options and defaults (1280×800, 30 s navigation, 3 s to go network-idle). |
|
|
35
|
+
|
|
36
|
+
## Rules
|
|
37
|
+
|
|
38
|
+
- **One browser context per page**, closed after: viewport set, downloads off, service workers blocked.
|
|
39
|
+
- **Proxy**: the context's proxy is the lease's `endpoint()` — the upstream HTTP(S) proxy, or the
|
|
40
|
+
reader's SOCKS bridge, which answers a 407 challenge so Chromium sends the route credentials
|
|
41
|
+
(Chromium has no SOCKS authentication of its own). Chromium sends loopback targets direct unless
|
|
42
|
+
launched with `--proxy-bypass-list=<-loopback>`.
|
|
43
|
+
- **Guard**: every request of the page goes through the reader's address guard (a direct render
|
|
44
|
+
resolves names, a proxied one checks literal addresses); refused ones are aborted and listed in
|
|
45
|
+
`RenderedPage.blocked`.
|
|
46
|
+
- **Local bundles**: a request under the origin of a bundle in `bundles` (or the render request's
|
|
47
|
+
`bundle`, which the reader sets for a bundle URL) is fulfilled from the bundle (`page.route`) —
|
|
48
|
+
never resolved, never proxied; a missing file is a 404, a refused path is aborted and listed in
|
|
49
|
+
`blocked`. Everything else the page loads still goes through the guard, so a prototype that pulls
|
|
50
|
+
a picture from a private address is refused as any page would be. Chromium's own local-network
|
|
51
|
+
rules may still stop an `https` bundle page from reaching a loopback host the guard allowed.
|
|
52
|
+
- **Probe**: computed styles of visible elements, by role (body, heading-1…6, link, button, input,
|
|
53
|
+
card, nav, header, footer, text), each property's dominant value weighted by on-screen area;
|
|
54
|
+
`:root` custom properties from readable stylesheets; blocks at least 80 px tall.
|
|
55
|
+
|
|
56
|
+
## Depends On
|
|
57
|
+
|
|
58
|
+
- `@owlmeans/llm-reader` (the seam, the guard, leases) · `@owlmeans/error` · `@owlmeans/log` · peer `playwright`
|
|
59
|
+
|
|
60
|
+
## External docs
|
|
61
|
+
|
|
62
|
+
- https://playwright.dev/docs/api/class-browser#browser-new-context-option-proxy — per-context `proxy: { server, username, password, bypass }`; HTTP proxy authentication only (2026-11)
|
|
63
|
+
- https://playwright.dev/docs/api/class-browsercontext#browser-context-route — one `route('**/*')` handler per context answers every request: `route.fulfill({ status, contentType, body })` for a bundle file (no DNS, no proxy), `continue()` / `abort('blockedbyclient')` after the guard. Observed with Playwright 1.64's Chromium: an `https` page's `http://<IP>` sub-requests are not upgraded and do reach the route; a loopback one the guard allows can still be stopped by Chromium's local-network rules (2026-11)
|
|
64
|
+
|
|
65
|
+
## Related
|
|
66
|
+
|
|
67
|
+
- [[llm-reader]] — the reader, its proxies and its design analysis
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/** The viewport pages are rendered and screenshot in. */
|
|
2
|
+
export declare const DEFAULT_VIEWPORT: Readonly<{
|
|
3
|
+
width: 1280;
|
|
4
|
+
height: 800;
|
|
5
|
+
}>;
|
|
6
|
+
/** Navigation deadline, ms. */
|
|
7
|
+
export declare const DEFAULT_RENDER_TIMEOUT_MS = 30000;
|
|
8
|
+
/** How long a loaded page may keep fetching before it is read anyway, ms. */
|
|
9
|
+
export declare const DEFAULT_SETTLE_MS = 3000;
|
|
10
|
+
/** JPEG quality of the viewport screenshot. */
|
|
11
|
+
export declare const SCREENSHOT_QUALITY = 70;
|
|
12
|
+
/** Elements the style probe samples, at most. */
|
|
13
|
+
export declare const PROBE_ELEMENTS = 4000;
|
|
14
|
+
//# sourceMappingURL=consts.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"consts.d.ts","sourceRoot":"","sources":["../src/consts.ts"],"names":[],"mappings":"AAAA,yDAAyD;AACzD,eAAO,MAAM,gBAAgB;;;EAA8C,CAAA;AAE3E,+BAA+B;AAC/B,eAAO,MAAM,yBAAyB,QAAS,CAAA;AAE/C,6EAA6E;AAC7E,eAAO,MAAM,iBAAiB,OAAQ,CAAA;AAEtC,+CAA+C;AAC/C,eAAO,MAAM,kBAAkB,KAAK,CAAA;AAEpC,iDAAiD;AACjD,eAAO,MAAM,cAAc,OAAO,CAAA"}
|
package/build/consts.js
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/** The viewport pages are rendered and screenshot in. */
|
|
2
|
+
export const DEFAULT_VIEWPORT = Object.freeze({ width: 1280, height: 800 });
|
|
3
|
+
/** Navigation deadline, ms. */
|
|
4
|
+
export const DEFAULT_RENDER_TIMEOUT_MS = 30_000;
|
|
5
|
+
/** How long a loaded page may keep fetching before it is read anyway, ms. */
|
|
6
|
+
export const DEFAULT_SETTLE_MS = 3_000;
|
|
7
|
+
/** JPEG quality of the viewport screenshot. */
|
|
8
|
+
export const SCREENSHOT_QUALITY = 70;
|
|
9
|
+
/** Elements the style probe samples, at most. */
|
|
10
|
+
export const PROBE_ELEMENTS = 4000;
|
|
11
|
+
//# sourceMappingURL=consts.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"consts.js","sourceRoot":"","sources":["../src/consts.ts"],"names":[],"mappings":"AAAA,yDAAyD;AACzD,MAAM,CAAC,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAA;AAE3E,+BAA+B;AAC/B,MAAM,CAAC,MAAM,yBAAyB,GAAG,MAAM,CAAA;AAE/C,6EAA6E;AAC7E,MAAM,CAAC,MAAM,iBAAiB,GAAG,KAAK,CAAA;AAEtC,+CAA+C;AAC/C,MAAM,CAAC,MAAM,kBAAkB,GAAG,EAAE,CAAA;AAEpC,iDAAiD;AACjD,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,CAAA"}
|
package/build/index.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAA;AAC3B,mBAAmB,qBAAqB,CAAA;AACxC,cAAc,eAAe,CAAA;AAC7B,cAAc,aAAa,CAAA"}
|
package/build/index.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAA;AAE3B,cAAc,eAAe,CAAA;AAC7B,cAAc,aAAa,CAAA"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { ReaderFetchPlugin } from '@owlmeans/llm-reader';
|
|
2
|
+
import type { PlaywrightFetchPluginOptions, PlaywrightRenderer } from './renderer/types.js';
|
|
3
|
+
/**
|
|
4
|
+
* A reader fetch plugin that renders pages instead of downloading them — for a site whose every
|
|
5
|
+
* page needs its scripts. Put it after the proxy plugin, so the lease it renders through is chosen
|
|
6
|
+
* (and failed over) by the proxy plugin like any fetch.
|
|
7
|
+
*/
|
|
8
|
+
export declare const makePlaywrightFetchPlugin: (renderer: PlaywrightRenderer, options?: PlaywrightFetchPluginOptions) => ReaderFetchPlugin;
|
|
9
|
+
//# sourceMappingURL=plugin.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"plugin.d.ts","sourceRoot":"","sources":["../src/plugin.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAA;AAC7D,OAAO,KAAK,EAAE,4BAA4B,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAA;AAE3F;;;;GAIG;AACH,eAAO,MAAM,yBAAyB,aAAc,kBAAkB,YAAW,4BAA4B,KAAQ,iBAkCpH,CAAA"}
|
package/build/plugin.js
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A reader fetch plugin that renders pages instead of downloading them — for a site whose every
|
|
3
|
+
* page needs its scripts. Put it after the proxy plugin, so the lease it renders through is chosen
|
|
4
|
+
* (and failed over) by the proxy plugin like any fetch.
|
|
5
|
+
*/
|
|
6
|
+
export const makePlaywrightFetchPlugin = (renderer, options = {}) => {
|
|
7
|
+
let guard;
|
|
8
|
+
return {
|
|
9
|
+
name: 'playwright',
|
|
10
|
+
attach: context => {
|
|
11
|
+
guard = context.guard;
|
|
12
|
+
},
|
|
13
|
+
fetch: async (request, next) => {
|
|
14
|
+
const wanted = options.when != null ? options.when(request) : (request.purpose ?? 'page') === 'page';
|
|
15
|
+
if (!wanted)
|
|
16
|
+
return next(request);
|
|
17
|
+
const rendered = await renderer.render({
|
|
18
|
+
url: request.url, styles: true, screenshot: true,
|
|
19
|
+
...(guard != null ? { guard } : {}),
|
|
20
|
+
...(request.lease != null ? { lease: request.lease } : {}),
|
|
21
|
+
...(request.signal != null ? { signal: request.signal } : {}),
|
|
22
|
+
...(request.timeout != null ? { timeout: request.timeout } : {}),
|
|
23
|
+
});
|
|
24
|
+
return {
|
|
25
|
+
url: request.url,
|
|
26
|
+
finalUrl: rendered.finalUrl,
|
|
27
|
+
status: rendered.status,
|
|
28
|
+
headers: {},
|
|
29
|
+
contentType: 'text/html',
|
|
30
|
+
charset: 'utf-8',
|
|
31
|
+
text: rendered.html,
|
|
32
|
+
fetchedAt: new Date().toISOString(),
|
|
33
|
+
redirects: [],
|
|
34
|
+
...(request.lease != null ? { via: request.lease.id } : {}),
|
|
35
|
+
rendered,
|
|
36
|
+
};
|
|
37
|
+
},
|
|
38
|
+
close: () => renderer.close(),
|
|
39
|
+
};
|
|
40
|
+
};
|
|
41
|
+
//# sourceMappingURL=plugin.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"plugin.js","sourceRoot":"","sources":["../src/plugin.ts"],"names":[],"mappings":"AAGA;;;;GAIG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAG,CAAC,QAA4B,EAAE,OAAO,GAAiC,EAAE,EAAqB,EAAE;IACvI,IAAI,KAAmF,CAAA;IAEvF,OAAO;QACL,IAAI,EAAE,YAAY;QAClB,MAAM,EAAE,OAAO,CAAC,EAAE;YAChB,KAAK,GAAG,OAAO,CAAC,KAAK,CAAA;QACvB,CAAC;QACD,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE;YAC7B,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,IAAI,MAAM,CAAC,KAAK,MAAM,CAAA;YACpG,IAAI,CAAC,MAAM;gBAAE,OAAO,IAAI,CAAC,OAAO,CAAC,CAAA;YACjC,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC;gBACrC,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI;gBAChD,GAAG,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACnC,GAAG,CAAC,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC1D,GAAG,CAAC,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC7D,GAAG,CAAC,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACjE,CAAC,CAAA;YACF,OAAO;gBACL,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;gBAC3B,MAAM,EAAE,QAAQ,CAAC,MAAM;gBACvB,OAAO,EAAE,EAAE;gBACX,WAAW,EAAE,WAAW;gBACxB,OAAO,EAAE,OAAO;gBAChB,IAAI,EAAE,QAAQ,CAAC,IAAI;gBACnB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,SAAS,EAAE,EAAE;gBACb,GAAG,CAAC,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC3D,QAAQ;aACT,CAAA;QACH,CAAC;QACD,KAAK,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,KAAK,EAAE;KAC9B,CAAA;AACH,CAAC,CAAA"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { ProbeResult } from './types.js';
|
|
2
|
+
/**
|
|
3
|
+
* Runs INSIDE the rendered page (serialized by Playwright — it may use nothing from this module's
|
|
4
|
+
* scope). Samples the computed styles of visible elements by role, weighted by on-screen area, and
|
|
5
|
+
* reads the `:root` custom properties and the large blocks of the layout.
|
|
6
|
+
*/
|
|
7
|
+
export declare const probePage: (limit: number) => ProbeResult;
|
|
8
|
+
//# sourceMappingURL=probe.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"probe.d.ts","sourceRoot":"","sources":["../../src/renderer/probe.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAA;AAG7C;;;;GAIG;AACH,eAAO,MAAM,SAAS,UAAW,MAAM,KAAG,WA8FzC,CAAA"}
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Runs INSIDE the rendered page (serialized by Playwright — it may use nothing from this module's
|
|
3
|
+
* scope). Samples the computed styles of visible elements by role, weighted by on-screen area, and
|
|
4
|
+
* reads the `:root` custom properties and the large blocks of the layout.
|
|
5
|
+
*/
|
|
6
|
+
export const probePage = (limit) => {
|
|
7
|
+
const roleOf = (element) => {
|
|
8
|
+
const tag = element.tagName.toLowerCase();
|
|
9
|
+
const cls = typeof element.className === 'string' ? element.className.toLowerCase() : '';
|
|
10
|
+
if (tag === 'body')
|
|
11
|
+
return 'body';
|
|
12
|
+
if (/^h[1-6]$/.test(tag))
|
|
13
|
+
return `heading-${tag[1]}`;
|
|
14
|
+
if (tag === 'button' || element.getAttribute('role') === 'button' || /\b(btn|button)\b/.test(cls))
|
|
15
|
+
return 'button';
|
|
16
|
+
if (tag === 'a')
|
|
17
|
+
return 'link';
|
|
18
|
+
if (tag === 'input' || tag === 'textarea' || tag === 'select')
|
|
19
|
+
return 'input';
|
|
20
|
+
if (/\bcard\b/.test(cls))
|
|
21
|
+
return 'card';
|
|
22
|
+
if (tag === 'nav' || tag === 'header' || tag === 'footer')
|
|
23
|
+
return tag;
|
|
24
|
+
const ownText = Array.from(element.childNodes).some(node => node.nodeType === 3 && (node.textContent ?? '').trim().length > 0);
|
|
25
|
+
return ownText ? 'text' : undefined;
|
|
26
|
+
};
|
|
27
|
+
const roles = new Map();
|
|
28
|
+
const vote = (tally, property, value, weight) => {
|
|
29
|
+
if (value == null || value === '' || value === 'none' || value === 'normal' || value === 'rgba(0, 0, 0, 0)')
|
|
30
|
+
return;
|
|
31
|
+
const votes = tally[property] ??= {};
|
|
32
|
+
votes[value] = (votes[value] ?? 0) + weight;
|
|
33
|
+
};
|
|
34
|
+
let total = 0;
|
|
35
|
+
const viewport = { width: window.innerWidth, height: window.innerHeight };
|
|
36
|
+
const elements = Array.from(document.querySelectorAll('body, body *')).slice(0, limit);
|
|
37
|
+
for (const element of elements) {
|
|
38
|
+
const rect = element.getBoundingClientRect();
|
|
39
|
+
const area = Math.min(rect.width, viewport.width) * Math.min(rect.height, viewport.height * 3);
|
|
40
|
+
if (area <= 0)
|
|
41
|
+
continue;
|
|
42
|
+
const style = getComputedStyle(element);
|
|
43
|
+
if (style.visibility === 'hidden' || style.display === 'none' || style.opacity === '0')
|
|
44
|
+
continue;
|
|
45
|
+
const role = roleOf(element);
|
|
46
|
+
if (role == null)
|
|
47
|
+
continue;
|
|
48
|
+
const entry = roles.get(role) ?? { count: 0, area: 0, values: {} };
|
|
49
|
+
entry.count++;
|
|
50
|
+
entry.area += area;
|
|
51
|
+
total += area;
|
|
52
|
+
const weight = role === 'body' ? 1 : area;
|
|
53
|
+
vote(entry.values, 'color', style.color, weight);
|
|
54
|
+
vote(entry.values, 'background', style.backgroundColor, weight);
|
|
55
|
+
if (Number.parseFloat(style.borderTopWidth) > 0)
|
|
56
|
+
vote(entry.values, 'borderColor', style.borderTopColor, weight);
|
|
57
|
+
vote(entry.values, 'fontFamily', style.fontFamily, weight);
|
|
58
|
+
vote(entry.values, 'fontSize', style.fontSize, weight);
|
|
59
|
+
vote(entry.values, 'fontWeight', style.fontWeight, weight);
|
|
60
|
+
vote(entry.values, 'lineHeight', style.lineHeight, weight);
|
|
61
|
+
vote(entry.values, 'letterSpacing', style.letterSpacing, weight);
|
|
62
|
+
if (style.borderTopLeftRadius !== '0px')
|
|
63
|
+
vote(entry.values, 'radius', style.borderTopLeftRadius, weight);
|
|
64
|
+
vote(entry.values, 'shadow', style.boxShadow, weight);
|
|
65
|
+
if (style.padding !== '0px')
|
|
66
|
+
vote(entry.values, 'padding', style.padding, weight);
|
|
67
|
+
if (style.gap !== 'normal' && style.gap !== '0px')
|
|
68
|
+
vote(entry.values, 'gap', style.gap, weight);
|
|
69
|
+
roles.set(role, entry);
|
|
70
|
+
}
|
|
71
|
+
const top = (votes) => votes == null ? undefined : Object.entries(votes).sort((a, b) => b[1] - a[1])[0]?.[0];
|
|
72
|
+
const styles = [...roles.entries()].map(([role, entry]) => {
|
|
73
|
+
const sample = { role, count: entry.count, weight: total > 0 ? entry.area / total : 0 };
|
|
74
|
+
for (const property of ['color', 'background', 'borderColor', 'fontFamily', 'fontSize', 'fontWeight', 'lineHeight', 'letterSpacing', 'radius', 'shadow', 'padding', 'gap']) {
|
|
75
|
+
const value = top(entry.values[property]);
|
|
76
|
+
if (value != null)
|
|
77
|
+
sample[property] = value;
|
|
78
|
+
}
|
|
79
|
+
return sample;
|
|
80
|
+
});
|
|
81
|
+
const variables = {};
|
|
82
|
+
const root = getComputedStyle(document.documentElement);
|
|
83
|
+
for (const sheet of Array.from(document.styleSheets)) {
|
|
84
|
+
let rules;
|
|
85
|
+
try {
|
|
86
|
+
rules = sheet.cssRules;
|
|
87
|
+
}
|
|
88
|
+
catch {
|
|
89
|
+
continue; // a cross-origin sheet without CORS cannot be read
|
|
90
|
+
}
|
|
91
|
+
for (const rule of Array.from(rules)) {
|
|
92
|
+
const styleRule = rule;
|
|
93
|
+
if (styleRule.selectorText == null || !/(^|,)\s*(:root|html)\s*(,|$)/.test(styleRule.selectorText))
|
|
94
|
+
continue;
|
|
95
|
+
for (let i = 0; i < styleRule.style.length; i++) {
|
|
96
|
+
const name = styleRule.style[i];
|
|
97
|
+
if (name.startsWith('--'))
|
|
98
|
+
variables[name] = root.getPropertyValue(name).trim();
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
const container = document.querySelector('main') ?? document.body;
|
|
103
|
+
const blocks = Array.from(document.body.children).concat(container !== document.body ? Array.from(container.children) : [])
|
|
104
|
+
.map(element => ({ element, rect: element.getBoundingClientRect() }))
|
|
105
|
+
.filter(({ rect }) => rect.height >= 80 && rect.width >= viewport.width * 0.3)
|
|
106
|
+
.slice(0, 30)
|
|
107
|
+
.map(({ element, rect }) => ({
|
|
108
|
+
tag: element.tagName.toLowerCase(),
|
|
109
|
+
label: element.id || (typeof element.className === 'string' ? element.className.split(/\s+/)[0] : '') || undefined,
|
|
110
|
+
x: Math.round(rect.x), y: Math.round(rect.y + window.scrollY), width: Math.round(rect.width), height: Math.round(rect.height),
|
|
111
|
+
background: getComputedStyle(element).backgroundColor,
|
|
112
|
+
text: (element.innerText ?? '').replace(/\s+/g, ' ').trim().slice(0, 100),
|
|
113
|
+
}));
|
|
114
|
+
return { styles, variables, blocks, viewport };
|
|
115
|
+
};
|
|
116
|
+
//# sourceMappingURL=probe.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"probe.js","sourceRoot":"","sources":["../../src/renderer/probe.ts"],"names":[],"mappings":"AAGA;;;;GAIG;AACH,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,KAAa,EAAe,EAAE;IACtD,MAAM,MAAM,GAAG,CAAC,OAAgB,EAAsB,EAAE;QACtD,MAAM,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,CAAA;QACzC,MAAM,GAAG,GAAG,OAAO,OAAO,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAA;QACxF,IAAI,GAAG,KAAK,MAAM;YAAE,OAAO,MAAM,CAAA;QACjC,IAAI,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,OAAO,WAAW,GAAG,CAAC,CAAC,CAAC,EAAE,CAAA;QACpD,IAAI,GAAG,KAAK,QAAQ,IAAI,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC,KAAK,QAAQ,IAAI,kBAAkB,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,OAAO,QAAQ,CAAA;QAClH,IAAI,GAAG,KAAK,GAAG;YAAE,OAAO,MAAM,CAAA;QAC9B,IAAI,GAAG,KAAK,OAAO,IAAI,GAAG,KAAK,UAAU,IAAI,GAAG,KAAK,QAAQ;YAAE,OAAO,OAAO,CAAA;QAC7E,IAAI,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,OAAO,MAAM,CAAA;QACvC,IAAI,GAAG,KAAK,KAAK,IAAI,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,QAAQ;YAAE,OAAO,GAAG,CAAA;QACrE,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;QAC9H,OAAO,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAA;IACrC,CAAC,CAAA;IACD,MAAM,KAAK,GAAG,IAAI,GAAG,EAA0D,CAAA;IAC/E,MAAM,IAAI,GAAG,CAAC,KAAY,EAAE,QAAgB,EAAE,KAAyB,EAAE,MAAc,EAAQ,EAAE;QAC/F,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,KAAK,EAAE,IAAI,KAAK,KAAK,MAAM,IAAI,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,kBAAkB;YAAE,OAAM;QACnH,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAA;QACpC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,MAAM,CAAA;IAC7C,CAAC,CAAA;IACD,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,MAAM,QAAQ,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,CAAC,WAAW,EAAE,CAAA;IACzE,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,cAAc,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAA;IACtF,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,MAAM,IAAI,GAAG,OAAO,CAAC,qBAAqB,EAAE,CAAA;QAC5C,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;QAC9F,IAAI,IAAI,IAAI,CAAC;YAAE,SAAQ;QACvB,MAAM,KAAK,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAA;QACvC,IAAI,KAAK,CAAC,UAAU,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,KAAK,MAAM,IAAI,KAAK,CAAC,OAAO,KAAK,GAAG;YAAE,SAAQ;QAChG,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,CAAA;QAC5B,IAAI,IAAI,IAAI,IAAI;YAAE,SAAQ;QAC1B,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAA;QAClE,KAAK,CAAC,KAAK,EAAE,CAAA;QACb,KAAK,CAAC,IAAI,IAAI,IAAI,CAAA;QAClB,KAAK,IAAI,IAAI,CAAA;QACb,MAAM,MAAM,GAAG,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;QACzC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;QAChD,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,YAAY,EAAE,KAAK,CAAC,eAAe,EAAE,MAAM,CAAC,CAAA;QAC/D,IAAI,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,cAAc,CAAC,GAAG,CAAC;YAAE,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,CAAC,cAAc,EAAE,MAAM,CAAC,CAAA;QAChH,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,YAAY,EAAE,KAAK,CAAC,UAAU,EAAE,MAAM,CAAC,CAAA;QAC1D,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;QACtD,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,YAAY,EAAE,KAAK,CAAC,UAAU,EAAE,MAAM,CAAC,CAAA;QAC1D,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,YAAY,EAAE,KAAK,CAAC,UAAU,EAAE,MAAM,CAAC,CAAA;QAC1D,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,eAAe,EAAE,KAAK,CAAC,aAAa,EAAE,MAAM,CAAC,CAAA;QAChE,IAAI,KAAK,CAAC,mBAAmB,KAAK,KAAK;YAAE,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,QAAQ,EAAE,KAAK,CAAC,mBAAmB,EAAE,MAAM,CAAC,CAAA;QACxG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,QAAQ,EAAE,KAAK,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;QACrD,IAAI,KAAK,CAAC,OAAO,KAAK,KAAK;YAAE,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,SAAS,EAAE,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;QACjF,IAAI,KAAK,CAAC,GAAG,KAAK,QAAQ,IAAI,KAAK,CAAC,GAAG,KAAK,KAAK;YAAE,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,GAAG,EAAE,MAAM,CAAC,CAAA;QAC/F,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;IACxB,CAAC;IACD,MAAM,GAAG,GAAG,CAAC,KAAyC,EAAsB,EAAE,CAC5E,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;IACvF,MAAM,MAAM,GAAG,CAAC,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE;QACxD,MAAM,MAAM,GAAoC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;QACxH,KAAK,MAAM,QAAQ,IAAI,CAAC,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,YAAY,EAAE,UAAU,EAAE,YAAY,EAAE,YAAY,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,CAAC,EAAE,CAAC;YAC3K,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAA;YACzC,IAAI,KAAK,IAAI,IAAI;gBAAE,MAAM,CAAC,QAAQ,CAAC,GAAG,KAAK,CAAA;QAC7C,CAAC;QACD,OAAO,MAAkD,CAAA;IAC3D,CAAC,CAAC,CAAA;IAEF,MAAM,SAAS,GAA2B,EAAE,CAAA;IAC5C,MAAM,IAAI,GAAG,gBAAgB,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAA;IACvD,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;QACrD,IAAI,KAAkB,CAAA;QACtB,IAAI,CAAC;YACH,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAA;QACxB,CAAC;QAAC,MAAM,CAAC;YACP,SAAQ,CAAC,mDAAmD;QAC9D,CAAC;QACD,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YACrC,MAAM,SAAS,GAAG,IAAoB,CAAA;YACtC,IAAI,SAAS,CAAC,YAAY,IAAI,IAAI,IAAI,CAAC,8BAA8B,CAAC,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC;gBAAE,SAAQ;YAC5G,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAChD,MAAM,IAAI,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,CAAE,CAAA;gBAChC,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;oBAAE,SAAS,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAA;YACjF,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,SAAS,GAAG,QAAQ,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,QAAQ,CAAC,IAAI,CAAA;IACjE,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,SAAS,KAAK,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;SACxH,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,CAAC,qBAAqB,EAAE,EAAE,CAAC,CAAC;SACpE,MAAM,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,IAAI,IAAI,CAAC,KAAK,IAAI,QAAQ,CAAC,KAAK,GAAG,GAAG,CAAC;SAC7E,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;SACZ,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;QAC3B,GAAG,EAAE,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE;QAClC,KAAK,EAAE,OAAO,CAAC,EAAE,IAAI,CAAC,OAAO,OAAO,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,SAAS;QAClH,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC;QAC7H,UAAU,EAAE,gBAAgB,CAAC,OAAO,CAAC,CAAC,eAAe;QACrD,IAAI,EAAE,CAAE,OAAuB,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;KAC3F,CAAC,CAAC,CAAA;IAEL,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAA;AAChD,CAAC,CAAA"}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import type { Browser, LaunchOptions } from 'playwright';
|
|
2
|
+
import type { FetchRequest, LocalBundle, PageRenderer, RenderedBlock, StyleSample } from '@owlmeans/llm-reader';
|
|
3
|
+
export interface PlaywrightRendererOptions {
|
|
4
|
+
/** Your own browser, or how to get one; otherwise Chromium is launched headless on first use. */
|
|
5
|
+
browser?: Browser | (() => Promise<Browser>);
|
|
6
|
+
launch?: LaunchOptions;
|
|
7
|
+
viewport?: {
|
|
8
|
+
width: number;
|
|
9
|
+
height: number;
|
|
10
|
+
};
|
|
11
|
+
/** Navigation deadline, ms. */
|
|
12
|
+
timeout?: number;
|
|
13
|
+
/** `load` by default; the page then gets up to `settle` ms to go network-idle. */
|
|
14
|
+
waitUntil?: 'load' | 'domcontentloaded' | 'networkidle' | 'commit';
|
|
15
|
+
settle?: number;
|
|
16
|
+
userAgent?: string;
|
|
17
|
+
/**
|
|
18
|
+
* Local bundles (`createLocalBundlePlugin`) whose synthetic origins every page may load from: a
|
|
19
|
+
* request under one is answered from its root (`page.route`), never resolved or sent. A render
|
|
20
|
+
* request's own `bundle` — set by the reader for a bundle URL — is routed the same way.
|
|
21
|
+
*/
|
|
22
|
+
bundles?: LocalBundle[];
|
|
23
|
+
}
|
|
24
|
+
/** A {@link PageRenderer} on Playwright; `close` shuts down a browser it launched itself. */
|
|
25
|
+
export interface PlaywrightRenderer extends PageRenderer {
|
|
26
|
+
close: () => Promise<void>;
|
|
27
|
+
}
|
|
28
|
+
/** What the in-page probe returns. */
|
|
29
|
+
export interface ProbeResult {
|
|
30
|
+
styles: StyleSample[];
|
|
31
|
+
variables: Record<string, string>;
|
|
32
|
+
blocks: RenderedBlock[];
|
|
33
|
+
viewport: {
|
|
34
|
+
width: number;
|
|
35
|
+
height: number;
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
export interface PlaywrightFetchPluginOptions {
|
|
39
|
+
/** Render this request? Default: every page request (`purpose: 'page'`). */
|
|
40
|
+
when?: (request: FetchRequest) => boolean;
|
|
41
|
+
}
|
|
42
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/renderer/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAA;AACxD,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,YAAY,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAA;AAE/G,MAAM,WAAW,yBAAyB;IACxC,iGAAiG;IACjG,OAAO,CAAC,EAAE,OAAO,GAAG,CAAC,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC,CAAA;IAC5C,MAAM,CAAC,EAAE,aAAa,CAAA;IACtB,QAAQ,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAA;IAC5C,+BAA+B;IAC/B,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,kFAAkF;IAClF,SAAS,CAAC,EAAE,MAAM,GAAG,kBAAkB,GAAG,aAAa,GAAG,QAAQ,CAAA;IAClE,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB;;;;OAIG;IACH,OAAO,CAAC,EAAE,WAAW,EAAE,CAAA;CACxB;AAED,6FAA6F;AAC7F,MAAM,WAAW,kBAAmB,SAAQ,YAAY;IACtD,KAAK,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;CAC3B;AAED,sCAAsC;AACtC,MAAM,WAAW,WAAW;IAC1B,MAAM,EAAE,WAAW,EAAE,CAAA;IACrB,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IACjC,MAAM,EAAE,aAAa,EAAE,CAAA;IACvB,QAAQ,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAA;CAC5C;AAED,MAAM,WAAW,4BAA4B;IAC3C,4EAA4E;IAC5E,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,OAAO,CAAA;CAC1C"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/renderer/types.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.local.d.ts","sourceRoot":"","sources":["../../src/renderer/types.local.ts"],"names":[],"mappings":"AAAA,mEAAmE;AACnE,MAAM,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAA"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.local.js","sourceRoot":"","sources":["../../src/renderer/types.local.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { PlaywrightRenderer, PlaywrightRendererOptions } from './renderer/types.js';
|
|
2
|
+
/**
|
|
3
|
+
* Render pages in Chromium (or the browser you pass). Each page gets its own browser context:
|
|
4
|
+
* its proxy is the reader's lease (`endpoint()` — the upstream, or the SOCKS bridge, which
|
|
5
|
+
* challenges with a 407 so Chromium sends the route credentials), every request it makes is put
|
|
6
|
+
* through the reader's address guard, downloads are off and service workers blocked. A page of a
|
|
7
|
+
* local bundle (`bundles`, or the request's own `bundle`) is served from disk under its synthetic
|
|
8
|
+
* origin, while everything it loads from elsewhere still passes the guard.
|
|
9
|
+
*/
|
|
10
|
+
export declare const createPlaywrightRenderer: (options?: PlaywrightRendererOptions) => PlaywrightRenderer;
|
|
11
|
+
//# sourceMappingURL=renderer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"renderer.d.ts","sourceRoot":"","sources":["../src/renderer.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,kBAAkB,EAAE,yBAAyB,EAAE,MAAM,qBAAqB,CAAA;AAExF;;;;;;;GAOG;AACH,eAAO,MAAM,wBAAwB,aAAa,yBAAyB,KAAQ,kBAkFlF,CAAA"}
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
import { DEFAULT_RENDER_TIMEOUT_MS, DEFAULT_SETTLE_MS, DEFAULT_VIEWPORT, PROBE_ELEMENTS, SCREENSHOT_QUALITY } from './consts.js';
|
|
2
|
+
import { probePage } from './renderer/probe.js';
|
|
3
|
+
/**
|
|
4
|
+
* Render pages in Chromium (or the browser you pass). Each page gets its own browser context:
|
|
5
|
+
* its proxy is the reader's lease (`endpoint()` — the upstream, or the SOCKS bridge, which
|
|
6
|
+
* challenges with a 407 so Chromium sends the route credentials), every request it makes is put
|
|
7
|
+
* through the reader's address guard, downloads are off and service workers blocked. A page of a
|
|
8
|
+
* local bundle (`bundles`, or the request's own `bundle`) is served from disk under its synthetic
|
|
9
|
+
* origin, while everything it loads from elsewhere still passes the guard.
|
|
10
|
+
*/
|
|
11
|
+
export const createPlaywrightRenderer = (options = {}) => {
|
|
12
|
+
let launched;
|
|
13
|
+
const browserOf = async () => {
|
|
14
|
+
if (options.browser != null) {
|
|
15
|
+
return typeof options.browser === 'function' ? options.browser() : options.browser;
|
|
16
|
+
}
|
|
17
|
+
return launched ??= import('playwright').then(({ chromium }) => chromium.launch({ headless: true, ...options.launch }));
|
|
18
|
+
};
|
|
19
|
+
const render = async (request) => {
|
|
20
|
+
const browser = await browserOf();
|
|
21
|
+
const endpoint = request.lease != null ? await request.lease.endpoint() : undefined;
|
|
22
|
+
const viewport = options.viewport ?? DEFAULT_VIEWPORT;
|
|
23
|
+
const context = await browser.newContext({
|
|
24
|
+
viewport,
|
|
25
|
+
serviceWorkers: 'block',
|
|
26
|
+
acceptDownloads: false,
|
|
27
|
+
...(options.userAgent != null ? { userAgent: options.userAgent } : {}),
|
|
28
|
+
...(endpoint != null
|
|
29
|
+
? { proxy: { server: endpoint.server, ...(endpoint.username != null ? { username: endpoint.username, password: endpoint.password ?? '' } : {}) } }
|
|
30
|
+
: {}),
|
|
31
|
+
});
|
|
32
|
+
const blocked = [];
|
|
33
|
+
const bundles = [...(options.bundles ?? []), ...(request.bundle != null ? [request.bundle] : [])];
|
|
34
|
+
try {
|
|
35
|
+
if (request.guard != null || bundles.length > 0) {
|
|
36
|
+
const guard = request.guard;
|
|
37
|
+
await context.route('**/*', async (route) => {
|
|
38
|
+
const url = route.request().url();
|
|
39
|
+
if (url.startsWith('data:') || url.startsWith('blob:'))
|
|
40
|
+
return route.continue();
|
|
41
|
+
// A bundle file is answered from disk; its synthetic origin never reaches DNS or a proxy.
|
|
42
|
+
const bundle = bundles.find(candidate => candidate.serves(url));
|
|
43
|
+
try {
|
|
44
|
+
if (bundle != null) {
|
|
45
|
+
const file = await bundle.read(url);
|
|
46
|
+
await route.fulfill(file != null
|
|
47
|
+
? { status: 200, contentType: file.contentType, body: Buffer.from(file.bytes) }
|
|
48
|
+
: { status: 404, contentType: 'text/plain', body: '' });
|
|
49
|
+
return;
|
|
50
|
+
}
|
|
51
|
+
await guard?.check(url, endpoint == null);
|
|
52
|
+
await route.continue();
|
|
53
|
+
}
|
|
54
|
+
catch {
|
|
55
|
+
blocked.push(url);
|
|
56
|
+
await route.abort('blockedbyclient');
|
|
57
|
+
}
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
const page = await context.newPage();
|
|
61
|
+
request.signal?.addEventListener('abort', () => void page.close().catch(() => undefined), { once: true });
|
|
62
|
+
const response = await page.goto(request.url, {
|
|
63
|
+
waitUntil: options.waitUntil ?? 'load', timeout: request.timeout ?? options.timeout ?? DEFAULT_RENDER_TIMEOUT_MS,
|
|
64
|
+
});
|
|
65
|
+
await page.waitForLoadState('networkidle', { timeout: options.settle ?? DEFAULT_SETTLE_MS }).catch(() => undefined);
|
|
66
|
+
const probe = request.styles === true ? await page.evaluate(probePage, PROBE_ELEMENTS) : undefined;
|
|
67
|
+
const screenshot = request.screenshot === true
|
|
68
|
+
? new Uint8Array(await page.screenshot({ type: 'jpeg', quality: SCREENSHOT_QUALITY }))
|
|
69
|
+
: undefined;
|
|
70
|
+
const rendered = {
|
|
71
|
+
url: request.url,
|
|
72
|
+
finalUrl: page.url(),
|
|
73
|
+
status: response?.status() ?? 0,
|
|
74
|
+
html: await page.content(),
|
|
75
|
+
viewport,
|
|
76
|
+
...(screenshot != null ? { screenshot } : {}),
|
|
77
|
+
...(probe != null ? { styles: probe.styles, variables: probe.variables, blocks: probe.blocks } : {}),
|
|
78
|
+
...(blocked.length > 0 ? { blocked } : {}),
|
|
79
|
+
};
|
|
80
|
+
return rendered;
|
|
81
|
+
}
|
|
82
|
+
finally {
|
|
83
|
+
await context.close().catch(() => undefined);
|
|
84
|
+
}
|
|
85
|
+
};
|
|
86
|
+
const close = async () => {
|
|
87
|
+
const browser = launched;
|
|
88
|
+
launched = undefined;
|
|
89
|
+
if (browser != null)
|
|
90
|
+
await (await browser).close();
|
|
91
|
+
};
|
|
92
|
+
return { render, close };
|
|
93
|
+
};
|
|
94
|
+
//# sourceMappingURL=renderer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"renderer.js","sourceRoot":"","sources":["../src/renderer.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,yBAAyB,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,cAAc,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAA;AAChI,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAA;AAG/C;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,CAAC,OAAO,GAA8B,EAAE,EAAsB,EAAE;IACtG,IAAI,QAAsC,CAAA;IAE1C,MAAM,SAAS,GAAG,KAAK,IAAsB,EAAE;QAC7C,IAAI,OAAO,CAAC,OAAO,IAAI,IAAI,EAAE,CAAC;YAC5B,OAAO,OAAO,OAAO,CAAC,OAAO,KAAK,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAA;QACpF,CAAC;QACD,OAAO,QAAQ,KAAK,MAAM,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAA;IACzH,CAAC,CAAA;IAED,MAAM,MAAM,GAAiC,KAAK,EAAC,OAAO,EAAC,EAAE;QAC3D,MAAM,OAAO,GAAG,MAAM,SAAS,EAAE,CAAA;QACjC,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,SAAS,CAAA;QACnF,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,gBAAgB,CAAA;QACrD,MAAM,OAAO,GAAmB,MAAM,OAAO,CAAC,UAAU,CAAC;YACvD,QAAQ;YACR,cAAc,EAAE,OAAO;YACvB,eAAe,EAAE,KAAK;YACtB,GAAG,CAAC,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACtE,GAAG,CAAC,QAAQ,IAAI,IAAI;gBAClB,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,GAAG,CAAC,QAAQ,CAAC,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;gBAClJ,CAAC,CAAC,EAAE,CAAC;SACR,CAAC,CAAA;QACF,MAAM,OAAO,GAAa,EAAE,CAAA;QAC5B,MAAM,OAAO,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;QACjG,IAAI,CAAC;YACH,IAAI,OAAO,CAAC,KAAK,IAAI,IAAI,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAChD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAA;gBAC3B,MAAM,OAAO,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,EAAC,KAAK,EAAC,EAAE;oBACxC,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,CAAA;oBACjC,IAAI,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC;wBAAE,OAAO,KAAK,CAAC,QAAQ,EAAE,CAAA;oBAC/E,0FAA0F;oBAC1F,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;oBAC/D,IAAI,CAAC;wBACH,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;4BACnB,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;4BACnC,MAAM,KAAK,CAAC,OAAO,CAAC,IAAI,IAAI,IAAI;gCAC9B,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;gCAC/E,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,WAAW,EAAE,YAAY,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAA;4BACzD,OAAM;wBACR,CAAC;wBACD,MAAM,KAAK,EAAE,KAAK,CAAC,GAAG,EAAE,QAAQ,IAAI,IAAI,CAAC,CAAA;wBACzC,MAAM,KAAK,CAAC,QAAQ,EAAE,CAAA;oBACxB,CAAC;oBAAC,MAAM,CAAC;wBACP,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;wBACjB,MAAM,KAAK,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAA;oBACtC,CAAC;gBACH,CAAC,CAAC,CAAA;YACJ,CAAC;YACD,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAA;YACpC,OAAO,CAAC,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,KAAK,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAA;YACzG,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE;gBAC5C,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,OAAO,IAAI,yBAAyB;aACjH,CAAC,CAAA;YACF,MAAM,IAAI,CAAC,gBAAgB,CAAC,aAAa,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,IAAI,iBAAiB,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAA;YACnH,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC,CAAC,CAAC,SAAS,CAAA;YAClG,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,KAAK,IAAI;gBAC5C,CAAC,CAAC,IAAI,UAAU,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,kBAAkB,EAAE,CAAC,CAAC;gBACtF,CAAC,CAAC,SAAS,CAAA;YACb,MAAM,QAAQ,GAAiB;gBAC7B,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE;gBACpB,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC;gBAC/B,IAAI,EAAE,MAAM,IAAI,CAAC,OAAO,EAAE;gBAC1B,QAAQ;gBACR,GAAG,CAAC,UAAU,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC7C,GAAG,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACpG,GAAG,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC3C,CAAA;YACD,OAAO,QAAQ,CAAA;QACjB,CAAC;gBAAS,CAAC;YACT,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAA;QAC9C,CAAC;IACH,CAAC,CAAA;IAED,MAAM,KAAK,GAAG,KAAK,IAAmB,EAAE;QACtC,MAAM,OAAO,GAAG,QAAQ,CAAA;QACxB,QAAQ,GAAG,SAAS,CAAA;QACpB,IAAI,OAAO,IAAI,IAAI;YAAE,MAAM,CAAC,MAAM,OAAO,CAAC,CAAC,KAAK,EAAE,CAAA;IACpD,CAAC,CAAA;IAED,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAA;AAC1B,CAAC,CAAA"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,55 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@owlmeans/llm-reader-playwright",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "0.1.22-rc.0",
|
|
4
|
+
"license": "MIT",
|
|
5
|
+
"homepage": "https://owlmeans.com/docs/",
|
|
6
|
+
"repository": {
|
|
7
|
+
"type": "git",
|
|
8
|
+
"url": "git+https://github.com/owlmeans/common.git",
|
|
9
|
+
"directory": "packages/llm-reader-playwright"
|
|
10
|
+
},
|
|
11
|
+
"bugs": {
|
|
12
|
+
"url": "https://github.com/owlmeans/common/issues"
|
|
13
|
+
},
|
|
14
|
+
"author": "Igor Tkachenko <igor.tkachenko@owlmeans.com> (https://owlmeans.com)",
|
|
15
|
+
"type": "module",
|
|
16
|
+
"scripts": {
|
|
17
|
+
"build": "tsc -b",
|
|
18
|
+
"dev": "sleep 184 && nodemon -e ts,tsx,json --watch src --exec \"tsc -p ./tsconfig.json\"",
|
|
19
|
+
"watch": "tsc -b -w --preserveWatchOutput --pretty",
|
|
20
|
+
"test": "bun test ./tests"
|
|
21
|
+
},
|
|
22
|
+
"main": "build/index.js",
|
|
23
|
+
"module": "build/index.js",
|
|
24
|
+
"types": "build/index.d.ts",
|
|
25
|
+
"exports": {
|
|
26
|
+
".": {
|
|
27
|
+
"import": "./build/index.js",
|
|
28
|
+
"require": "./build/index.js",
|
|
29
|
+
"default": "./build/index.js",
|
|
30
|
+
"module": "./build/index.js",
|
|
31
|
+
"types": "./build/index.d.ts"
|
|
32
|
+
}
|
|
33
|
+
},
|
|
34
|
+
"dependencies": {
|
|
35
|
+
"@owlmeans/error": "^0.1.21",
|
|
36
|
+
"@owlmeans/llm-reader": "^0.1.22-rc.0",
|
|
37
|
+
"@owlmeans/log": "^0.1.21"
|
|
38
|
+
},
|
|
39
|
+
"devDependencies": {
|
|
40
|
+
"@owlmeans/dep-config": "^0.1.21",
|
|
41
|
+
"@owlmeans/test": "^0.1.22-rc.0",
|
|
42
|
+
"@owlmeans/test-ui": "^0.1.22-rc.0",
|
|
43
|
+
"@types/bun": "^1.4.4",
|
|
44
|
+
"@types/node": "^26.6.5",
|
|
45
|
+
"nodemon": "^3.1.14",
|
|
46
|
+
"playwright": "^1.64.1",
|
|
47
|
+
"typescript": "^7.0.2"
|
|
48
|
+
},
|
|
49
|
+
"peerDependencies": {
|
|
50
|
+
"playwright": "^1.64.1"
|
|
51
|
+
},
|
|
52
|
+
"publishConfig": {
|
|
53
|
+
"access": "public"
|
|
54
|
+
}
|
|
55
|
+
}
|
package/src/consts.ts
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/** The viewport pages are rendered and screenshot in. */
|
|
2
|
+
export const DEFAULT_VIEWPORT = Object.freeze({ width: 1280, height: 800 })
|
|
3
|
+
|
|
4
|
+
/** Navigation deadline, ms. */
|
|
5
|
+
export const DEFAULT_RENDER_TIMEOUT_MS = 30_000
|
|
6
|
+
|
|
7
|
+
/** How long a loaded page may keep fetching before it is read anyway, ms. */
|
|
8
|
+
export const DEFAULT_SETTLE_MS = 3_000
|
|
9
|
+
|
|
10
|
+
/** JPEG quality of the viewport screenshot. */
|
|
11
|
+
export const SCREENSHOT_QUALITY = 70
|
|
12
|
+
|
|
13
|
+
/** Elements the style probe samples, at most. */
|
|
14
|
+
export const PROBE_ELEMENTS = 4000
|
package/src/index.ts
ADDED
package/src/plugin.ts
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import type { ReaderFetchPlugin } from '@owlmeans/llm-reader'
|
|
2
|
+
import type { PlaywrightFetchPluginOptions, PlaywrightRenderer } from './renderer/types.js'
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* A reader fetch plugin that renders pages instead of downloading them — for a site whose every
|
|
6
|
+
* page needs its scripts. Put it after the proxy plugin, so the lease it renders through is chosen
|
|
7
|
+
* (and failed over) by the proxy plugin like any fetch.
|
|
8
|
+
*/
|
|
9
|
+
export const makePlaywrightFetchPlugin = (renderer: PlaywrightRenderer, options: PlaywrightFetchPluginOptions = {}): ReaderFetchPlugin => {
|
|
10
|
+
let guard: Parameters<NonNullable<ReaderFetchPlugin['attach']>>[0]['guard'] | undefined
|
|
11
|
+
|
|
12
|
+
return {
|
|
13
|
+
name: 'playwright',
|
|
14
|
+
attach: context => {
|
|
15
|
+
guard = context.guard
|
|
16
|
+
},
|
|
17
|
+
fetch: async (request, next) => {
|
|
18
|
+
const wanted = options.when != null ? options.when(request) : (request.purpose ?? 'page') === 'page'
|
|
19
|
+
if (!wanted) return next(request)
|
|
20
|
+
const rendered = await renderer.render({
|
|
21
|
+
url: request.url, styles: true, screenshot: true,
|
|
22
|
+
...(guard != null ? { guard } : {}),
|
|
23
|
+
...(request.lease != null ? { lease: request.lease } : {}),
|
|
24
|
+
...(request.signal != null ? { signal: request.signal } : {}),
|
|
25
|
+
...(request.timeout != null ? { timeout: request.timeout } : {}),
|
|
26
|
+
})
|
|
27
|
+
return {
|
|
28
|
+
url: request.url,
|
|
29
|
+
finalUrl: rendered.finalUrl,
|
|
30
|
+
status: rendered.status,
|
|
31
|
+
headers: {},
|
|
32
|
+
contentType: 'text/html',
|
|
33
|
+
charset: 'utf-8',
|
|
34
|
+
text: rendered.html,
|
|
35
|
+
fetchedAt: new Date().toISOString(),
|
|
36
|
+
redirects: [],
|
|
37
|
+
...(request.lease != null ? { via: request.lease.id } : {}),
|
|
38
|
+
rendered,
|
|
39
|
+
}
|
|
40
|
+
},
|
|
41
|
+
close: () => renderer.close(),
|
|
42
|
+
}
|
|
43
|
+
}
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
import type { ProbeResult } from './types.js'
|
|
2
|
+
import type { Tally } from './types.local.js'
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Runs INSIDE the rendered page (serialized by Playwright — it may use nothing from this module's
|
|
6
|
+
* scope). Samples the computed styles of visible elements by role, weighted by on-screen area, and
|
|
7
|
+
* reads the `:root` custom properties and the large blocks of the layout.
|
|
8
|
+
*/
|
|
9
|
+
export const probePage = (limit: number): ProbeResult => {
|
|
10
|
+
const roleOf = (element: Element): string | undefined => {
|
|
11
|
+
const tag = element.tagName.toLowerCase()
|
|
12
|
+
const cls = typeof element.className === 'string' ? element.className.toLowerCase() : ''
|
|
13
|
+
if (tag === 'body') return 'body'
|
|
14
|
+
if (/^h[1-6]$/.test(tag)) return `heading-${tag[1]}`
|
|
15
|
+
if (tag === 'button' || element.getAttribute('role') === 'button' || /\b(btn|button)\b/.test(cls)) return 'button'
|
|
16
|
+
if (tag === 'a') return 'link'
|
|
17
|
+
if (tag === 'input' || tag === 'textarea' || tag === 'select') return 'input'
|
|
18
|
+
if (/\bcard\b/.test(cls)) return 'card'
|
|
19
|
+
if (tag === 'nav' || tag === 'header' || tag === 'footer') return tag
|
|
20
|
+
const ownText = Array.from(element.childNodes).some(node => node.nodeType === 3 && (node.textContent ?? '').trim().length > 0)
|
|
21
|
+
return ownText ? 'text' : undefined
|
|
22
|
+
}
|
|
23
|
+
const roles = new Map<string, { count: number, area: number, values: Tally }>()
|
|
24
|
+
const vote = (tally: Tally, property: string, value: string | undefined, weight: number): void => {
|
|
25
|
+
if (value == null || value === '' || value === 'none' || value === 'normal' || value === 'rgba(0, 0, 0, 0)') return
|
|
26
|
+
const votes = tally[property] ??= {}
|
|
27
|
+
votes[value] = (votes[value] ?? 0) + weight
|
|
28
|
+
}
|
|
29
|
+
let total = 0
|
|
30
|
+
const viewport = { width: window.innerWidth, height: window.innerHeight }
|
|
31
|
+
const elements = Array.from(document.querySelectorAll('body, body *')).slice(0, limit)
|
|
32
|
+
for (const element of elements) {
|
|
33
|
+
const rect = element.getBoundingClientRect()
|
|
34
|
+
const area = Math.min(rect.width, viewport.width) * Math.min(rect.height, viewport.height * 3)
|
|
35
|
+
if (area <= 0) continue
|
|
36
|
+
const style = getComputedStyle(element)
|
|
37
|
+
if (style.visibility === 'hidden' || style.display === 'none' || style.opacity === '0') continue
|
|
38
|
+
const role = roleOf(element)
|
|
39
|
+
if (role == null) continue
|
|
40
|
+
const entry = roles.get(role) ?? { count: 0, area: 0, values: {} }
|
|
41
|
+
entry.count++
|
|
42
|
+
entry.area += area
|
|
43
|
+
total += area
|
|
44
|
+
const weight = role === 'body' ? 1 : area
|
|
45
|
+
vote(entry.values, 'color', style.color, weight)
|
|
46
|
+
vote(entry.values, 'background', style.backgroundColor, weight)
|
|
47
|
+
if (Number.parseFloat(style.borderTopWidth) > 0) vote(entry.values, 'borderColor', style.borderTopColor, weight)
|
|
48
|
+
vote(entry.values, 'fontFamily', style.fontFamily, weight)
|
|
49
|
+
vote(entry.values, 'fontSize', style.fontSize, weight)
|
|
50
|
+
vote(entry.values, 'fontWeight', style.fontWeight, weight)
|
|
51
|
+
vote(entry.values, 'lineHeight', style.lineHeight, weight)
|
|
52
|
+
vote(entry.values, 'letterSpacing', style.letterSpacing, weight)
|
|
53
|
+
if (style.borderTopLeftRadius !== '0px') vote(entry.values, 'radius', style.borderTopLeftRadius, weight)
|
|
54
|
+
vote(entry.values, 'shadow', style.boxShadow, weight)
|
|
55
|
+
if (style.padding !== '0px') vote(entry.values, 'padding', style.padding, weight)
|
|
56
|
+
if (style.gap !== 'normal' && style.gap !== '0px') vote(entry.values, 'gap', style.gap, weight)
|
|
57
|
+
roles.set(role, entry)
|
|
58
|
+
}
|
|
59
|
+
const top = (votes: Record<string, number> | undefined): string | undefined =>
|
|
60
|
+
votes == null ? undefined : Object.entries(votes).sort((a, b) => b[1] - a[1])[0]?.[0]
|
|
61
|
+
const styles = [...roles.entries()].map(([role, entry]) => {
|
|
62
|
+
const sample: Record<string, string | number> = { role, count: entry.count, weight: total > 0 ? entry.area / total : 0 }
|
|
63
|
+
for (const property of ['color', 'background', 'borderColor', 'fontFamily', 'fontSize', 'fontWeight', 'lineHeight', 'letterSpacing', 'radius', 'shadow', 'padding', 'gap']) {
|
|
64
|
+
const value = top(entry.values[property])
|
|
65
|
+
if (value != null) sample[property] = value
|
|
66
|
+
}
|
|
67
|
+
return sample as unknown as ProbeResult['styles'][number]
|
|
68
|
+
})
|
|
69
|
+
|
|
70
|
+
const variables: Record<string, string> = {}
|
|
71
|
+
const root = getComputedStyle(document.documentElement)
|
|
72
|
+
for (const sheet of Array.from(document.styleSheets)) {
|
|
73
|
+
let rules: CSSRuleList
|
|
74
|
+
try {
|
|
75
|
+
rules = sheet.cssRules
|
|
76
|
+
} catch {
|
|
77
|
+
continue // a cross-origin sheet without CORS cannot be read
|
|
78
|
+
}
|
|
79
|
+
for (const rule of Array.from(rules)) {
|
|
80
|
+
const styleRule = rule as CSSStyleRule
|
|
81
|
+
if (styleRule.selectorText == null || !/(^|,)\s*(:root|html)\s*(,|$)/.test(styleRule.selectorText)) continue
|
|
82
|
+
for (let i = 0; i < styleRule.style.length; i++) {
|
|
83
|
+
const name = styleRule.style[i]!
|
|
84
|
+
if (name.startsWith('--')) variables[name] = root.getPropertyValue(name).trim()
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
const container = document.querySelector('main') ?? document.body
|
|
90
|
+
const blocks = Array.from(document.body.children).concat(container !== document.body ? Array.from(container.children) : [])
|
|
91
|
+
.map(element => ({ element, rect: element.getBoundingClientRect() }))
|
|
92
|
+
.filter(({ rect }) => rect.height >= 80 && rect.width >= viewport.width * 0.3)
|
|
93
|
+
.slice(0, 30)
|
|
94
|
+
.map(({ element, rect }) => ({
|
|
95
|
+
tag: element.tagName.toLowerCase(),
|
|
96
|
+
label: element.id || (typeof element.className === 'string' ? element.className.split(/\s+/)[0] : '') || undefined,
|
|
97
|
+
x: Math.round(rect.x), y: Math.round(rect.y + window.scrollY), width: Math.round(rect.width), height: Math.round(rect.height),
|
|
98
|
+
background: getComputedStyle(element).backgroundColor,
|
|
99
|
+
text: ((element as HTMLElement).innerText ?? '').replace(/\s+/g, ' ').trim().slice(0, 100),
|
|
100
|
+
}))
|
|
101
|
+
|
|
102
|
+
return { styles, variables, blocks, viewport }
|
|
103
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import type { Browser, LaunchOptions } from 'playwright'
|
|
2
|
+
import type { FetchRequest, LocalBundle, PageRenderer, RenderedBlock, StyleSample } from '@owlmeans/llm-reader'
|
|
3
|
+
|
|
4
|
+
export interface PlaywrightRendererOptions {
|
|
5
|
+
/** Your own browser, or how to get one; otherwise Chromium is launched headless on first use. */
|
|
6
|
+
browser?: Browser | (() => Promise<Browser>)
|
|
7
|
+
launch?: LaunchOptions
|
|
8
|
+
viewport?: { width: number, height: number }
|
|
9
|
+
/** Navigation deadline, ms. */
|
|
10
|
+
timeout?: number
|
|
11
|
+
/** `load` by default; the page then gets up to `settle` ms to go network-idle. */
|
|
12
|
+
waitUntil?: 'load' | 'domcontentloaded' | 'networkidle' | 'commit'
|
|
13
|
+
settle?: number
|
|
14
|
+
userAgent?: string
|
|
15
|
+
/**
|
|
16
|
+
* Local bundles (`createLocalBundlePlugin`) whose synthetic origins every page may load from: a
|
|
17
|
+
* request under one is answered from its root (`page.route`), never resolved or sent. A render
|
|
18
|
+
* request's own `bundle` — set by the reader for a bundle URL — is routed the same way.
|
|
19
|
+
*/
|
|
20
|
+
bundles?: LocalBundle[]
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/** A {@link PageRenderer} on Playwright; `close` shuts down a browser it launched itself. */
|
|
24
|
+
export interface PlaywrightRenderer extends PageRenderer {
|
|
25
|
+
close: () => Promise<void>
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** What the in-page probe returns. */
|
|
29
|
+
export interface ProbeResult {
|
|
30
|
+
styles: StyleSample[]
|
|
31
|
+
variables: Record<string, string>
|
|
32
|
+
blocks: RenderedBlock[]
|
|
33
|
+
viewport: { width: number, height: number }
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export interface PlaywrightFetchPluginOptions {
|
|
37
|
+
/** Render this request? Default: every page request (`purpose: 'page'`). */
|
|
38
|
+
when?: (request: FetchRequest) => boolean
|
|
39
|
+
}
|
package/src/renderer.ts
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
import type { Browser, BrowserContext } from 'playwright'
|
|
2
|
+
import type { RenderedPage } from '@owlmeans/llm-reader'
|
|
3
|
+
import { DEFAULT_RENDER_TIMEOUT_MS, DEFAULT_SETTLE_MS, DEFAULT_VIEWPORT, PROBE_ELEMENTS, SCREENSHOT_QUALITY } from './consts.js'
|
|
4
|
+
import { probePage } from './renderer/probe.js'
|
|
5
|
+
import type { PlaywrightRenderer, PlaywrightRendererOptions } from './renderer/types.js'
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Render pages in Chromium (or the browser you pass). Each page gets its own browser context:
|
|
9
|
+
* its proxy is the reader's lease (`endpoint()` — the upstream, or the SOCKS bridge, which
|
|
10
|
+
* challenges with a 407 so Chromium sends the route credentials), every request it makes is put
|
|
11
|
+
* through the reader's address guard, downloads are off and service workers blocked. A page of a
|
|
12
|
+
* local bundle (`bundles`, or the request's own `bundle`) is served from disk under its synthetic
|
|
13
|
+
* origin, while everything it loads from elsewhere still passes the guard.
|
|
14
|
+
*/
|
|
15
|
+
export const createPlaywrightRenderer = (options: PlaywrightRendererOptions = {}): PlaywrightRenderer => {
|
|
16
|
+
let launched: Promise<Browser> | undefined
|
|
17
|
+
|
|
18
|
+
const browserOf = async (): Promise<Browser> => {
|
|
19
|
+
if (options.browser != null) {
|
|
20
|
+
return typeof options.browser === 'function' ? options.browser() : options.browser
|
|
21
|
+
}
|
|
22
|
+
return launched ??= import('playwright').then(({ chromium }) => chromium.launch({ headless: true, ...options.launch }))
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
const render: PlaywrightRenderer['render'] = async request => {
|
|
26
|
+
const browser = await browserOf()
|
|
27
|
+
const endpoint = request.lease != null ? await request.lease.endpoint() : undefined
|
|
28
|
+
const viewport = options.viewport ?? DEFAULT_VIEWPORT
|
|
29
|
+
const context: BrowserContext = await browser.newContext({
|
|
30
|
+
viewport,
|
|
31
|
+
serviceWorkers: 'block',
|
|
32
|
+
acceptDownloads: false,
|
|
33
|
+
...(options.userAgent != null ? { userAgent: options.userAgent } : {}),
|
|
34
|
+
...(endpoint != null
|
|
35
|
+
? { proxy: { server: endpoint.server, ...(endpoint.username != null ? { username: endpoint.username, password: endpoint.password ?? '' } : {}) } }
|
|
36
|
+
: {}),
|
|
37
|
+
})
|
|
38
|
+
const blocked: string[] = []
|
|
39
|
+
const bundles = [...(options.bundles ?? []), ...(request.bundle != null ? [request.bundle] : [])]
|
|
40
|
+
try {
|
|
41
|
+
if (request.guard != null || bundles.length > 0) {
|
|
42
|
+
const guard = request.guard
|
|
43
|
+
await context.route('**/*', async route => {
|
|
44
|
+
const url = route.request().url()
|
|
45
|
+
if (url.startsWith('data:') || url.startsWith('blob:')) return route.continue()
|
|
46
|
+
// A bundle file is answered from disk; its synthetic origin never reaches DNS or a proxy.
|
|
47
|
+
const bundle = bundles.find(candidate => candidate.serves(url))
|
|
48
|
+
try {
|
|
49
|
+
if (bundle != null) {
|
|
50
|
+
const file = await bundle.read(url)
|
|
51
|
+
await route.fulfill(file != null
|
|
52
|
+
? { status: 200, contentType: file.contentType, body: Buffer.from(file.bytes) }
|
|
53
|
+
: { status: 404, contentType: 'text/plain', body: '' })
|
|
54
|
+
return
|
|
55
|
+
}
|
|
56
|
+
await guard?.check(url, endpoint == null)
|
|
57
|
+
await route.continue()
|
|
58
|
+
} catch {
|
|
59
|
+
blocked.push(url)
|
|
60
|
+
await route.abort('blockedbyclient')
|
|
61
|
+
}
|
|
62
|
+
})
|
|
63
|
+
}
|
|
64
|
+
const page = await context.newPage()
|
|
65
|
+
request.signal?.addEventListener('abort', () => void page.close().catch(() => undefined), { once: true })
|
|
66
|
+
const response = await page.goto(request.url, {
|
|
67
|
+
waitUntil: options.waitUntil ?? 'load', timeout: request.timeout ?? options.timeout ?? DEFAULT_RENDER_TIMEOUT_MS,
|
|
68
|
+
})
|
|
69
|
+
await page.waitForLoadState('networkidle', { timeout: options.settle ?? DEFAULT_SETTLE_MS }).catch(() => undefined)
|
|
70
|
+
const probe = request.styles === true ? await page.evaluate(probePage, PROBE_ELEMENTS) : undefined
|
|
71
|
+
const screenshot = request.screenshot === true
|
|
72
|
+
? new Uint8Array(await page.screenshot({ type: 'jpeg', quality: SCREENSHOT_QUALITY }))
|
|
73
|
+
: undefined
|
|
74
|
+
const rendered: RenderedPage = {
|
|
75
|
+
url: request.url,
|
|
76
|
+
finalUrl: page.url(),
|
|
77
|
+
status: response?.status() ?? 0,
|
|
78
|
+
html: await page.content(),
|
|
79
|
+
viewport,
|
|
80
|
+
...(screenshot != null ? { screenshot } : {}),
|
|
81
|
+
...(probe != null ? { styles: probe.styles, variables: probe.variables, blocks: probe.blocks } : {}),
|
|
82
|
+
...(blocked.length > 0 ? { blocked } : {}),
|
|
83
|
+
}
|
|
84
|
+
return rendered
|
|
85
|
+
} finally {
|
|
86
|
+
await context.close().catch(() => undefined)
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
const close = async (): Promise<void> => {
|
|
91
|
+
const browser = launched
|
|
92
|
+
launched = undefined
|
|
93
|
+
if (browser != null) await (await browser).close()
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
return { render, close }
|
|
97
|
+
}
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
import { afterAll, describe, expect, test } from 'bun:test'
|
|
2
|
+
import { existsSync } from 'node:fs'
|
|
3
|
+
import { mkdtemp, rm, writeFile } from 'node:fs/promises'
|
|
4
|
+
import { tmpdir } from 'node:os'
|
|
5
|
+
import { join } from 'node:path'
|
|
6
|
+
import { fileURLToPath } from 'node:url'
|
|
7
|
+
import { chromium } from 'playwright'
|
|
8
|
+
import { createBrowserHelper } from '@owlmeans/test-ui'
|
|
9
|
+
import type { LlmModel } from '@owlmeans/llm'
|
|
10
|
+
import { createAddressGuard, createLlmReader, createLocalBundlePlugin, makeDesignAnalyst } from '@owlmeans/llm-reader'
|
|
11
|
+
import type { ReaderModels } from '@owlmeans/llm-reader'
|
|
12
|
+
import { createPlaywrightRenderer } from '../src/index.js'
|
|
13
|
+
// The reader's own fixture site, for a sub-request the guard lets through.
|
|
14
|
+
import { startSite } from '../../llm-reader/tests/servers.js'
|
|
15
|
+
|
|
16
|
+
const skip = existsSync(chromium.executablePath()) ? null : `no Chromium at ${chromium.executablePath()}`
|
|
17
|
+
const titled = (name: string, reason: string | null) => reason != null ? `${name} — SKIPPED: ${reason}` : name
|
|
18
|
+
|
|
19
|
+
const ROOT = fileURLToPath(new URL('../../llm-reader/tests/fixtures/bundle', import.meta.url))
|
|
20
|
+
const site = startSite()
|
|
21
|
+
const browsers = createBrowserHelper()
|
|
22
|
+
const browser = () => browsers.launchBrowser()
|
|
23
|
+
const guard = createAddressGuard({ allowHosts: ['127.0.0.1'] })
|
|
24
|
+
const bundle = createLocalBundlePlugin({ root: ROOT })
|
|
25
|
+
const home = bundle.urlOf('home.html')
|
|
26
|
+
const scratch = await mkdtemp(join(tmpdir(), 'llm-reader-playwright-bundle-'))
|
|
27
|
+
|
|
28
|
+
/** Facts are measured without a model; one that is never called stands in. */
|
|
29
|
+
const unused = new Proxy({}, { get: () => () => { throw new Error('no model call expected') } }) as LlmModel
|
|
30
|
+
const models: ReaderModels = { model: () => unused, helper: () => undefined }
|
|
31
|
+
|
|
32
|
+
afterAll(async () => {
|
|
33
|
+
site.stop()
|
|
34
|
+
await browsers.closeBrowser()
|
|
35
|
+
await rm(scratch, { recursive: true, force: true })
|
|
36
|
+
})
|
|
37
|
+
|
|
38
|
+
describe('@owlmeans/llm-reader-playwright — local bundles', () => {
|
|
39
|
+
test.skipIf(skip != null)(titled('a prototype renders from its bundle: computed styles, variables, a screenshot', skip), async () => {
|
|
40
|
+
const renderer = createPlaywrightRenderer({ browser, bundles: [bundle] })
|
|
41
|
+
const page = await renderer.render({ url: home, guard, styles: true, screenshot: true })
|
|
42
|
+
|
|
43
|
+
expect(page.status).toBe(200)
|
|
44
|
+
expect(page.finalUrl).toBe(home)
|
|
45
|
+
expect(page.html).toContain('Care for the whole family')
|
|
46
|
+
const h1 = page.styles!.find(sample => sample.role === 'heading-1')!
|
|
47
|
+
expect(h1).toMatchObject({ fontSize: '44px', fontWeight: '700' })
|
|
48
|
+
expect(h1.fontFamily).toContain('Merriweather')
|
|
49
|
+
expect(page.variables!['--brand']).toBe('#2f7d5b')
|
|
50
|
+
expect(page.blocks!.length).toBeGreaterThan(0)
|
|
51
|
+
expect([...page.screenshot!.subarray(0, 2)]).toEqual([0xff, 0xd8])
|
|
52
|
+
expect(page.blocked).toBeUndefined()
|
|
53
|
+
}, 60_000)
|
|
54
|
+
|
|
55
|
+
test.skipIf(skip != null)(titled('through the reader: a bundle URL renders with no proxy and no DNS, and design facts merge it', skip), async () => {
|
|
56
|
+
const reader = createLlmReader({ renderer: createPlaywrightRenderer({ browser }), plugins: [bundle], allowHosts: ['127.0.0.1'] })
|
|
57
|
+
const page = await reader.page(home, { render: true })
|
|
58
|
+
|
|
59
|
+
expect(page.rendered?.html).toContain('Book a visit')
|
|
60
|
+
const facts = await makeDesignAnalyst(page, { models, fetcher: reader.fetcher }).facts()
|
|
61
|
+
expect(facts.source).toBe('both')
|
|
62
|
+
expect(facts.variables).toContainEqual({ name: '--brand', value: '#2f7d5b' })
|
|
63
|
+
expect(facts.typeScale.find(type => type.role === 'h1')?.fontSize).toBe('44px')
|
|
64
|
+
}, 60_000)
|
|
65
|
+
|
|
66
|
+
test.skipIf(skip != null)(titled('what a bundle page loads from elsewhere still passes the guard', skip), async () => {
|
|
67
|
+
await writeFile(join(scratch, 'index.html'), [
|
|
68
|
+
'<!doctype html><html><head><meta charset="utf-8"><link rel="stylesheet" href="style.css"></head><body>',
|
|
69
|
+
`<h1>Mixed</h1><img src="${site.url}/img/logo.svg" alt="allowed"><img src="http://169.254.169.254/latest/meta-data" alt="refused">`,
|
|
70
|
+
'</body></html>',
|
|
71
|
+
].join(''))
|
|
72
|
+
await writeFile(join(scratch, 'style.css'), 'h1 { font-size: 30px; }')
|
|
73
|
+
const mixed = createLocalBundlePlugin({ root: scratch, origin: 'https://mixed.local' })
|
|
74
|
+
const page = await createPlaywrightRenderer({ browser }).render({ url: mixed.urlOf(''), guard, bundle: mixed, styles: true })
|
|
75
|
+
|
|
76
|
+
expect(page.styles!.find(sample => sample.role === 'heading-1')!.fontSize).toBe('30px')
|
|
77
|
+
// The allowed host passed the guard (Chromium's own local-network rules may still stop an https
|
|
78
|
+
// page reaching loopback); the cloud-metadata address was refused and reported.
|
|
79
|
+
expect(page.blocked).toEqual(['http://169.254.169.254/latest/meta-data'])
|
|
80
|
+
}, 60_000)
|
|
81
|
+
})
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import { afterAll, describe, expect, test } from 'bun:test'
|
|
2
|
+
import { existsSync } from 'node:fs'
|
|
3
|
+
import { chromium } from 'playwright'
|
|
4
|
+
import { createBrowserHelper } from '@owlmeans/test-ui'
|
|
5
|
+
import { lint } from '@google/design.md/linter'
|
|
6
|
+
import { createAddressGuard, createLlmReader, createProxyPlugin } from '@owlmeans/llm-reader'
|
|
7
|
+
import { createPlaywrightRenderer, makePlaywrightFetchPlugin } from '../src/index.js'
|
|
8
|
+
// The reader's own test servers — a fixture site, an HTTP proxy and a SOCKS server.
|
|
9
|
+
import { startHttpProxy, startSite, startSocksServer } from '../../llm-reader/tests/servers.js'
|
|
10
|
+
import { liveModels, liveSkip } from '../../llm-reader/tests/context.js'
|
|
11
|
+
|
|
12
|
+
const skip = existsSync(chromium.executablePath()) ? null : `no Chromium at ${chromium.executablePath()}`
|
|
13
|
+
const titled = (name: string, reason: string | null) => reason != null ? `${name} — SKIPPED: ${reason}` : name
|
|
14
|
+
|
|
15
|
+
const site = startSite()
|
|
16
|
+
const credentials = { username: 'pw', password: 'right' }
|
|
17
|
+
const httpProxy = await startHttpProxy(credentials)
|
|
18
|
+
const socks = await startSocksServer(credentials)
|
|
19
|
+
const browsers = createBrowserHelper()
|
|
20
|
+
/** Chromium sends loopback targets direct unless told otherwise — the fixture site is on loopback. */
|
|
21
|
+
const browser = () => browsers.launchBrowser({ args: ['--proxy-bypass-list=<-loopback>'] })
|
|
22
|
+
const renderer = createPlaywrightRenderer({ browser })
|
|
23
|
+
const guard = createAddressGuard({ allowHosts: ['127.0.0.1'] })
|
|
24
|
+
|
|
25
|
+
afterAll(async () => {
|
|
26
|
+
site.stop()
|
|
27
|
+
await Promise.all([httpProxy.stop(), socks.stop()])
|
|
28
|
+
await browsers.closeBrowser()
|
|
29
|
+
})
|
|
30
|
+
|
|
31
|
+
describe('@owlmeans/llm-reader-playwright — rendering', () => {
|
|
32
|
+
test.skipIf(skip != null)(titled('an app shell is rendered: content, computed styles, variables, a screenshot', skip), async () => {
|
|
33
|
+
const page = await renderer.render({ url: `${site.url}/app`, guard, styles: true, screenshot: true })
|
|
34
|
+
|
|
35
|
+
expect(page.status).toBe(200)
|
|
36
|
+
expect(page.html).toContain('Revenue overview')
|
|
37
|
+
const h1 = page.styles!.find(sample => sample.role === 'heading-1')!
|
|
38
|
+
expect(h1).toMatchObject({ fontSize: '48px', fontWeight: '700' })
|
|
39
|
+
expect(h1.fontFamily).toContain('Fraunces')
|
|
40
|
+
expect(page.variables!['--color-primary']).toBe('#0b5fff')
|
|
41
|
+
expect([...page.screenshot!.subarray(0, 2)]).toEqual([0xff, 0xd8])
|
|
42
|
+
}, 60_000)
|
|
43
|
+
|
|
44
|
+
test.skipIf(skip != null)(titled('an HTTP proxy with credentials carries the page and its sub-requests', skip), async () => {
|
|
45
|
+
const lease = await createProxyPlugin({ proxies: [httpProxy.url] }).lease({ url: `${site.url}/app` })
|
|
46
|
+
const before = httpProxy.hits.length
|
|
47
|
+
const page = await renderer.render({ url: `${site.url}/app`, guard, lease })
|
|
48
|
+
|
|
49
|
+
expect(page.html).toContain('Revenue overview')
|
|
50
|
+
expect(httpProxy.hits.slice(before)).toContain(`GET ${site.url}/app`)
|
|
51
|
+
}, 60_000)
|
|
52
|
+
|
|
53
|
+
test.skipIf(skip != null)(titled('SOCKS5 with credentials: Chromium goes through the bridge', skip), async () => {
|
|
54
|
+
const plugin = createProxyPlugin({ proxies: [socks.url] })
|
|
55
|
+
const lease = await plugin.lease({ url: `${site.url}/app` })
|
|
56
|
+
const before = socks.hits.length
|
|
57
|
+
const page = await renderer.render({ url: `${site.url}/app`, guard, lease })
|
|
58
|
+
|
|
59
|
+
expect(page.html).toContain('Revenue overview')
|
|
60
|
+
expect(socks.hits.slice(before)).toContain(`CONNECT 127.0.0.1:${new URL(site.url).port}`)
|
|
61
|
+
await plugin.close()
|
|
62
|
+
}, 60_000)
|
|
63
|
+
|
|
64
|
+
test.skipIf(skip != null)(titled('a sub-request into private space is refused and reported; an allowed one is not', skip), async () => {
|
|
65
|
+
const page = await renderer.render({ url: `${site.url}/mixed`, guard })
|
|
66
|
+
|
|
67
|
+
expect(page.blocked).toEqual(['http://169.254.169.254/latest/meta-data'])
|
|
68
|
+
expect(site.hits).toContain('/images/logo.svg')
|
|
69
|
+
}, 60_000)
|
|
70
|
+
|
|
71
|
+
test.skipIf(skip != null)(titled('as a fetch plugin: every page is rendered inside the reader', skip), async () => {
|
|
72
|
+
const reader = createLlmReader({ allowHosts: ['127.0.0.1'], plugins: [makePlaywrightFetchPlugin(createPlaywrightRenderer({ browser }))] })
|
|
73
|
+
const page = await reader.page(`${site.url}/app`)
|
|
74
|
+
expect(page.rendered?.html).toContain('Revenue overview')
|
|
75
|
+
expect(page.signals().appShell).toBe(false)
|
|
76
|
+
}, 60_000)
|
|
77
|
+
|
|
78
|
+
const liveReason = skip ?? liveSkip
|
|
79
|
+
test.skipIf(liveReason != null)(titled('design from computed styles: a DESIGN.md the linter accepts', liveReason), async () => {
|
|
80
|
+
const reader = createLlmReader({ allowHosts: ['127.0.0.1'], renderer, models: liveModels('playwright-live') })
|
|
81
|
+
const report = await reader.design(`${site.url}/`)
|
|
82
|
+
|
|
83
|
+
expect(report.facts.source).toBe('both')
|
|
84
|
+
expect(lint(report.markdown).summary.errors).toBe(0)
|
|
85
|
+
expect(Object.values(report.tokens.colors)).toContain('#0B5FFF')
|
|
86
|
+
}, 300_000)
|
|
87
|
+
})
|
package/tsconfig.json
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
{
|
|
2
|
+
"extends": [
|
|
3
|
+
"@owlmeans/dep-config/tsconfig.base.json",
|
|
4
|
+
"@owlmeans/dep-config/tsconfig.node.json"
|
|
5
|
+
],
|
|
6
|
+
"compilerOptions": {
|
|
7
|
+
"lib": [
|
|
8
|
+
"ESNext",
|
|
9
|
+
"DOM"
|
|
10
|
+
],
|
|
11
|
+
"rootDir": "./src/",
|
|
12
|
+
"outDir": "./build/"
|
|
13
|
+
},
|
|
14
|
+
"include": [
|
|
15
|
+
"src/**/*"
|
|
16
|
+
],
|
|
17
|
+
"exclude": [
|
|
18
|
+
"./dist/**/*",
|
|
19
|
+
"./build/**/*",
|
|
20
|
+
"./tests/**/*",
|
|
21
|
+
"./*.ts"
|
|
22
|
+
]
|
|
23
|
+
}
|