@anweat/dsh-browser 0.1.7 → 0.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +356 -245
- package/cordis.patch.yml +26 -16
- package/lib/approval-policy.js +34 -0
- package/lib/approval-policy.js.map +1 -1
- package/lib/auth-profiles.js +10 -1
- package/lib/auth-profiles.js.map +1 -1
- package/lib/automation-assets-rpc.d.ts +5 -0
- package/lib/automation-assets-rpc.js +63 -0
- package/lib/automation-assets-rpc.js.map +1 -0
- package/lib/automation-assets.d.ts +139 -0
- package/lib/automation-assets.js +372 -0
- package/lib/automation-assets.js.map +1 -0
- package/lib/automation-development.d.ts +13 -0
- package/lib/automation-development.js +37 -0
- package/lib/automation-development.js.map +1 -0
- package/lib/automation-execution.d.ts +14 -0
- package/lib/automation-execution.js +55 -0
- package/lib/automation-execution.js.map +1 -0
- package/lib/browser-service.d.ts +73 -25
- package/lib/browser-service.js +286 -74
- package/lib/browser-service.js.map +1 -1
- package/lib/client/SettingsCard.d.ts +2 -0
- package/lib/client/SettingsCard.js +69 -0
- package/lib/client/SettingsCard.js.map +1 -0
- package/lib/client/automation-assets-client.d.ts +38 -0
- package/lib/client/automation-assets-client.js +83 -0
- package/lib/client/automation-assets-client.js.map +1 -0
- package/lib/client/context-types.d.ts +8 -0
- package/lib/client/context-types.js +2 -0
- package/lib/client/context-types.js.map +1 -0
- package/lib/client/form.d.ts +62 -0
- package/lib/client/form.js +224 -0
- package/lib/client/form.js.map +1 -0
- package/lib/client/index.d.ts +27 -0
- package/lib/client/index.js +28 -0
- package/lib/client/index.js.map +1 -0
- package/lib/client/locales.d.ts +86 -0
- package/lib/client/locales.js +65 -0
- package/lib/client/locales.js.map +1 -0
- package/lib/client/settings-namespace.d.ts +2 -0
- package/lib/client/settings-namespace.js +3 -0
- package/lib/client/settings-namespace.js.map +1 -0
- package/lib/client/styles.d.ts +48 -0
- package/lib/client/styles.js +28 -0
- package/lib/client/styles.js.map +1 -0
- package/lib/client.js +1282 -0
- package/lib/client.js.map +1 -0
- package/lib/config.d.ts +14 -0
- package/lib/config.js +43 -0
- package/lib/config.js.map +1 -1
- package/lib/deps.d.ts +7 -1
- package/lib/deps.js +22 -6
- package/lib/deps.js.map +1 -1
- package/lib/freedom.d.ts +4 -1
- package/lib/freedom.js +14 -0
- package/lib/freedom.js.map +1 -1
- package/lib/index.d.ts +3 -1
- package/lib/index.js +20 -3
- package/lib/index.js.map +1 -1
- package/lib/opencli-catalog.d.ts +21 -0
- package/lib/opencli-catalog.js +49 -0
- package/lib/opencli-catalog.js.map +1 -0
- package/lib/scripts.d.ts +1 -1
- package/lib/scripts.js +30 -29
- package/lib/scripts.js.map +1 -1
- package/lib/tools.d.ts +2 -1
- package/lib/tools.js +222 -11
- package/lib/tools.js.map +1 -1
- package/lib/usage-policy.d.ts +49 -0
- package/lib/usage-policy.js +171 -0
- package/lib/usage-policy.js.map +1 -0
- package/package.json +133 -78
- package/scripts/check-client-bundle.mjs +15 -0
- package/scripts/clean-lib.mjs +3 -0
package/lib/browser-service.js
CHANGED
|
@@ -21,60 +21,75 @@
|
|
|
21
21
|
import fs from 'node:fs';
|
|
22
22
|
import path from 'node:path';
|
|
23
23
|
import crypto from 'node:crypto';
|
|
24
|
-
import {
|
|
24
|
+
import { browserRuntimeCliPath, loadBrowserRuntime, opencliEntryPath, runOpencli, runNode } from "./deps.js";
|
|
25
25
|
import { AuthProfileStore } from "./auth-profiles.js";
|
|
26
26
|
import { applyRuleSteps, resolveRulePack } from "./rule-packs.js";
|
|
27
27
|
import { runRecipe } from "./automation.js";
|
|
28
28
|
import { BUILTIN_SCRIPTS, builtinScript, executeUserscript, validateUserscript } from "./scripts.js";
|
|
29
|
-
import {
|
|
29
|
+
import { configuredBrowserTools } from "./freedom.js";
|
|
30
|
+
import { filterOpencliCatalog, parseOpencliCatalog } from "./opencli-catalog.js";
|
|
31
|
+
import { UsageGovernor } from "./usage-policy.js";
|
|
30
32
|
/** Rules-aware content extractor (runs in the page). */
|
|
31
|
-
const EXTRACTOR_FN = `(ruleList) => {
|
|
32
|
-
const doc = document
|
|
33
|
-
const title = doc.title ? doc.title.trim() : ''
|
|
34
|
-
let host = ''
|
|
35
|
-
try { host = location.hostname } catch (e) {}
|
|
36
|
-
const norm = (h) => { const x = h.toLowerCase(); return x.startsWith('www.') ? x.slice(4) : x }
|
|
37
|
-
let rule = null
|
|
38
|
-
for (const r of ruleList) {
|
|
39
|
-
const rh = norm(r.hostname)
|
|
40
|
-
if (host === rh || host.endsWith('.' + rh)) rule = r
|
|
41
|
-
}
|
|
42
|
-
const pick = (selectors) => {
|
|
43
|
-
for (const sel of selectors) {
|
|
44
|
-
try {
|
|
45
|
-
const el = doc.querySelector(sel)
|
|
46
|
-
if (el && (el.textContent || '').trim().length > 40) return el
|
|
47
|
-
} catch (e) {}
|
|
48
|
-
}
|
|
49
|
-
return null
|
|
50
|
-
}
|
|
51
|
-
const content = rule ? pick(rule.contentSelectors) : (pick(['article', 'main', '[role="main"]']) || doc.body)
|
|
52
|
-
if (content && rule && rule.removeSelectors) {
|
|
53
|
-
for (const sel of rule.removeSelectors) {
|
|
54
|
-
try { content.querySelectorAll(sel).forEach((el) => el.remove()) } catch (e) {}
|
|
55
|
-
}
|
|
56
|
-
}
|
|
57
|
-
const text = content ? (content.innerText || content.textContent || '') : ''
|
|
58
|
-
return { title, text, html: doc.documentElement.outerHTML.slice(0, 2000000), usedRule: rule ? rule.hostname : null }
|
|
33
|
+
const EXTRACTOR_FN = `(ruleList) => {
|
|
34
|
+
const doc = document
|
|
35
|
+
const title = doc.title ? doc.title.trim() : ''
|
|
36
|
+
let host = ''
|
|
37
|
+
try { host = location.hostname } catch (e) {}
|
|
38
|
+
const norm = (h) => { const x = h.toLowerCase(); return x.startsWith('www.') ? x.slice(4) : x }
|
|
39
|
+
let rule = null
|
|
40
|
+
for (const r of ruleList) {
|
|
41
|
+
const rh = norm(r.hostname)
|
|
42
|
+
if (host === rh || host.endsWith('.' + rh)) rule = r
|
|
43
|
+
}
|
|
44
|
+
const pick = (selectors) => {
|
|
45
|
+
for (const sel of selectors) {
|
|
46
|
+
try {
|
|
47
|
+
const el = doc.querySelector(sel)
|
|
48
|
+
if (el && (el.textContent || '').trim().length > 40) return el
|
|
49
|
+
} catch (e) {}
|
|
50
|
+
}
|
|
51
|
+
return null
|
|
52
|
+
}
|
|
53
|
+
const content = rule ? pick(rule.contentSelectors) : (pick(['article', 'main', '[role="main"]']) || doc.body)
|
|
54
|
+
if (content && rule && rule.removeSelectors) {
|
|
55
|
+
for (const sel of rule.removeSelectors) {
|
|
56
|
+
try { content.querySelectorAll(sel).forEach((el) => el.remove()) } catch (e) {}
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
const text = content ? (content.innerText || content.textContent || '') : ''
|
|
60
|
+
return { title, text, html: doc.documentElement.outerHTML.slice(0, 2000000), usedRule: rule ? rule.hostname : null }
|
|
59
61
|
}`;
|
|
60
62
|
/** Platform search-page list extractor (runs in the page). */
|
|
61
|
-
const LIST_EXTRACTOR = `(spec) => {
|
|
62
|
-
const items = []
|
|
63
|
-
const nodes = document.querySelectorAll(spec.item)
|
|
64
|
-
for (let i = 0; i < nodes.length && items.length < 20; i++) {
|
|
65
|
-
const el = nodes[i]
|
|
66
|
-
const titleEl = spec.title ? el.querySelector(spec.title) : null
|
|
67
|
-
const linkEl = spec.link ? el.querySelector(spec.link) : null
|
|
68
|
-
const textEl = spec.text ? el.querySelector(spec.text) : null
|
|
69
|
-
const title = (titleEl ? titleEl.textContent : el.textContent || '').trim().replace(/\\s+/g, ' ')
|
|
70
|
-
let url = linkEl ? (linkEl.href || linkEl.getAttribute('href') || '') : ''
|
|
71
|
-
if (url && url.startsWith('/')) url = location.origin + url
|
|
72
|
-
if (!url && linkEl === null && titleEl) { const a = titleEl.closest ? titleEl.closest('a') : null; if (a) url = a.href }
|
|
73
|
-
const snippet = textEl ? (textEl.textContent || '').trim().replace(/\\s+/g, ' ').slice(0, 300) : ''
|
|
74
|
-
if (!url || !title || title.length < 2) continue
|
|
75
|
-
items.push({ url, title, snippet })
|
|
76
|
-
}
|
|
77
|
-
return items
|
|
63
|
+
const LIST_EXTRACTOR = `(spec) => {
|
|
64
|
+
const items = []
|
|
65
|
+
const nodes = document.querySelectorAll(spec.item)
|
|
66
|
+
for (let i = 0; i < nodes.length && items.length < 20; i++) {
|
|
67
|
+
const el = nodes[i]
|
|
68
|
+
const titleEl = spec.title ? el.querySelector(spec.title) : null
|
|
69
|
+
const linkEl = spec.link ? el.querySelector(spec.link) : null
|
|
70
|
+
const textEl = spec.text ? el.querySelector(spec.text) : null
|
|
71
|
+
const title = (titleEl ? titleEl.textContent : el.textContent || '').trim().replace(/\\s+/g, ' ')
|
|
72
|
+
let url = linkEl ? (linkEl.href || linkEl.getAttribute('href') || '') : ''
|
|
73
|
+
if (url && url.startsWith('/')) url = location.origin + url
|
|
74
|
+
if (!url && linkEl === null && titleEl) { const a = titleEl.closest ? titleEl.closest('a') : null; if (a) url = a.href }
|
|
75
|
+
const snippet = textEl ? (textEl.textContent || '').trim().replace(/\\s+/g, ' ').slice(0, 300) : ''
|
|
76
|
+
if (!url || !title || title.length < 2) continue
|
|
77
|
+
items.push({ url, title, snippet })
|
|
78
|
+
}
|
|
79
|
+
return items
|
|
80
|
+
}`;
|
|
81
|
+
const CRAWL_EXTRACTOR = `(maxChars) => {
|
|
82
|
+
const root = document.querySelector('article, main, [role="main"]') || document.body
|
|
83
|
+
const text = ((root && (root.innerText || root.textContent)) || '').replace(/\\n{3,}/g, '\\n\\n').trim().slice(0, maxChars)
|
|
84
|
+
const links = []
|
|
85
|
+
for (const anchor of document.querySelectorAll('a[href]')) {
|
|
86
|
+
try {
|
|
87
|
+
const url = new URL(anchor.href, location.href)
|
|
88
|
+
if ((url.protocol === 'http:' || url.protocol === 'https:') && !links.includes(url.href)) links.push(url.href)
|
|
89
|
+
if (links.length >= 500) break
|
|
90
|
+
} catch (e) {}
|
|
91
|
+
}
|
|
92
|
+
return { title: document.title || '', text, links }
|
|
78
93
|
}`;
|
|
79
94
|
function uid() {
|
|
80
95
|
return crypto.randomUUID();
|
|
@@ -90,6 +105,18 @@ function specArg(spec) {
|
|
|
90
105
|
function evaluateExtractor(page, rules) {
|
|
91
106
|
return page.evaluate('(' + EXTRACTOR_FN + ')(' + JSON.stringify(rules) + ')');
|
|
92
107
|
}
|
|
108
|
+
function storageStateOptions(statePath, label, allowMissing = false) {
|
|
109
|
+
if (!statePath)
|
|
110
|
+
return {};
|
|
111
|
+
if (!fs.existsSync(statePath)) {
|
|
112
|
+
if (allowMissing)
|
|
113
|
+
return {};
|
|
114
|
+
throw new Error(`dsh-browser: ${label} storageState file does not exist: ${statePath}`);
|
|
115
|
+
}
|
|
116
|
+
if (!fs.statSync(statePath).isFile())
|
|
117
|
+
throw new Error(`dsh-browser: ${label} storageState path is not a file: ${statePath}`);
|
|
118
|
+
return { storageState: statePath };
|
|
119
|
+
}
|
|
93
120
|
export class BrowserService {
|
|
94
121
|
config;
|
|
95
122
|
browser;
|
|
@@ -99,19 +126,27 @@ export class BrowserService {
|
|
|
99
126
|
activeProfile;
|
|
100
127
|
activeRulePack;
|
|
101
128
|
authProfiles;
|
|
129
|
+
usageGovernor;
|
|
130
|
+
opencliCatalogCache;
|
|
102
131
|
constructor(config) {
|
|
103
132
|
this.config = config;
|
|
104
133
|
this.authProfiles = new AuthProfileStore(config.authProfiles);
|
|
134
|
+
this.usageGovernor = new UsageGovernor(config.usagePolicy);
|
|
105
135
|
}
|
|
106
136
|
available() {
|
|
107
137
|
return this.config.enabled;
|
|
108
138
|
}
|
|
139
|
+
assertEnabled() {
|
|
140
|
+
if (!this.config.enabled)
|
|
141
|
+
throw new Error('dsh-browser: browser service is disabled');
|
|
142
|
+
}
|
|
109
143
|
async ensure() {
|
|
144
|
+
this.assertEnabled();
|
|
110
145
|
if (this.browser)
|
|
111
146
|
return this.browser;
|
|
112
147
|
if (!this.launching) {
|
|
113
148
|
this.launching = (async () => {
|
|
114
|
-
const pw =
|
|
149
|
+
const pw = loadBrowserRuntime(this.config.browserRuntime);
|
|
115
150
|
const launchOptions = { headless: this.config.headless };
|
|
116
151
|
if (this.config.channel)
|
|
117
152
|
launchOptions.channel = this.config.channel;
|
|
@@ -128,7 +163,7 @@ export class BrowserService {
|
|
|
128
163
|
this.browser = await pw.chromium.launch(launchOptions);
|
|
129
164
|
}
|
|
130
165
|
else {
|
|
131
|
-
throw new Error('dsh-browser: chromium is not installed. Run the browser_install tool, or: node "' +
|
|
166
|
+
throw new Error('dsh-browser: chromium is not installed for ' + this.config.browserRuntime + '. Run the browser_install tool, or: node "' + browserRuntimeCliPath(this.config.browserRuntime) + '" install chromium');
|
|
132
167
|
}
|
|
133
168
|
}
|
|
134
169
|
else {
|
|
@@ -145,16 +180,37 @@ export class BrowserService {
|
|
|
145
180
|
}
|
|
146
181
|
/** Run `playwright install chromium` from the bundled playwright CLI. */
|
|
147
182
|
installChromium() {
|
|
148
|
-
|
|
183
|
+
this.assertEnabled();
|
|
184
|
+
return runNode(browserRuntimeCliPath(this.config.browserRuntime), ['install', 'chromium'], { timeoutMs: 600_000, signal: undefined, maxOutput: 256 * 1024 });
|
|
185
|
+
}
|
|
186
|
+
async navigate(page, url, options, signal) {
|
|
187
|
+
let response;
|
|
188
|
+
for (let attempt = 0; attempt <= this.config.usagePolicy.retryLimit; attempt++) {
|
|
189
|
+
response = await this.usageGovernor.run(url, () => page.goto(url, options), signal);
|
|
190
|
+
const status = Number(response?.status?.() ?? 0);
|
|
191
|
+
if (status >= 200 && status < 400)
|
|
192
|
+
this.usageGovernor.noteResponse(url, status);
|
|
193
|
+
if (![429, 502, 503, 504].includes(status))
|
|
194
|
+
return response;
|
|
195
|
+
const rawRetryAfter = String(response?.headers?.()?.['retry-after'] ?? '');
|
|
196
|
+
const retryAfterMs = /^\d+(?:\.\d+)?$/.test(rawRetryAfter)
|
|
197
|
+
? Number(rawRetryAfter) * 1000
|
|
198
|
+
: (Number.isFinite(Date.parse(rawRetryAfter)) ? Math.max(Date.parse(rawRetryAfter) - Date.now(), 0) : undefined);
|
|
199
|
+
this.usageGovernor.noteResponse(url, status, retryAfterMs);
|
|
200
|
+
if (attempt === this.config.usagePolicy.retryLimit)
|
|
201
|
+
return response;
|
|
202
|
+
}
|
|
203
|
+
return response;
|
|
149
204
|
}
|
|
150
205
|
async transientContext(url, opts = {}) {
|
|
151
206
|
const browser = await this.ensure();
|
|
152
|
-
const profileId = opts.authProfile ?? this.config.defaultAuthProfile;
|
|
207
|
+
const profileId = opts.anonymous ? undefined : (opts.authProfile ?? this.config.defaultAuthProfile);
|
|
153
208
|
const profile = profileId ? this.authProfiles.resolve(profileId, url) : undefined;
|
|
154
209
|
const rulePack = resolveRulePack(this.config.rulePacks, opts.rulePack, url);
|
|
155
|
-
const
|
|
156
|
-
?
|
|
157
|
-
: (
|
|
210
|
+
const stateOptions = profile
|
|
211
|
+
? storageStateOptions(profile.storageStatePath, `auth profile ${profile.id}`, profile.persistState)
|
|
212
|
+
: (!opts.anonymous ? storageStateOptions(this.config.storageStatePath, 'global') : {});
|
|
213
|
+
const context = await browser.newContext(stateOptions);
|
|
158
214
|
try {
|
|
159
215
|
if (rulePack?.initScriptPath)
|
|
160
216
|
await context.addInitScript({ path: rulePack.initScriptPath });
|
|
@@ -192,7 +248,7 @@ export class BrowserService {
|
|
|
192
248
|
signal?.addEventListener('abort', onAbort);
|
|
193
249
|
try {
|
|
194
250
|
page.setDefaultTimeout(20_000);
|
|
195
|
-
await
|
|
251
|
+
await this.navigate(page, url, { waitUntil: 'domcontentloaded', timeout: 25_000 }, signal);
|
|
196
252
|
await page.waitForLoadState('networkidle', { timeout: 8_000 }).catch(() => { });
|
|
197
253
|
await applyRuleSteps(page, session.rulePack);
|
|
198
254
|
if (opts.waitMs)
|
|
@@ -225,21 +281,22 @@ export class BrowserService {
|
|
|
225
281
|
signal?.addEventListener('abort', onAbort);
|
|
226
282
|
fs.mkdirSync(opts.outDir, { recursive: true });
|
|
227
283
|
const stamp = Date.now() + '-' + uid().slice(0, 8);
|
|
228
|
-
const screenshotPath = path.join(opts.outDir, stamp + '.png');
|
|
284
|
+
const screenshotPath = opts.screenshot === false ? undefined : path.join(opts.outDir, stamp + '.png');
|
|
229
285
|
const htmlPath = path.join(opts.outDir, stamp + '.html');
|
|
230
286
|
try {
|
|
231
287
|
page.setDefaultTimeout(25_000);
|
|
232
|
-
await
|
|
288
|
+
await this.navigate(page, url, { waitUntil: 'domcontentloaded', timeout: 30_000 }, signal);
|
|
233
289
|
await page.waitForLoadState('networkidle', { timeout: 10_000 }).catch(() => { });
|
|
234
290
|
await applyRuleSteps(page, session.rulePack);
|
|
235
|
-
|
|
291
|
+
if (screenshotPath)
|
|
292
|
+
await page.screenshot({ path: screenshotPath, fullPage: true });
|
|
236
293
|
const html = await page.content();
|
|
237
294
|
fs.writeFileSync(htmlPath, html, 'utf8');
|
|
238
295
|
const data = await evaluateExtractor(page, rules);
|
|
239
296
|
return {
|
|
240
297
|
title: String(data.title ?? ''),
|
|
241
298
|
text: capText(String(data.text ?? '').replace(/\n{3,}/g, '\n\n').trim(), opts.maxChars ?? 200_000),
|
|
242
|
-
screenshotPath,
|
|
299
|
+
...screenshotPath ? { screenshotPath } : {},
|
|
243
300
|
htmlPath,
|
|
244
301
|
...data.usedRule ? { usedRule: String(data.usedRule) } : {},
|
|
245
302
|
};
|
|
@@ -266,7 +323,7 @@ export class BrowserService {
|
|
|
266
323
|
signal?.addEventListener('abort', onAbort);
|
|
267
324
|
try {
|
|
268
325
|
page.setDefaultTimeout(25_000);
|
|
269
|
-
await
|
|
326
|
+
await this.navigate(page, url, { waitUntil: 'domcontentloaded', timeout: 30_000 }, signal);
|
|
270
327
|
await page.waitForLoadState('networkidle', { timeout: 8_000 }).catch(() => { });
|
|
271
328
|
await applyRuleSteps(page, session.rulePack);
|
|
272
329
|
if (opts.waitMs)
|
|
@@ -291,19 +348,140 @@ export class BrowserService {
|
|
|
291
348
|
}
|
|
292
349
|
// ── bundled opencli ───────────────────────────────────────────────────────
|
|
293
350
|
opencliAvailable() {
|
|
294
|
-
|
|
351
|
+
if (!this.config.enabled || !this.config.opencliEnabled)
|
|
352
|
+
return false;
|
|
353
|
+
try {
|
|
354
|
+
return fs.existsSync(opencliEntryPath());
|
|
355
|
+
}
|
|
356
|
+
catch {
|
|
357
|
+
return false;
|
|
358
|
+
}
|
|
295
359
|
}
|
|
296
360
|
opencli(args, opts = {}) {
|
|
361
|
+
if (!this.config.enabled)
|
|
362
|
+
return Promise.resolve({ code: -1, stdout: '', stderr: 'dsh-browser: browser service is disabled', timedOut: false });
|
|
297
363
|
if (!this.config.opencliEnabled)
|
|
298
364
|
return Promise.resolve({ code: -1, stdout: '', stderr: 'dsh-browser: OpenCLI is disabled', timedOut: false });
|
|
365
|
+
if (!this.opencliAvailable())
|
|
366
|
+
return Promise.resolve({ code: -1, stdout: '', stderr: 'dsh-browser: OpenCLI entry is not installed', timedOut: false });
|
|
299
367
|
if (args.length < 1 || args.length > 40 || args.some(arg => typeof arg !== 'string' || arg.length > 2_000)) {
|
|
300
368
|
return Promise.resolve({ code: -1, stdout: '', stderr: 'dsh-browser: OpenCLI requires 1 to 40 arguments, each at most 2000 characters', timedOut: false });
|
|
301
369
|
}
|
|
302
|
-
|
|
370
|
+
// OpenCLI adapters can issue real site traffic outside Playwright, so they
|
|
371
|
+
// share the same approval-independent concurrency and burst buffer.
|
|
372
|
+
return this.usageGovernor.run('https://opencli.local/', () => runOpencli(args, { ...opts, signal: opts.signal }), opts.signal);
|
|
303
373
|
}
|
|
304
374
|
opencliDoctor(signal) {
|
|
305
375
|
return this.opencli(['doctor'], { timeoutMs: 30_000, signal });
|
|
306
376
|
}
|
|
377
|
+
async opencliCatalog(filter = {}, signal) {
|
|
378
|
+
if (!this.config.opencliEnabled)
|
|
379
|
+
throw new Error('dsh-browser: OpenCLI is disabled');
|
|
380
|
+
if (!this.opencliCatalogCache) {
|
|
381
|
+
const result = await this.opencli(['list', '-f', 'json'], { timeoutMs: 60_000, signal });
|
|
382
|
+
if (result.code !== 0 || result.timedOut)
|
|
383
|
+
throw new Error('OpenCLI catalog failed: ' + (result.stderr || result.stdout).slice(0, 500));
|
|
384
|
+
this.opencliCatalogCache = parseOpencliCatalog(result.stdout);
|
|
385
|
+
}
|
|
386
|
+
return filterOpencliCatalog(this.opencliCatalogCache, filter);
|
|
387
|
+
}
|
|
388
|
+
async crawl(startUrls, opts = {}) {
|
|
389
|
+
if (startUrls.length < 1 || startUrls.length > 5)
|
|
390
|
+
throw new Error('browser crawl requires 1 to 5 start URLs');
|
|
391
|
+
const normalized = startUrls.map(value => {
|
|
392
|
+
const url = new URL(value);
|
|
393
|
+
if (!['http:', 'https:'].includes(url.protocol))
|
|
394
|
+
throw new Error('browser crawl only supports HTTP(S) URLs');
|
|
395
|
+
url.hash = '';
|
|
396
|
+
return url.href;
|
|
397
|
+
});
|
|
398
|
+
const maxPages = opts.maxPages ?? this.config.usagePolicy.maxPagesPerRun;
|
|
399
|
+
const maxDepth = opts.maxDepth ?? this.config.usagePolicy.maxDepth;
|
|
400
|
+
if (!Number.isInteger(maxPages) || maxPages < 1 || maxPages > this.config.usagePolicy.maxPagesPerRun) {
|
|
401
|
+
throw new Error('browser crawl maxPages must be from 1 to configured usagePolicy.maxPagesPerRun (' + this.config.usagePolicy.maxPagesPerRun + ')');
|
|
402
|
+
}
|
|
403
|
+
if (!Number.isInteger(maxDepth) || maxDepth < 0 || maxDepth > this.config.usagePolicy.maxDepth) {
|
|
404
|
+
throw new Error('browser crawl maxDepth must be from 0 to configured usagePolicy.maxDepth (' + this.config.usagePolicy.maxDepth + ')');
|
|
405
|
+
}
|
|
406
|
+
const maxCharsPerPage = Math.min(Math.max(opts.maxCharsPerPage ?? 20_000, 1_000), 50_000);
|
|
407
|
+
const sameOrigin = opts.sameOrigin ?? true;
|
|
408
|
+
const allowedOrigins = new Set(normalized.map(value => new URL(value).origin));
|
|
409
|
+
const queue = normalized.map(url => ({ url, depth: 0 }));
|
|
410
|
+
const seen = new Set(normalized);
|
|
411
|
+
const pages = [];
|
|
412
|
+
const errors = [];
|
|
413
|
+
const before = this.usageGovernor.snapshot();
|
|
414
|
+
const started = Date.now();
|
|
415
|
+
const session = await this.transientContext(normalized[0], { anonymous: true });
|
|
416
|
+
try {
|
|
417
|
+
while (queue.length && pages.length + errors.length < maxPages) {
|
|
418
|
+
if (opts.signal?.aborted)
|
|
419
|
+
throw new Error('browser crawl aborted');
|
|
420
|
+
const item = queue.shift();
|
|
421
|
+
const page = await session.context.newPage();
|
|
422
|
+
try {
|
|
423
|
+
page.setDefaultTimeout(30_000);
|
|
424
|
+
const response = await this.navigate(page, item.url, { waitUntil: 'domcontentloaded', timeout: 30_000 }, opts.signal);
|
|
425
|
+
const status = Number(response?.status?.() ?? 0);
|
|
426
|
+
if (sameOrigin && !allowedOrigins.has(new URL(page.url()).origin)) {
|
|
427
|
+
errors.push({ url: item.url, depth: item.depth, status, error: 'cross-origin redirect blocked: ' + page.url() });
|
|
428
|
+
continue;
|
|
429
|
+
}
|
|
430
|
+
if (status >= 400) {
|
|
431
|
+
errors.push({ url: item.url, depth: item.depth, status, error: 'HTTP ' + status });
|
|
432
|
+
continue;
|
|
433
|
+
}
|
|
434
|
+
await page.waitForLoadState('networkidle', { timeout: 5_000 }).catch(() => { });
|
|
435
|
+
const data = await page.evaluate('(' + CRAWL_EXTRACTOR + ')(' + maxCharsPerPage + ')');
|
|
436
|
+
pages.push({ url: page.url(), title: String(data.title ?? ''), text: String(data.text ?? ''), depth: item.depth, status });
|
|
437
|
+
if (item.depth >= maxDepth)
|
|
438
|
+
continue;
|
|
439
|
+
for (const rawLink of Array.isArray(data.links) ? data.links : []) {
|
|
440
|
+
let link;
|
|
441
|
+
try {
|
|
442
|
+
link = new URL(rawLink);
|
|
443
|
+
link.hash = '';
|
|
444
|
+
}
|
|
445
|
+
catch {
|
|
446
|
+
continue;
|
|
447
|
+
}
|
|
448
|
+
if (sameOrigin && !allowedOrigins.has(link.origin))
|
|
449
|
+
continue;
|
|
450
|
+
const href = link.href;
|
|
451
|
+
if (seen.has(href) || seen.size >= maxPages * 25)
|
|
452
|
+
continue;
|
|
453
|
+
seen.add(href);
|
|
454
|
+
queue.push({ url: href, depth: item.depth + 1 });
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
catch (error) {
|
|
458
|
+
errors.push({ url: item.url, depth: item.depth, error: String(error).slice(0, 500) });
|
|
459
|
+
}
|
|
460
|
+
finally {
|
|
461
|
+
await page.close().catch(() => { });
|
|
462
|
+
}
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
finally {
|
|
466
|
+
await this.persistAndClose(session);
|
|
467
|
+
}
|
|
468
|
+
const after = this.usageGovernor.snapshot();
|
|
469
|
+
return {
|
|
470
|
+
pages,
|
|
471
|
+
errors,
|
|
472
|
+
stats: {
|
|
473
|
+
pagesVisited: pages.length + errors.length,
|
|
474
|
+
queued: queue.length,
|
|
475
|
+
elapsedMs: Date.now() - started,
|
|
476
|
+
waitMs: after.totalWaitMs - before.totalWaitMs,
|
|
477
|
+
backoffEvents: after.backoffEvents - before.backoffEvents,
|
|
478
|
+
},
|
|
479
|
+
warnings: [
|
|
480
|
+
'Bounded crawl: respect each site\'s terms, robots directives, copyright, privacy, and applicable law.',
|
|
481
|
+
'No-approval mode skips human confirmation only; concurrency, burst, page/depth budgets, and server-pressure backoff remain active.',
|
|
482
|
+
],
|
|
483
|
+
};
|
|
484
|
+
}
|
|
307
485
|
scriptCatalog() {
|
|
308
486
|
return BUILTIN_SCRIPTS.map(script => ({
|
|
309
487
|
id: script.id,
|
|
@@ -330,7 +508,7 @@ export class BrowserService {
|
|
|
330
508
|
let timer;
|
|
331
509
|
try {
|
|
332
510
|
page.setDefaultTimeout(30_000);
|
|
333
|
-
await
|
|
511
|
+
await this.navigate(page, url, { waitUntil: 'domcontentloaded', timeout: 30_000 }, signal);
|
|
334
512
|
await page.waitForLoadState('networkidle', { timeout: 8_000 }).catch(() => { });
|
|
335
513
|
await applyRuleSteps(page, session.rulePack);
|
|
336
514
|
const timeoutMs = Math.min(Math.max(opts.timeoutMs ?? 15_000, 1_000), 30_000);
|
|
@@ -340,7 +518,7 @@ export class BrowserService {
|
|
|
340
518
|
reject(new Error('userscript timed out after ' + timeoutMs + 'ms'));
|
|
341
519
|
}, timeoutMs);
|
|
342
520
|
});
|
|
343
|
-
const executed = await Promise.race([executeUserscript(page, source), timeout]);
|
|
521
|
+
const executed = await Promise.race([executeUserscript(page, source, 100_000, opts.inputs), timeout]);
|
|
344
522
|
return {
|
|
345
523
|
url: page.url(),
|
|
346
524
|
name: validation.metadata.name,
|
|
@@ -385,7 +563,7 @@ export class BrowserService {
|
|
|
385
563
|
}
|
|
386
564
|
else {
|
|
387
565
|
const browser = await this.ensure();
|
|
388
|
-
this.activeContext = await browser.newContext(this.config.storageStatePath
|
|
566
|
+
this.activeContext = await browser.newContext(storageStateOptions(this.config.storageStatePath, 'global'));
|
|
389
567
|
}
|
|
390
568
|
this.activePage = await this.activeContext.newPage();
|
|
391
569
|
return this.activePage;
|
|
@@ -410,7 +588,7 @@ export class BrowserService {
|
|
|
410
588
|
async open(url, opts = {}) {
|
|
411
589
|
const page = await this.ensureActivePage(url, opts);
|
|
412
590
|
page.setDefaultTimeout(30_000);
|
|
413
|
-
await
|
|
591
|
+
await this.navigate(page, url, { waitUntil: 'domcontentloaded', timeout: 30_000 });
|
|
414
592
|
await page.waitForLoadState('networkidle', { timeout: 8_000 }).catch(() => { });
|
|
415
593
|
await applyRuleSteps(page, this.activeRulePack);
|
|
416
594
|
if (opts.waitMs)
|
|
@@ -453,7 +631,7 @@ export class BrowserService {
|
|
|
453
631
|
const page = await this.ensureActivePage(opts.url, opts);
|
|
454
632
|
if (opts.url) {
|
|
455
633
|
page.setDefaultTimeout(30_000);
|
|
456
|
-
await
|
|
634
|
+
await this.navigate(page, opts.url, { waitUntil: 'domcontentloaded', timeout: 30_000 }, opts.signal);
|
|
457
635
|
await page.waitForLoadState('networkidle', { timeout: 8_000 }).catch(() => { });
|
|
458
636
|
await applyRuleSteps(page, this.activeRulePack);
|
|
459
637
|
if (opts.waitMs)
|
|
@@ -485,25 +663,59 @@ export class BrowserService {
|
|
|
485
663
|
}
|
|
486
664
|
async status() {
|
|
487
665
|
let chromiumInstalled = false;
|
|
666
|
+
let chromiumExecutablePath;
|
|
488
667
|
try {
|
|
489
|
-
const pw =
|
|
490
|
-
|
|
668
|
+
const pw = loadBrowserRuntime(this.config.browserRuntime);
|
|
669
|
+
const expectedPath = this.config.executablePath || pw.chromium.executablePath();
|
|
670
|
+
if (typeof expectedPath === 'string' && expectedPath.trim()) {
|
|
671
|
+
chromiumExecutablePath = path.resolve(expectedPath);
|
|
672
|
+
chromiumInstalled = fs.existsSync(chromiumExecutablePath);
|
|
673
|
+
}
|
|
491
674
|
}
|
|
492
675
|
catch {
|
|
493
676
|
chromiumInstalled = false;
|
|
494
677
|
}
|
|
678
|
+
let opencliInstalled = false;
|
|
679
|
+
let resolvedOpencliEntryPath;
|
|
680
|
+
try {
|
|
681
|
+
resolvedOpencliEntryPath = path.resolve(opencliEntryPath());
|
|
682
|
+
opencliInstalled = fs.existsSync(resolvedOpencliEntryPath);
|
|
683
|
+
}
|
|
684
|
+
catch {
|
|
685
|
+
opencliInstalled = false;
|
|
686
|
+
}
|
|
687
|
+
const runtimeWarnings = this.config.browserRuntime === 'patchright'
|
|
688
|
+
? [
|
|
689
|
+
'Patchright is Chromium-only and disables Playwright console APIs to avoid Runtime.enable detection.',
|
|
690
|
+
...(this.config.channel !== 'chrome' || this.config.headless
|
|
691
|
+
? ['Patchright stealth is strongest with channel=chrome and headless=false; current settings favor automation/test compatibility.']
|
|
692
|
+
: []),
|
|
693
|
+
]
|
|
694
|
+
: [];
|
|
695
|
+
if (!chromiumInstalled && chromiumExecutablePath && (this.config.channel === 'chromium' || !!this.config.executablePath)) {
|
|
696
|
+
runtimeWarnings.push(`Expected Chromium executable is missing: ${chromiumExecutablePath}. Run browser_install for ${this.config.browserRuntime}.`);
|
|
697
|
+
}
|
|
698
|
+
if (this.config.opencliEnabled && !opencliInstalled)
|
|
699
|
+
runtimeWarnings.push('OpenCLI is enabled but its package entry is not installed.');
|
|
495
700
|
return {
|
|
496
701
|
enabled: this.config.enabled,
|
|
497
702
|
channel: this.config.channel,
|
|
703
|
+
browserRuntime: this.config.browserRuntime,
|
|
704
|
+
runtimeWarnings,
|
|
498
705
|
headless: this.config.headless,
|
|
499
706
|
opencliEnabled: this.config.opencliEnabled,
|
|
707
|
+
opencliInstalled,
|
|
708
|
+
...resolvedOpencliEntryPath ? { opencliEntryPath: resolvedOpencliEntryPath } : {},
|
|
500
709
|
automationMode: this.config.automationMode,
|
|
501
|
-
exposedTools:
|
|
502
|
-
directInteractionPolicy: this.config.automationMode === 'read-only' ? 'deny' : this.config.automationMode === 'standard' ? 'ask' : 'allow',
|
|
503
|
-
mutatingRecipePolicy: this.config.automationMode === 'read-only' ? 'deny' : this.config.automationMode === 'standard' ? 'ask' : 'allow',
|
|
504
|
-
externalUserscriptPolicy: this.config.automationMode === 'read-only' ? 'deny' : this.config.automationMode === 'unrestricted' ? 'allow' : 'ask',
|
|
505
|
-
opencliRunPolicy: this.config.automationMode === 'read-only' ? 'deny' : this.config.automationMode === 'unrestricted' ? 'allow' : 'ask',
|
|
710
|
+
exposedTools: configuredBrowserTools(this.config.automationMode, this.config.automationAssets, this.config.enabled),
|
|
711
|
+
directInteractionPolicy: !this.config.enabled || this.config.automationMode === 'read-only' ? 'deny' : this.config.automationMode === 'standard' ? 'ask' : 'allow',
|
|
712
|
+
mutatingRecipePolicy: !this.config.enabled || this.config.automationMode === 'read-only' ? 'deny' : this.config.automationMode === 'standard' ? 'ask' : 'allow',
|
|
713
|
+
externalUserscriptPolicy: !this.config.enabled || this.config.automationMode === 'read-only' ? 'deny' : this.config.automationMode === 'unrestricted' ? 'allow' : 'ask',
|
|
714
|
+
opencliRunPolicy: !this.config.enabled || this.config.automationMode === 'read-only' ? 'deny' : this.config.automationMode === 'unrestricted' ? 'allow' : 'ask',
|
|
506
715
|
chromiumInstalled,
|
|
716
|
+
...chromiumExecutablePath ? { chromiumExecutablePath } : {},
|
|
717
|
+
usagePolicy: this.config.usagePolicy,
|
|
718
|
+
usageGovernor: this.usageGovernor.snapshot(),
|
|
507
719
|
authProfiles: this.authProfiles.list(),
|
|
508
720
|
rulePacks: Object.keys(this.config.rulePacks).sort(),
|
|
509
721
|
builtinScripts: BUILTIN_SCRIPTS.map(script => script.id),
|