@anweat/dsh-browser 0.1.7 → 0.1.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/README.md +356 -245
  2. package/cordis.patch.yml +26 -16
  3. package/lib/approval-policy.js +34 -0
  4. package/lib/approval-policy.js.map +1 -1
  5. package/lib/auth-profiles.js +10 -1
  6. package/lib/auth-profiles.js.map +1 -1
  7. package/lib/automation-assets-rpc.d.ts +5 -0
  8. package/lib/automation-assets-rpc.js +63 -0
  9. package/lib/automation-assets-rpc.js.map +1 -0
  10. package/lib/automation-assets.d.ts +139 -0
  11. package/lib/automation-assets.js +372 -0
  12. package/lib/automation-assets.js.map +1 -0
  13. package/lib/automation-development.d.ts +13 -0
  14. package/lib/automation-development.js +37 -0
  15. package/lib/automation-development.js.map +1 -0
  16. package/lib/automation-execution.d.ts +14 -0
  17. package/lib/automation-execution.js +55 -0
  18. package/lib/automation-execution.js.map +1 -0
  19. package/lib/browser-service.d.ts +73 -25
  20. package/lib/browser-service.js +286 -74
  21. package/lib/browser-service.js.map +1 -1
  22. package/lib/client/SettingsCard.d.ts +2 -0
  23. package/lib/client/SettingsCard.js +69 -0
  24. package/lib/client/SettingsCard.js.map +1 -0
  25. package/lib/client/automation-assets-client.d.ts +38 -0
  26. package/lib/client/automation-assets-client.js +83 -0
  27. package/lib/client/automation-assets-client.js.map +1 -0
  28. package/lib/client/context-types.d.ts +8 -0
  29. package/lib/client/context-types.js +2 -0
  30. package/lib/client/context-types.js.map +1 -0
  31. package/lib/client/form.d.ts +62 -0
  32. package/lib/client/form.js +224 -0
  33. package/lib/client/form.js.map +1 -0
  34. package/lib/client/index.d.ts +27 -0
  35. package/lib/client/index.js +28 -0
  36. package/lib/client/index.js.map +1 -0
  37. package/lib/client/locales.d.ts +86 -0
  38. package/lib/client/locales.js +65 -0
  39. package/lib/client/locales.js.map +1 -0
  40. package/lib/client/settings-namespace.d.ts +2 -0
  41. package/lib/client/settings-namespace.js +3 -0
  42. package/lib/client/settings-namespace.js.map +1 -0
  43. package/lib/client/styles.d.ts +48 -0
  44. package/lib/client/styles.js +28 -0
  45. package/lib/client/styles.js.map +1 -0
  46. package/lib/client.js +1282 -0
  47. package/lib/client.js.map +1 -0
  48. package/lib/config.d.ts +14 -0
  49. package/lib/config.js +43 -0
  50. package/lib/config.js.map +1 -1
  51. package/lib/deps.d.ts +7 -1
  52. package/lib/deps.js +22 -6
  53. package/lib/deps.js.map +1 -1
  54. package/lib/freedom.d.ts +4 -1
  55. package/lib/freedom.js +14 -0
  56. package/lib/freedom.js.map +1 -1
  57. package/lib/index.d.ts +3 -1
  58. package/lib/index.js +20 -3
  59. package/lib/index.js.map +1 -1
  60. package/lib/opencli-catalog.d.ts +21 -0
  61. package/lib/opencli-catalog.js +49 -0
  62. package/lib/opencli-catalog.js.map +1 -0
  63. package/lib/scripts.d.ts +1 -1
  64. package/lib/scripts.js +30 -29
  65. package/lib/scripts.js.map +1 -1
  66. package/lib/tools.d.ts +2 -1
  67. package/lib/tools.js +222 -11
  68. package/lib/tools.js.map +1 -1
  69. package/lib/usage-policy.d.ts +49 -0
  70. package/lib/usage-policy.js +171 -0
  71. package/lib/usage-policy.js.map +1 -0
  72. package/package.json +133 -78
  73. package/scripts/check-client-bundle.mjs +15 -0
  74. package/scripts/clean-lib.mjs +3 -0
@@ -21,60 +21,75 @@
21
21
  import fs from 'node:fs';
22
22
  import path from 'node:path';
23
23
  import crypto from 'node:crypto';
24
- import { loadPlaywright, runOpencli, runNode, playwrightCliPath } from "./deps.js";
24
+ import { browserRuntimeCliPath, loadBrowserRuntime, opencliEntryPath, runOpencli, runNode } from "./deps.js";
25
25
  import { AuthProfileStore } from "./auth-profiles.js";
26
26
  import { applyRuleSteps, resolveRulePack } from "./rule-packs.js";
27
27
  import { runRecipe } from "./automation.js";
28
28
  import { BUILTIN_SCRIPTS, builtinScript, executeUserscript, validateUserscript } from "./scripts.js";
29
- import { browserToolsForMode } from "./freedom.js";
29
+ import { configuredBrowserTools } from "./freedom.js";
30
+ import { filterOpencliCatalog, parseOpencliCatalog } from "./opencli-catalog.js";
31
+ import { UsageGovernor } from "./usage-policy.js";
30
32
  /** Rules-aware content extractor (runs in the page). */
31
- const EXTRACTOR_FN = `(ruleList) => {
32
- const doc = document
33
- const title = doc.title ? doc.title.trim() : ''
34
- let host = ''
35
- try { host = location.hostname } catch (e) {}
36
- const norm = (h) => { const x = h.toLowerCase(); return x.startsWith('www.') ? x.slice(4) : x }
37
- let rule = null
38
- for (const r of ruleList) {
39
- const rh = norm(r.hostname)
40
- if (host === rh || host.endsWith('.' + rh)) rule = r
41
- }
42
- const pick = (selectors) => {
43
- for (const sel of selectors) {
44
- try {
45
- const el = doc.querySelector(sel)
46
- if (el && (el.textContent || '').trim().length > 40) return el
47
- } catch (e) {}
48
- }
49
- return null
50
- }
51
- const content = rule ? pick(rule.contentSelectors) : (pick(['article', 'main', '[role="main"]']) || doc.body)
52
- if (content && rule && rule.removeSelectors) {
53
- for (const sel of rule.removeSelectors) {
54
- try { content.querySelectorAll(sel).forEach((el) => el.remove()) } catch (e) {}
55
- }
56
- }
57
- const text = content ? (content.innerText || content.textContent || '') : ''
58
- return { title, text, html: doc.documentElement.outerHTML.slice(0, 2000000), usedRule: rule ? rule.hostname : null }
33
+ const EXTRACTOR_FN = `(ruleList) => {
34
+ const doc = document
35
+ const title = doc.title ? doc.title.trim() : ''
36
+ let host = ''
37
+ try { host = location.hostname } catch (e) {}
38
+ const norm = (h) => { const x = h.toLowerCase(); return x.startsWith('www.') ? x.slice(4) : x }
39
+ let rule = null
40
+ for (const r of ruleList) {
41
+ const rh = norm(r.hostname)
42
+ if (host === rh || host.endsWith('.' + rh)) rule = r
43
+ }
44
+ const pick = (selectors) => {
45
+ for (const sel of selectors) {
46
+ try {
47
+ const el = doc.querySelector(sel)
48
+ if (el && (el.textContent || '').trim().length > 40) return el
49
+ } catch (e) {}
50
+ }
51
+ return null
52
+ }
53
+ const content = rule ? pick(rule.contentSelectors) : (pick(['article', 'main', '[role="main"]']) || doc.body)
54
+ if (content && rule && rule.removeSelectors) {
55
+ for (const sel of rule.removeSelectors) {
56
+ try { content.querySelectorAll(sel).forEach((el) => el.remove()) } catch (e) {}
57
+ }
58
+ }
59
+ const text = content ? (content.innerText || content.textContent || '') : ''
60
+ return { title, text, html: doc.documentElement.outerHTML.slice(0, 2000000), usedRule: rule ? rule.hostname : null }
59
61
  }`;
60
62
  /** Platform search-page list extractor (runs in the page). */
61
- const LIST_EXTRACTOR = `(spec) => {
62
- const items = []
63
- const nodes = document.querySelectorAll(spec.item)
64
- for (let i = 0; i < nodes.length && items.length < 20; i++) {
65
- const el = nodes[i]
66
- const titleEl = spec.title ? el.querySelector(spec.title) : null
67
- const linkEl = spec.link ? el.querySelector(spec.link) : null
68
- const textEl = spec.text ? el.querySelector(spec.text) : null
69
- const title = (titleEl ? titleEl.textContent : el.textContent || '').trim().replace(/\\s+/g, ' ')
70
- let url = linkEl ? (linkEl.href || linkEl.getAttribute('href') || '') : ''
71
- if (url && url.startsWith('/')) url = location.origin + url
72
- if (!url && linkEl === null && titleEl) { const a = titleEl.closest ? titleEl.closest('a') : null; if (a) url = a.href }
73
- const snippet = textEl ? (textEl.textContent || '').trim().replace(/\\s+/g, ' ').slice(0, 300) : ''
74
- if (!url || !title || title.length < 2) continue
75
- items.push({ url, title, snippet })
76
- }
77
- return items
63
+ const LIST_EXTRACTOR = `(spec) => {
64
+ const items = []
65
+ const nodes = document.querySelectorAll(spec.item)
66
+ for (let i = 0; i < nodes.length && items.length < 20; i++) {
67
+ const el = nodes[i]
68
+ const titleEl = spec.title ? el.querySelector(spec.title) : null
69
+ const linkEl = spec.link ? el.querySelector(spec.link) : null
70
+ const textEl = spec.text ? el.querySelector(spec.text) : null
71
+ const title = (titleEl ? titleEl.textContent : el.textContent || '').trim().replace(/\\s+/g, ' ')
72
+ let url = linkEl ? (linkEl.href || linkEl.getAttribute('href') || '') : ''
73
+ if (url && url.startsWith('/')) url = location.origin + url
74
+ if (!url && linkEl === null && titleEl) { const a = titleEl.closest ? titleEl.closest('a') : null; if (a) url = a.href }
75
+ const snippet = textEl ? (textEl.textContent || '').trim().replace(/\\s+/g, ' ').slice(0, 300) : ''
76
+ if (!url || !title || title.length < 2) continue
77
+ items.push({ url, title, snippet })
78
+ }
79
+ return items
80
+ }`;
81
+ const CRAWL_EXTRACTOR = `(maxChars) => {
82
+ const root = document.querySelector('article, main, [role="main"]') || document.body
83
+ const text = ((root && (root.innerText || root.textContent)) || '').replace(/\\n{3,}/g, '\\n\\n').trim().slice(0, maxChars)
84
+ const links = []
85
+ for (const anchor of document.querySelectorAll('a[href]')) {
86
+ try {
87
+ const url = new URL(anchor.href, location.href)
88
+ if ((url.protocol === 'http:' || url.protocol === 'https:') && !links.includes(url.href)) links.push(url.href)
89
+ if (links.length >= 500) break
90
+ } catch (e) {}
91
+ }
92
+ return { title: document.title || '', text, links }
78
93
  }`;
79
94
  function uid() {
80
95
  return crypto.randomUUID();
@@ -90,6 +105,18 @@ function specArg(spec) {
90
105
  function evaluateExtractor(page, rules) {
91
106
  return page.evaluate('(' + EXTRACTOR_FN + ')(' + JSON.stringify(rules) + ')');
92
107
  }
108
+ function storageStateOptions(statePath, label, allowMissing = false) {
109
+ if (!statePath)
110
+ return {};
111
+ if (!fs.existsSync(statePath)) {
112
+ if (allowMissing)
113
+ return {};
114
+ throw new Error(`dsh-browser: ${label} storageState file does not exist: ${statePath}`);
115
+ }
116
+ if (!fs.statSync(statePath).isFile())
117
+ throw new Error(`dsh-browser: ${label} storageState path is not a file: ${statePath}`);
118
+ return { storageState: statePath };
119
+ }
93
120
  export class BrowserService {
94
121
  config;
95
122
  browser;
@@ -99,19 +126,27 @@ export class BrowserService {
99
126
  activeProfile;
100
127
  activeRulePack;
101
128
  authProfiles;
129
+ usageGovernor;
130
+ opencliCatalogCache;
102
131
  constructor(config) {
103
132
  this.config = config;
104
133
  this.authProfiles = new AuthProfileStore(config.authProfiles);
134
+ this.usageGovernor = new UsageGovernor(config.usagePolicy);
105
135
  }
106
136
  available() {
107
137
  return this.config.enabled;
108
138
  }
139
+ assertEnabled() {
140
+ if (!this.config.enabled)
141
+ throw new Error('dsh-browser: browser service is disabled');
142
+ }
109
143
  async ensure() {
144
+ this.assertEnabled();
110
145
  if (this.browser)
111
146
  return this.browser;
112
147
  if (!this.launching) {
113
148
  this.launching = (async () => {
114
- const pw = loadPlaywright();
149
+ const pw = loadBrowserRuntime(this.config.browserRuntime);
115
150
  const launchOptions = { headless: this.config.headless };
116
151
  if (this.config.channel)
117
152
  launchOptions.channel = this.config.channel;
@@ -128,7 +163,7 @@ export class BrowserService {
128
163
  this.browser = await pw.chromium.launch(launchOptions);
129
164
  }
130
165
  else {
131
- throw new Error('dsh-browser: chromium is not installed. Run the browser_install tool, or: node "' + playwrightCliPath() + '" install chromium');
166
+ throw new Error('dsh-browser: chromium is not installed for ' + this.config.browserRuntime + '. Run the browser_install tool, or: node "' + browserRuntimeCliPath(this.config.browserRuntime) + '" install chromium');
132
167
  }
133
168
  }
134
169
  else {
@@ -145,16 +180,37 @@ export class BrowserService {
145
180
  }
146
181
  /** Run `playwright install chromium` from the bundled playwright CLI. */
147
182
  installChromium() {
148
- return runNode(playwrightCliPath(), ['install', 'chromium'], { timeoutMs: 600_000, signal: undefined, maxOutput: 256 * 1024 });
183
+ this.assertEnabled();
184
+ return runNode(browserRuntimeCliPath(this.config.browserRuntime), ['install', 'chromium'], { timeoutMs: 600_000, signal: undefined, maxOutput: 256 * 1024 });
185
+ }
186
+ async navigate(page, url, options, signal) {
187
+ let response;
188
+ for (let attempt = 0; attempt <= this.config.usagePolicy.retryLimit; attempt++) {
189
+ response = await this.usageGovernor.run(url, () => page.goto(url, options), signal);
190
+ const status = Number(response?.status?.() ?? 0);
191
+ if (status >= 200 && status < 400)
192
+ this.usageGovernor.noteResponse(url, status);
193
+ if (![429, 502, 503, 504].includes(status))
194
+ return response;
195
+ const rawRetryAfter = String(response?.headers?.()?.['retry-after'] ?? '');
196
+ const retryAfterMs = /^\d+(?:\.\d+)?$/.test(rawRetryAfter)
197
+ ? Number(rawRetryAfter) * 1000
198
+ : (Number.isFinite(Date.parse(rawRetryAfter)) ? Math.max(Date.parse(rawRetryAfter) - Date.now(), 0) : undefined);
199
+ this.usageGovernor.noteResponse(url, status, retryAfterMs);
200
+ if (attempt === this.config.usagePolicy.retryLimit)
201
+ return response;
202
+ }
203
+ return response;
149
204
  }
150
205
  async transientContext(url, opts = {}) {
151
206
  const browser = await this.ensure();
152
- const profileId = opts.authProfile ?? this.config.defaultAuthProfile;
207
+ const profileId = opts.anonymous ? undefined : (opts.authProfile ?? this.config.defaultAuthProfile);
153
208
  const profile = profileId ? this.authProfiles.resolve(profileId, url) : undefined;
154
209
  const rulePack = resolveRulePack(this.config.rulePacks, opts.rulePack, url);
155
- const context = await browser.newContext(profile?.storageStatePath
156
- ? { storageState: profile.storageStatePath }
157
- : (this.config.storageStatePath ? { storageState: this.config.storageStatePath } : {}));
210
+ const stateOptions = profile
211
+ ? storageStateOptions(profile.storageStatePath, `auth profile ${profile.id}`, profile.persistState)
212
+ : (!opts.anonymous ? storageStateOptions(this.config.storageStatePath, 'global') : {});
213
+ const context = await browser.newContext(stateOptions);
158
214
  try {
159
215
  if (rulePack?.initScriptPath)
160
216
  await context.addInitScript({ path: rulePack.initScriptPath });
@@ -192,7 +248,7 @@ export class BrowserService {
192
248
  signal?.addEventListener('abort', onAbort);
193
249
  try {
194
250
  page.setDefaultTimeout(20_000);
195
- await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 25_000 });
251
+ await this.navigate(page, url, { waitUntil: 'domcontentloaded', timeout: 25_000 }, signal);
196
252
  await page.waitForLoadState('networkidle', { timeout: 8_000 }).catch(() => { });
197
253
  await applyRuleSteps(page, session.rulePack);
198
254
  if (opts.waitMs)
@@ -225,21 +281,22 @@ export class BrowserService {
225
281
  signal?.addEventListener('abort', onAbort);
226
282
  fs.mkdirSync(opts.outDir, { recursive: true });
227
283
  const stamp = Date.now() + '-' + uid().slice(0, 8);
228
- const screenshotPath = path.join(opts.outDir, stamp + '.png');
284
+ const screenshotPath = opts.screenshot === false ? undefined : path.join(opts.outDir, stamp + '.png');
229
285
  const htmlPath = path.join(opts.outDir, stamp + '.html');
230
286
  try {
231
287
  page.setDefaultTimeout(25_000);
232
- await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30_000 });
288
+ await this.navigate(page, url, { waitUntil: 'domcontentloaded', timeout: 30_000 }, signal);
233
289
  await page.waitForLoadState('networkidle', { timeout: 10_000 }).catch(() => { });
234
290
  await applyRuleSteps(page, session.rulePack);
235
- await page.screenshot({ path: screenshotPath, fullPage: true });
291
+ if (screenshotPath)
292
+ await page.screenshot({ path: screenshotPath, fullPage: true });
236
293
  const html = await page.content();
237
294
  fs.writeFileSync(htmlPath, html, 'utf8');
238
295
  const data = await evaluateExtractor(page, rules);
239
296
  return {
240
297
  title: String(data.title ?? ''),
241
298
  text: capText(String(data.text ?? '').replace(/\n{3,}/g, '\n\n').trim(), opts.maxChars ?? 200_000),
242
- screenshotPath,
299
+ ...screenshotPath ? { screenshotPath } : {},
243
300
  htmlPath,
244
301
  ...data.usedRule ? { usedRule: String(data.usedRule) } : {},
245
302
  };
@@ -266,7 +323,7 @@ export class BrowserService {
266
323
  signal?.addEventListener('abort', onAbort);
267
324
  try {
268
325
  page.setDefaultTimeout(25_000);
269
- await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30_000 });
326
+ await this.navigate(page, url, { waitUntil: 'domcontentloaded', timeout: 30_000 }, signal);
270
327
  await page.waitForLoadState('networkidle', { timeout: 8_000 }).catch(() => { });
271
328
  await applyRuleSteps(page, session.rulePack);
272
329
  if (opts.waitMs)
@@ -291,19 +348,140 @@ export class BrowserService {
291
348
  }
292
349
  // ── bundled opencli ───────────────────────────────────────────────────────
293
350
  opencliAvailable() {
294
- return this.config.opencliEnabled;
351
+ if (!this.config.enabled || !this.config.opencliEnabled)
352
+ return false;
353
+ try {
354
+ return fs.existsSync(opencliEntryPath());
355
+ }
356
+ catch {
357
+ return false;
358
+ }
295
359
  }
296
360
  opencli(args, opts = {}) {
361
+ if (!this.config.enabled)
362
+ return Promise.resolve({ code: -1, stdout: '', stderr: 'dsh-browser: browser service is disabled', timedOut: false });
297
363
  if (!this.config.opencliEnabled)
298
364
  return Promise.resolve({ code: -1, stdout: '', stderr: 'dsh-browser: OpenCLI is disabled', timedOut: false });
365
+ if (!this.opencliAvailable())
366
+ return Promise.resolve({ code: -1, stdout: '', stderr: 'dsh-browser: OpenCLI entry is not installed', timedOut: false });
299
367
  if (args.length < 1 || args.length > 40 || args.some(arg => typeof arg !== 'string' || arg.length > 2_000)) {
300
368
  return Promise.resolve({ code: -1, stdout: '', stderr: 'dsh-browser: OpenCLI requires 1 to 40 arguments, each at most 2000 characters', timedOut: false });
301
369
  }
302
- return runOpencli(args, { ...opts, signal: opts.signal });
370
+ // OpenCLI adapters can issue real site traffic outside Playwright, so they
371
+ // share the same approval-independent concurrency and burst buffer.
372
+ return this.usageGovernor.run('https://opencli.local/', () => runOpencli(args, { ...opts, signal: opts.signal }), opts.signal);
303
373
  }
304
374
  opencliDoctor(signal) {
305
375
  return this.opencli(['doctor'], { timeoutMs: 30_000, signal });
306
376
  }
377
+ async opencliCatalog(filter = {}, signal) {
378
+ if (!this.config.opencliEnabled)
379
+ throw new Error('dsh-browser: OpenCLI is disabled');
380
+ if (!this.opencliCatalogCache) {
381
+ const result = await this.opencli(['list', '-f', 'json'], { timeoutMs: 60_000, signal });
382
+ if (result.code !== 0 || result.timedOut)
383
+ throw new Error('OpenCLI catalog failed: ' + (result.stderr || result.stdout).slice(0, 500));
384
+ this.opencliCatalogCache = parseOpencliCatalog(result.stdout);
385
+ }
386
+ return filterOpencliCatalog(this.opencliCatalogCache, filter);
387
+ }
388
+ async crawl(startUrls, opts = {}) {
389
+ if (startUrls.length < 1 || startUrls.length > 5)
390
+ throw new Error('browser crawl requires 1 to 5 start URLs');
391
+ const normalized = startUrls.map(value => {
392
+ const url = new URL(value);
393
+ if (!['http:', 'https:'].includes(url.protocol))
394
+ throw new Error('browser crawl only supports HTTP(S) URLs');
395
+ url.hash = '';
396
+ return url.href;
397
+ });
398
+ const maxPages = opts.maxPages ?? this.config.usagePolicy.maxPagesPerRun;
399
+ const maxDepth = opts.maxDepth ?? this.config.usagePolicy.maxDepth;
400
+ if (!Number.isInteger(maxPages) || maxPages < 1 || maxPages > this.config.usagePolicy.maxPagesPerRun) {
401
+ throw new Error('browser crawl maxPages must be from 1 to configured usagePolicy.maxPagesPerRun (' + this.config.usagePolicy.maxPagesPerRun + ')');
402
+ }
403
+ if (!Number.isInteger(maxDepth) || maxDepth < 0 || maxDepth > this.config.usagePolicy.maxDepth) {
404
+ throw new Error('browser crawl maxDepth must be from 0 to configured usagePolicy.maxDepth (' + this.config.usagePolicy.maxDepth + ')');
405
+ }
406
+ const maxCharsPerPage = Math.min(Math.max(opts.maxCharsPerPage ?? 20_000, 1_000), 50_000);
407
+ const sameOrigin = opts.sameOrigin ?? true;
408
+ const allowedOrigins = new Set(normalized.map(value => new URL(value).origin));
409
+ const queue = normalized.map(url => ({ url, depth: 0 }));
410
+ const seen = new Set(normalized);
411
+ const pages = [];
412
+ const errors = [];
413
+ const before = this.usageGovernor.snapshot();
414
+ const started = Date.now();
415
+ const session = await this.transientContext(normalized[0], { anonymous: true });
416
+ try {
417
+ while (queue.length && pages.length + errors.length < maxPages) {
418
+ if (opts.signal?.aborted)
419
+ throw new Error('browser crawl aborted');
420
+ const item = queue.shift();
421
+ const page = await session.context.newPage();
422
+ try {
423
+ page.setDefaultTimeout(30_000);
424
+ const response = await this.navigate(page, item.url, { waitUntil: 'domcontentloaded', timeout: 30_000 }, opts.signal);
425
+ const status = Number(response?.status?.() ?? 0);
426
+ if (sameOrigin && !allowedOrigins.has(new URL(page.url()).origin)) {
427
+ errors.push({ url: item.url, depth: item.depth, status, error: 'cross-origin redirect blocked: ' + page.url() });
428
+ continue;
429
+ }
430
+ if (status >= 400) {
431
+ errors.push({ url: item.url, depth: item.depth, status, error: 'HTTP ' + status });
432
+ continue;
433
+ }
434
+ await page.waitForLoadState('networkidle', { timeout: 5_000 }).catch(() => { });
435
+ const data = await page.evaluate('(' + CRAWL_EXTRACTOR + ')(' + maxCharsPerPage + ')');
436
+ pages.push({ url: page.url(), title: String(data.title ?? ''), text: String(data.text ?? ''), depth: item.depth, status });
437
+ if (item.depth >= maxDepth)
438
+ continue;
439
+ for (const rawLink of Array.isArray(data.links) ? data.links : []) {
440
+ let link;
441
+ try {
442
+ link = new URL(rawLink);
443
+ link.hash = '';
444
+ }
445
+ catch {
446
+ continue;
447
+ }
448
+ if (sameOrigin && !allowedOrigins.has(link.origin))
449
+ continue;
450
+ const href = link.href;
451
+ if (seen.has(href) || seen.size >= maxPages * 25)
452
+ continue;
453
+ seen.add(href);
454
+ queue.push({ url: href, depth: item.depth + 1 });
455
+ }
456
+ }
457
+ catch (error) {
458
+ errors.push({ url: item.url, depth: item.depth, error: String(error).slice(0, 500) });
459
+ }
460
+ finally {
461
+ await page.close().catch(() => { });
462
+ }
463
+ }
464
+ }
465
+ finally {
466
+ await this.persistAndClose(session);
467
+ }
468
+ const after = this.usageGovernor.snapshot();
469
+ return {
470
+ pages,
471
+ errors,
472
+ stats: {
473
+ pagesVisited: pages.length + errors.length,
474
+ queued: queue.length,
475
+ elapsedMs: Date.now() - started,
476
+ waitMs: after.totalWaitMs - before.totalWaitMs,
477
+ backoffEvents: after.backoffEvents - before.backoffEvents,
478
+ },
479
+ warnings: [
480
+ 'Bounded crawl: respect each site\'s terms, robots directives, copyright, privacy, and applicable law.',
481
+ 'No-approval mode skips human confirmation only; concurrency, burst, page/depth budgets, and server-pressure backoff remain active.',
482
+ ],
483
+ };
484
+ }
307
485
  scriptCatalog() {
308
486
  return BUILTIN_SCRIPTS.map(script => ({
309
487
  id: script.id,
@@ -330,7 +508,7 @@ export class BrowserService {
330
508
  let timer;
331
509
  try {
332
510
  page.setDefaultTimeout(30_000);
333
- await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30_000 });
511
+ await this.navigate(page, url, { waitUntil: 'domcontentloaded', timeout: 30_000 }, signal);
334
512
  await page.waitForLoadState('networkidle', { timeout: 8_000 }).catch(() => { });
335
513
  await applyRuleSteps(page, session.rulePack);
336
514
  const timeoutMs = Math.min(Math.max(opts.timeoutMs ?? 15_000, 1_000), 30_000);
@@ -340,7 +518,7 @@ export class BrowserService {
340
518
  reject(new Error('userscript timed out after ' + timeoutMs + 'ms'));
341
519
  }, timeoutMs);
342
520
  });
343
- const executed = await Promise.race([executeUserscript(page, source), timeout]);
521
+ const executed = await Promise.race([executeUserscript(page, source, 100_000, opts.inputs), timeout]);
344
522
  return {
345
523
  url: page.url(),
346
524
  name: validation.metadata.name,
@@ -385,7 +563,7 @@ export class BrowserService {
385
563
  }
386
564
  else {
387
565
  const browser = await this.ensure();
388
- this.activeContext = await browser.newContext(this.config.storageStatePath ? { storageState: this.config.storageStatePath } : {});
566
+ this.activeContext = await browser.newContext(storageStateOptions(this.config.storageStatePath, 'global'));
389
567
  }
390
568
  this.activePage = await this.activeContext.newPage();
391
569
  return this.activePage;
@@ -410,7 +588,7 @@ export class BrowserService {
410
588
  async open(url, opts = {}) {
411
589
  const page = await this.ensureActivePage(url, opts);
412
590
  page.setDefaultTimeout(30_000);
413
- await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30_000 });
591
+ await this.navigate(page, url, { waitUntil: 'domcontentloaded', timeout: 30_000 });
414
592
  await page.waitForLoadState('networkidle', { timeout: 8_000 }).catch(() => { });
415
593
  await applyRuleSteps(page, this.activeRulePack);
416
594
  if (opts.waitMs)
@@ -453,7 +631,7 @@ export class BrowserService {
453
631
  const page = await this.ensureActivePage(opts.url, opts);
454
632
  if (opts.url) {
455
633
  page.setDefaultTimeout(30_000);
456
- await page.goto(opts.url, { waitUntil: 'domcontentloaded', timeout: 30_000 });
634
+ await this.navigate(page, opts.url, { waitUntil: 'domcontentloaded', timeout: 30_000 }, opts.signal);
457
635
  await page.waitForLoadState('networkidle', { timeout: 8_000 }).catch(() => { });
458
636
  await applyRuleSteps(page, this.activeRulePack);
459
637
  if (opts.waitMs)
@@ -485,25 +663,59 @@ export class BrowserService {
485
663
  }
486
664
  async status() {
487
665
  let chromiumInstalled = false;
666
+ let chromiumExecutablePath;
488
667
  try {
489
- const pw = loadPlaywright();
490
- chromiumInstalled = !!pw.chromium.executablePath();
668
+ const pw = loadBrowserRuntime(this.config.browserRuntime);
669
+ const expectedPath = this.config.executablePath || pw.chromium.executablePath();
670
+ if (typeof expectedPath === 'string' && expectedPath.trim()) {
671
+ chromiumExecutablePath = path.resolve(expectedPath);
672
+ chromiumInstalled = fs.existsSync(chromiumExecutablePath);
673
+ }
491
674
  }
492
675
  catch {
493
676
  chromiumInstalled = false;
494
677
  }
678
+ let opencliInstalled = false;
679
+ let resolvedOpencliEntryPath;
680
+ try {
681
+ resolvedOpencliEntryPath = path.resolve(opencliEntryPath());
682
+ opencliInstalled = fs.existsSync(resolvedOpencliEntryPath);
683
+ }
684
+ catch {
685
+ opencliInstalled = false;
686
+ }
687
+ const runtimeWarnings = this.config.browserRuntime === 'patchright'
688
+ ? [
689
+ 'Patchright is Chromium-only and disables Playwright console APIs to avoid Runtime.enable detection.',
690
+ ...(this.config.channel !== 'chrome' || this.config.headless
691
+ ? ['Patchright stealth is strongest with channel=chrome and headless=false; current settings favor automation/test compatibility.']
692
+ : []),
693
+ ]
694
+ : [];
695
+ if (!chromiumInstalled && chromiumExecutablePath && (this.config.channel === 'chromium' || !!this.config.executablePath)) {
696
+ runtimeWarnings.push(`Expected Chromium executable is missing: ${chromiumExecutablePath}. Run browser_install for ${this.config.browserRuntime}.`);
697
+ }
698
+ if (this.config.opencliEnabled && !opencliInstalled)
699
+ runtimeWarnings.push('OpenCLI is enabled but its package entry is not installed.');
495
700
  return {
496
701
  enabled: this.config.enabled,
497
702
  channel: this.config.channel,
703
+ browserRuntime: this.config.browserRuntime,
704
+ runtimeWarnings,
498
705
  headless: this.config.headless,
499
706
  opencliEnabled: this.config.opencliEnabled,
707
+ opencliInstalled,
708
+ ...resolvedOpencliEntryPath ? { opencliEntryPath: resolvedOpencliEntryPath } : {},
500
709
  automationMode: this.config.automationMode,
501
- exposedTools: browserToolsForMode(this.config.automationMode),
502
- directInteractionPolicy: this.config.automationMode === 'read-only' ? 'deny' : this.config.automationMode === 'standard' ? 'ask' : 'allow',
503
- mutatingRecipePolicy: this.config.automationMode === 'read-only' ? 'deny' : this.config.automationMode === 'standard' ? 'ask' : 'allow',
504
- externalUserscriptPolicy: this.config.automationMode === 'read-only' ? 'deny' : this.config.automationMode === 'unrestricted' ? 'allow' : 'ask',
505
- opencliRunPolicy: this.config.automationMode === 'read-only' ? 'deny' : this.config.automationMode === 'unrestricted' ? 'allow' : 'ask',
710
+ exposedTools: configuredBrowserTools(this.config.automationMode, this.config.automationAssets, this.config.enabled),
711
+ directInteractionPolicy: !this.config.enabled || this.config.automationMode === 'read-only' ? 'deny' : this.config.automationMode === 'standard' ? 'ask' : 'allow',
712
+ mutatingRecipePolicy: !this.config.enabled || this.config.automationMode === 'read-only' ? 'deny' : this.config.automationMode === 'standard' ? 'ask' : 'allow',
713
+ externalUserscriptPolicy: !this.config.enabled || this.config.automationMode === 'read-only' ? 'deny' : this.config.automationMode === 'unrestricted' ? 'allow' : 'ask',
714
+ opencliRunPolicy: !this.config.enabled || this.config.automationMode === 'read-only' ? 'deny' : this.config.automationMode === 'unrestricted' ? 'allow' : 'ask',
506
715
  chromiumInstalled,
716
+ ...chromiumExecutablePath ? { chromiumExecutablePath } : {},
717
+ usagePolicy: this.config.usagePolicy,
718
+ usageGovernor: this.usageGovernor.snapshot(),
507
719
  authProfiles: this.authProfiles.list(),
508
720
  rulePacks: Object.keys(this.config.rulePacks).sort(),
509
721
  builtinScripts: BUILTIN_SCRIPTS.map(script => script.id),