@anweat/dsh-browser 0.2.1 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +267 -69
- package/assets/skills/dsh-browser/SKILL.md +13 -11
- package/assets/skills/dsh-browser/references/files.md +36 -0
- package/assets/skills/dsh-browser/references/observe.md +2 -2
- package/assets/skills/dsh-browser/references/policy.md +32 -0
- package/cordis.patch.yml +10 -3
- package/lib/actions/errors.js +7 -2
- package/lib/actions/errors.js.map +1 -1
- package/lib/actions/index-view.d.ts +4 -1
- package/lib/actions/index-view.js +16 -3
- package/lib/actions/index-view.js.map +1 -1
- package/lib/actions/inspect.js +15 -0
- package/lib/actions/inspect.js.map +1 -1
- package/lib/actions/observe.js +1 -1
- package/lib/actions/observe.js.map +1 -1
- package/lib/actions/opencli.d.ts +1 -1
- package/lib/actions/opencli.js +4 -4
- package/lib/actions/opencli.js.map +1 -1
- package/lib/actions/policy.d.ts +15 -0
- package/lib/actions/policy.js +131 -0
- package/lib/actions/policy.js.map +1 -0
- package/lib/actions/registry.js +6 -5
- package/lib/actions/registry.js.map +1 -1
- package/lib/actions/runtime.js +2 -2
- package/lib/actions/runtime.js.map +1 -1
- package/lib/actions/target.js +35 -3
- package/lib/actions/target.js.map +1 -1
- package/lib/actions/types.d.ts +1 -1
- package/lib/actions/types.js.map +1 -1
- package/lib/approval-policy.d.ts +10 -1
- package/lib/approval-policy.js +42 -1
- package/lib/approval-policy.js.map +1 -1
- package/lib/auth-profiles.d.ts +4 -1
- package/lib/auth-profiles.js +3 -0
- package/lib/auth-profiles.js.map +1 -1
- package/lib/automation-assets-rpc.js +26 -0
- package/lib/automation-assets-rpc.js.map +1 -1
- package/lib/browser-service.d.ts +165 -4
- package/lib/browser-service.js +505 -55
- package/lib/browser-service.js.map +1 -1
- package/lib/client/SettingsCard.d.ts +10 -8
- package/lib/client/SettingsCard.js +12 -137
- package/lib/client/SettingsCard.js.map +1 -1
- package/lib/client/auth-profiles-form.d.ts +23 -0
- package/lib/client/auth-profiles-form.js +64 -0
- package/lib/client/auth-profiles-form.js.map +1 -0
- package/lib/client/card-frame.d.ts +22 -0
- package/lib/client/card-frame.js +61 -0
- package/lib/client/card-frame.js.map +1 -0
- package/lib/client/field-rows.d.ts +59 -0
- package/lib/client/field-rows.js +63 -0
- package/lib/client/field-rows.js.map +1 -0
- package/lib/client/field-rules.d.ts +17 -0
- package/lib/client/field-rules.js +68 -0
- package/lib/client/field-rules.js.map +1 -0
- package/lib/client/form.d.ts +108 -7
- package/lib/client/form.js +255 -11
- package/lib/client/form.js.map +1 -1
- package/lib/client/group-access.d.ts +14 -0
- package/lib/client/group-access.js +97 -0
- package/lib/client/group-access.js.map +1 -0
- package/lib/client/group-advanced.d.ts +11 -0
- package/lib/client/group-advanced.js +32 -0
- package/lib/client/group-advanced.js.map +1 -0
- package/lib/client/group-assets.d.ts +13 -0
- package/lib/client/group-assets.js +50 -0
- package/lib/client/group-assets.js.map +1 -0
- package/lib/client/group-browser.d.ts +9 -0
- package/lib/client/group-browser.js +27 -0
- package/lib/client/group-browser.js.map +1 -0
- package/lib/client/group-common.d.ts +10 -0
- package/lib/client/group-common.js +21 -0
- package/lib/client/group-common.js.map +1 -0
- package/lib/client/group-context.d.ts +20 -0
- package/lib/client/group-context.js +76 -0
- package/lib/client/group-context.js.map +1 -0
- package/lib/client/group-downloads.d.ts +10 -0
- package/lib/client/group-downloads.js +33 -0
- package/lib/client/group-downloads.js.map +1 -0
- package/lib/client/group-limits.d.ts +9 -0
- package/lib/client/group-limits.js +57 -0
- package/lib/client/group-limits.js.map +1 -0
- package/lib/client/group-prompts.d.ts +13 -0
- package/lib/client/group-prompts.js +136 -0
- package/lib/client/group-prompts.js.map +1 -0
- package/lib/client/index.d.ts +21 -1
- package/lib/client/index.js +10 -4
- package/lib/client/index.js.map +1 -1
- package/lib/client/locales-policy.d.ts +83 -0
- package/lib/client/locales-policy.js +46 -0
- package/lib/client/locales-policy.js.map +1 -0
- package/lib/client/locales-prompts.d.ts +57 -0
- package/lib/client/locales-prompts.js +41 -0
- package/lib/client/locales-prompts.js.map +1 -0
- package/lib/client/locales-ui.d.ts +223 -0
- package/lib/client/locales-ui.js +157 -0
- package/lib/client/locales-ui.js.map +1 -0
- package/lib/client/locales.d.ts +359 -67
- package/lib/client/locales.js +68 -22
- package/lib/client/locales.js.map +1 -1
- package/lib/client/policy-fields.d.ts +31 -0
- package/lib/client/policy-fields.js +66 -0
- package/lib/client/policy-fields.js.map +1 -0
- package/lib/client/policy-status-client.d.ts +91 -0
- package/lib/client/policy-status-client.js +64 -0
- package/lib/client/policy-status-client.js.map +1 -0
- package/lib/client/prompts-form.d.ts +33 -0
- package/lib/client/prompts-form.js +87 -0
- package/lib/client/prompts-form.js.map +1 -1
- package/lib/client/prompts-status-client.d.ts +21 -1
- package/lib/client/prompts-status-client.js +36 -4
- package/lib/client/prompts-status-client.js.map +1 -1
- package/lib/client/properties-form.d.ts +90 -0
- package/lib/client/properties-form.js +187 -0
- package/lib/client/properties-form.js.map +1 -0
- package/lib/client/property-docs.d.ts +18 -0
- package/lib/client/property-docs.js +78 -0
- package/lib/client/property-docs.js.map +1 -0
- package/lib/client/size-units.d.ts +12 -0
- package/lib/client/size-units.js +29 -0
- package/lib/client/size-units.js.map +1 -0
- package/lib/client/styles.d.ts +47 -31
- package/lib/client/styles.js +59 -20
- package/lib/client/styles.js.map +1 -1
- package/lib/client/ui-kit.d.ts +173 -0
- package/lib/client/ui-kit.js +112 -0
- package/lib/client/ui-kit.js.map +1 -0
- package/lib/client.js +5291 -834
- package/lib/client.js.map +1 -1
- package/lib/config-rules.d.ts +40 -0
- package/lib/config-rules.js +103 -0
- package/lib/config-rules.js.map +1 -0
- package/lib/config.d.ts +64 -2
- package/lib/config.js +106 -0
- package/lib/config.js.map +1 -1
- package/lib/context-options.d.ts +85 -0
- package/lib/context-options.js +206 -0
- package/lib/context-options.js.map +1 -0
- package/lib/deps.d.ts +42 -5
- package/lib/deps.js +219 -9
- package/lib/deps.js.map +1 -1
- package/lib/downloads.d.ts +79 -0
- package/lib/downloads.js +189 -0
- package/lib/downloads.js.map +1 -0
- package/lib/index.d.ts +3 -2
- package/lib/index.js +7 -3
- package/lib/index.js.map +1 -1
- package/lib/policy-keys.d.ts +39 -0
- package/lib/policy-keys.js +21 -0
- package/lib/policy-keys.js.map +1 -1
- package/lib/policy-overlay.d.ts +189 -0
- package/lib/policy-overlay.js +649 -0
- package/lib/policy-overlay.js.map +1 -0
- package/lib/prompts-yaml.d.ts +11 -0
- package/lib/prompts-yaml.js +121 -0
- package/lib/prompts-yaml.js.map +1 -0
- package/lib/prompts.d.ts +2 -0
- package/lib/prompts.js +7 -3
- package/lib/prompts.js.map +1 -1
- package/lib/system-browser.d.ts +61 -0
- package/lib/system-browser.js +109 -0
- package/lib/system-browser.js.map +1 -0
- package/lib/tools.js +9 -1
- package/lib/tools.js.map +1 -1
- package/lib/usage-policy.d.ts +3 -1
- package/lib/usage-policy.js +2 -0
- package/lib/usage-policy.js.map +1 -1
- package/package.json +42 -38
- package/scripts/dump-prompts.mjs +12 -2
package/lib/browser-service.js
CHANGED
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
* - render / snapshot / searchResults are the drop-in replacements for
|
|
9
9
|
* web-search-pro's former PlaywrightManager (rules-aware page extraction and
|
|
10
10
|
* platform search-page list extraction run in the page itself).
|
|
11
|
-
* - opencli(...) runs
|
|
11
|
+
* - opencli(...) runs @jackwener/opencli when it is installed (optional, see deps.ts).
|
|
12
12
|
* - The interactive surface (open/click/type/hover/setFiles/evaluate/scroll/read/screenshot/closePage)
|
|
13
13
|
* drives ONE persistent context+page, giving the model multi-step browsing.
|
|
14
14
|
*
|
|
@@ -21,7 +21,8 @@
|
|
|
21
21
|
import fs from 'node:fs';
|
|
22
22
|
import path from 'node:path';
|
|
23
23
|
import crypto from 'node:crypto';
|
|
24
|
-
import { browserRuntimeCliPath, loadBrowserRuntime,
|
|
24
|
+
import { browserRuntimeCliPath, loadBrowserRuntime, opencliLocation, opencliMissingMessage, runOpencli, runNode } from "./deps.js";
|
|
25
|
+
import { browserInstallAdvice, detectSystemBrowsers, planBrowser } from "./system-browser.js";
|
|
25
26
|
import { AuthProfileStore, hostAllowed } from "./auth-profiles.js";
|
|
26
27
|
import { applyRuleSteps, resolveRulePack } from "./rule-packs.js";
|
|
27
28
|
import { runRecipe } from "./automation.js";
|
|
@@ -29,16 +30,32 @@ import { BUILTIN_SCRIPTS, builtinScript, executeUserscript, validateUserscript }
|
|
|
29
30
|
import { configuredBrowserActions, configuredBrowserTools } from "./freedom.js";
|
|
30
31
|
import { filterOpencliCatalog, parseOpencliCatalog } from "./opencli-catalog.js";
|
|
31
32
|
import { UsageGovernor } from "./usage-policy.js";
|
|
33
|
+
import { PolicyOverlayStore, domainBase } from "./policy-overlay.js";
|
|
32
34
|
import { installDialogGuard } from "./dialogs.js";
|
|
33
35
|
import { TargetStaleError, resolveLocator, withStrictLocator } from "./locator.js";
|
|
34
36
|
import { SENSITIVE_NAME, SessionJournal } from "./journal.js";
|
|
35
37
|
import { describeScan, fitObservation, normalizeObserve, scanPage, scopeBase } from "./observe.js";
|
|
38
|
+
import { contextSources, describeContextOptions, differingContextOptions, effectiveContextOptions, resolveCallOptions, toPlaywrightOptions } from "./context-options.js";
|
|
39
|
+
import { DOWNLOAD_HISTORY_LIMIT, DownloadLimitError, claimDownloadPath, contentDispositionFilename, removeQuietly, resolveDownloadFilename, sanitizeDownloadName, streamToFile, temporaryDownloadPath, writeBufferToFile, } from "./downloads.js";
|
|
36
40
|
/**
|
|
37
41
|
* How many sessions may hold a browser context+page at once. Past this, the
|
|
38
42
|
* least-recently-used session is closed, so a long-lived profile cannot
|
|
39
43
|
* accumulate contexts until the browser dies.
|
|
40
44
|
*/
|
|
41
45
|
const DEFAULT_MAX_SESSIONS = 8;
|
|
46
|
+
function downloadResult(record, requestedUrl) {
|
|
47
|
+
return {
|
|
48
|
+
id: record.id,
|
|
49
|
+
url: requestedUrl,
|
|
50
|
+
...record.url !== redactCaptureUrl(requestedUrl) ? { finalUrl: record.url } : {},
|
|
51
|
+
path: record.path,
|
|
52
|
+
filename: path.basename(record.path),
|
|
53
|
+
bytes: record.bytes,
|
|
54
|
+
sha256: record.sha256,
|
|
55
|
+
suggestedFilename: record.suggestedFilename,
|
|
56
|
+
source: record.source,
|
|
57
|
+
};
|
|
58
|
+
}
|
|
42
59
|
/** Rules-aware content extractor (runs in the page). */
|
|
43
60
|
const EXTRACTOR_FN = `(ruleList) => {
|
|
44
61
|
const doc = document
|
|
@@ -161,6 +178,9 @@ function newSessionState() {
|
|
|
161
178
|
pages: [],
|
|
162
179
|
nextTarget: 1,
|
|
163
180
|
generationClock: 0,
|
|
181
|
+
downloads: [],
|
|
182
|
+
nextDownload: 1,
|
|
183
|
+
downloadJobs: new WeakMap(),
|
|
164
184
|
captureConsole: false,
|
|
165
185
|
captureNetwork: false,
|
|
166
186
|
capturedConsole: [],
|
|
@@ -204,6 +224,7 @@ export function sessionKeyFor(agent) {
|
|
|
204
224
|
}
|
|
205
225
|
export class BrowserService {
|
|
206
226
|
config;
|
|
227
|
+
probe;
|
|
207
228
|
browser;
|
|
208
229
|
launching;
|
|
209
230
|
/** Per-session interactive state. Replaces the former global activePage. */
|
|
@@ -213,20 +234,124 @@ export class BrowserService {
|
|
|
213
234
|
clock = 0;
|
|
214
235
|
authProfiles;
|
|
215
236
|
usageGovernor;
|
|
237
|
+
/** The managed policy overlay merged over the configured authProfiles and usagePolicy (see policy-overlay.ts). */
|
|
238
|
+
policyStore;
|
|
216
239
|
opencliCatalogCache;
|
|
217
240
|
/** Per-session exploration journals (see src/journal.ts). Released with the session, never written to disk. */
|
|
218
241
|
journals = new Map();
|
|
219
242
|
/** How many sessions may hold a context+page at once before LRU eviction. */
|
|
220
243
|
sessionLimit;
|
|
221
|
-
|
|
244
|
+
/**
|
|
245
|
+
* `probe` replaces what the service reads from the machine (system browsers, file existence); tests use it, the
|
|
246
|
+
* plugin never passes it.
|
|
247
|
+
*/
|
|
248
|
+
constructor(config, probe = {}) {
|
|
222
249
|
this.config = config;
|
|
223
|
-
this.
|
|
224
|
-
this.
|
|
250
|
+
this.probe = probe;
|
|
251
|
+
this.policyStore = new PolicyOverlayStore({
|
|
252
|
+
file: config.policyOverlayFile,
|
|
253
|
+
authDir: config.authDir,
|
|
254
|
+
snapshotDir: config.snapshotDir,
|
|
255
|
+
downloadDir: config.downloadDir,
|
|
256
|
+
allowPrivateHosts: config.allowPrivateHosts,
|
|
257
|
+
baseline: { authProfiles: config.authProfiles, usagePolicy: config.usagePolicy },
|
|
258
|
+
});
|
|
259
|
+
const effective = this.policyStore.effective();
|
|
260
|
+
this.authProfiles = new AuthProfileStore(effective.authProfiles);
|
|
261
|
+
this.usageGovernor = new UsageGovernor(effective.usagePolicy);
|
|
225
262
|
this.sessionLimit = Math.max(1, Number.isInteger(config.maxSessions) ? config.maxSessions : DEFAULT_MAX_SESSIONS);
|
|
226
263
|
}
|
|
227
264
|
available() {
|
|
228
265
|
return this.config.enabled;
|
|
229
266
|
}
|
|
267
|
+
// -- policy overlay: restriction settings an agent may edit ---------------------------------
|
|
268
|
+
/** Pick up an overlay file that changed on disk (another process, the panel). Cheap: one stat. */
|
|
269
|
+
refreshPolicy() {
|
|
270
|
+
if (this.policyStore.refresh())
|
|
271
|
+
this.applyEffectivePolicy();
|
|
272
|
+
}
|
|
273
|
+
applyEffectivePolicy() {
|
|
274
|
+
const effective = this.policyStore.effective();
|
|
275
|
+
this.authProfiles.replace(effective.authProfiles);
|
|
276
|
+
this.usageGovernor.setPolicy(effective.usagePolicy);
|
|
277
|
+
}
|
|
278
|
+
/** What a policy change would do, without doing it; `ok: false` carries why it is invalid. */
|
|
279
|
+
previewPolicyChange(change) {
|
|
280
|
+
try {
|
|
281
|
+
this.refreshPolicy();
|
|
282
|
+
const planned = this.policyStore.plan(change);
|
|
283
|
+
return { ok: true, direction: planned.diff.direction, lines: planned.diff.lines };
|
|
284
|
+
}
|
|
285
|
+
catch (error) {
|
|
286
|
+
return { ok: false, message: String(error instanceof Error ? error.message : error) };
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
/**
|
|
290
|
+
* Apply a policy change: validated, written atomically with an audit entry, effective at once. Sessions whose
|
|
291
|
+
* context holds a login state this change took away are closed, so a revoked domain stops working immediately
|
|
292
|
+
* instead of at the next page load.
|
|
293
|
+
*/
|
|
294
|
+
async changePolicy(change, meta) {
|
|
295
|
+
this.assertEnabled();
|
|
296
|
+
this.refreshPolicy();
|
|
297
|
+
const applied = this.policyStore.apply(change, meta);
|
|
298
|
+
this.applyEffectivePolicy();
|
|
299
|
+
let closed = 0;
|
|
300
|
+
for (const state of [...this.sessions.values()]) {
|
|
301
|
+
const profile = state.profile;
|
|
302
|
+
if (!profile)
|
|
303
|
+
continue;
|
|
304
|
+
const now = this.authProfiles.get(profile.id);
|
|
305
|
+
const lost = !now
|
|
306
|
+
|| now.storageStatePath !== profile.storageStatePath
|
|
307
|
+
|| profile.allowedDomains.some(domain => !hostAllowed(domainBase(domain), now.allowedDomains));
|
|
308
|
+
if (lost) {
|
|
309
|
+
await this.closePage(state).catch(() => { });
|
|
310
|
+
closed++;
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
const summary = (policy) => ({
|
|
314
|
+
authProfiles: Object.fromEntries(applied.diff.profiles.map(entry => [entry.id, policy.authProfiles[entry.id] ? { allowedDomains: policy.authProfiles[entry.id].allowedDomains, persistState: policy.authProfiles[entry.id].persistState ?? false } : null])),
|
|
315
|
+
usagePolicy: Object.fromEntries(applied.diff.usage.map(entry => [entry.key, policy.usagePolicy[entry.key]])),
|
|
316
|
+
});
|
|
317
|
+
return { revision: applied.revision, direction: applied.diff.direction, diff: applied.diff, audit: applied.audit, sessionsClosed: closed, before: summary(applied.before), after: summary(applied.after) };
|
|
318
|
+
}
|
|
319
|
+
/** The effective policy, where each value comes from, and the newest audit entries. */
|
|
320
|
+
policyReport(opts = {}) {
|
|
321
|
+
this.refreshPolicy();
|
|
322
|
+
const overlay = this.policyStore.current();
|
|
323
|
+
const effective = this.policyStore.effective();
|
|
324
|
+
const baseline = this.policyStore.baseline;
|
|
325
|
+
const profiles = Object.entries(effective.authProfiles).sort(([a], [b]) => a.localeCompare(b)).map(([id, profile]) => {
|
|
326
|
+
const entry = overlay.authProfiles[id];
|
|
327
|
+
const origin = !baseline.authProfiles[id] ? 'overlay-created' : entry ? 'overlay-changed' : 'baseline';
|
|
328
|
+
return {
|
|
329
|
+
id, origin,
|
|
330
|
+
allowedDomains: profile.allowedDomains,
|
|
331
|
+
persistState: profile.persistState ?? false,
|
|
332
|
+
// Only a path the overlay chose (or one inside the auth directory) is shown: a baseline path is the user's own layout.
|
|
333
|
+
...entry?.storageStatePath || profile.storageStatePath.startsWith(this.config.authDir) ? { storageStatePath: profile.storageStatePath } : {},
|
|
334
|
+
...entry ? { overriddenFields: Object.keys(entry) } : {},
|
|
335
|
+
};
|
|
336
|
+
});
|
|
337
|
+
const limit = Math.min(Math.max(Math.trunc(opts.auditLimit ?? 10), 1), 50);
|
|
338
|
+
return {
|
|
339
|
+
revision: overlay.revision,
|
|
340
|
+
...overlay.revision > 0 ? { updatedAt: overlay.updatedAt } : {},
|
|
341
|
+
overlayActive: Object.keys(overlay.authProfiles).length > 0 || Object.keys(overlay.usagePolicy).length > 0,
|
|
342
|
+
...this.policyStore.warning ? { warning: this.policyStore.warning } : {},
|
|
343
|
+
allowPrivateHosts: this.config.allowPrivateHosts,
|
|
344
|
+
authDir: this.config.authDir,
|
|
345
|
+
...this.config.configWarnings.length ? { configWarnings: this.config.configWarnings } : {},
|
|
346
|
+
authProfiles: profiles,
|
|
347
|
+
removedByOverlay: Object.entries(overlay.authProfiles).filter(([, entry]) => entry === null).map(([id]) => id).sort(),
|
|
348
|
+
usagePolicy: Object.fromEntries(Object.entries(effective.usagePolicy).map(([key, value]) => [key, { value, baseline: baseline.usagePolicy[key], overridden: Object.hasOwn(overlay.usagePolicy, key) }])),
|
|
349
|
+
// An agent sees which session made a change only for its own; the settings card (no viewer) sees all.
|
|
350
|
+
audit: overlay.audit.slice(-limit).map(entry => opts.viewerSession === undefined || entry.session === undefined || entry.session === opts.viewerSession
|
|
351
|
+
? entry
|
|
352
|
+
: { ...entry, session: 'another session' }),
|
|
353
|
+
};
|
|
354
|
+
}
|
|
230
355
|
/**
|
|
231
356
|
* This session's exploration journal, created on first use. It outlives `target.close` (the session is
|
|
232
357
|
* still the same) and is released when the session is evicted or the service closes. A session that never
|
|
@@ -358,6 +483,8 @@ export class BrowserService {
|
|
|
358
483
|
state.context = undefined;
|
|
359
484
|
state.profile = undefined;
|
|
360
485
|
state.rulePack = undefined;
|
|
486
|
+
state.contextOptions = undefined;
|
|
487
|
+
state.downloads = [];
|
|
361
488
|
state.capturedConsole = [];
|
|
362
489
|
state.capturedRequests = [];
|
|
363
490
|
state.captureConsole = false;
|
|
@@ -397,11 +524,8 @@ export class BrowserService {
|
|
|
397
524
|
if (!this.launching) {
|
|
398
525
|
this.launching = (async () => {
|
|
399
526
|
const pw = loadBrowserRuntime(this.config.browserRuntime);
|
|
400
|
-
const
|
|
401
|
-
|
|
402
|
-
launchOptions.channel = this.config.channel;
|
|
403
|
-
if (this.config.executablePath)
|
|
404
|
-
launchOptions.executablePath = this.config.executablePath;
|
|
527
|
+
const plan = this.browserPlan(pw);
|
|
528
|
+
const launchOptions = { headless: this.config.headless, ...plan.launch };
|
|
405
529
|
const args = [...(this.config.args ?? [])];
|
|
406
530
|
if (this.config.cdpPort) {
|
|
407
531
|
args.push(`--remote-debugging-port=${this.config.cdpPort}`);
|
|
@@ -414,14 +538,13 @@ export class BrowserService {
|
|
|
414
538
|
}
|
|
415
539
|
catch (error) {
|
|
416
540
|
const msg = String(error);
|
|
417
|
-
if (/Executable doesn't exist|playwright install|not found/i.test(msg)) {
|
|
418
|
-
|
|
541
|
+
if (/Executable doesn't exist|playwright install|not found|distribution .* is not found/i.test(msg)) {
|
|
542
|
+
// Only the default Playwright Chromium can be downloaded; a configured channel / executablePath is the user's own choice.
|
|
543
|
+
if (this.config.autoInstall && plan.source === 'playwright-chromium') {
|
|
419
544
|
await this.installChromium();
|
|
420
545
|
return this.trackBrowser(await pw.chromium.launch(launchOptions));
|
|
421
546
|
}
|
|
422
|
-
|
|
423
|
-
throw new Error('dsh-browser: chromium is not installed for ' + this.config.browserRuntime + '. Run the runtime.install action, or: node "' + browserRuntimeCliPath(this.config.browserRuntime) + '" install chromium');
|
|
424
|
-
}
|
|
547
|
+
throw new Error('dsh-browser: ' + this.launchFailure(plan, msg));
|
|
425
548
|
}
|
|
426
549
|
else {
|
|
427
550
|
throw error;
|
|
@@ -438,6 +561,47 @@ export class BrowserService {
|
|
|
438
561
|
this.launching = undefined;
|
|
439
562
|
}
|
|
440
563
|
}
|
|
564
|
+
systemBrowsers() {
|
|
565
|
+
return this.probe.systemBrowsers ? this.probe.systemBrowsers() : detectSystemBrowsers();
|
|
566
|
+
}
|
|
567
|
+
/** What a launch would use right now. `pw` is the loaded runtime; without it (not installed) the Playwright Chromium path is unknown. */
|
|
568
|
+
browserPlan(pw) {
|
|
569
|
+
let bundledPath;
|
|
570
|
+
try {
|
|
571
|
+
const expected = pw?.chromium?.executablePath?.();
|
|
572
|
+
if (typeof expected === 'string' && expected.trim())
|
|
573
|
+
bundledPath = expected;
|
|
574
|
+
}
|
|
575
|
+
catch { /* an unsupported platform has no bundled path */ }
|
|
576
|
+
return planBrowser({
|
|
577
|
+
channel: this.config.channel,
|
|
578
|
+
...this.config.executablePath ? { executablePath: this.config.executablePath } : {},
|
|
579
|
+
...bundledPath ? { bundledPath } : {},
|
|
580
|
+
autoInstall: this.config.autoInstall,
|
|
581
|
+
systemBrowsers: this.systemBrowsers(),
|
|
582
|
+
exists: this.probe.exists ?? (file => { try {
|
|
583
|
+
return fs.existsSync(file);
|
|
584
|
+
}
|
|
585
|
+
catch {
|
|
586
|
+
return false;
|
|
587
|
+
} }),
|
|
588
|
+
});
|
|
589
|
+
}
|
|
590
|
+
/** The message for a launch that found no browser. Says which setting was wrong when the user chose one. */
|
|
591
|
+
launchFailure(plan, cause) {
|
|
592
|
+
if (plan.source === 'system-browser')
|
|
593
|
+
return `${plan.note ?? 'A system browser is used.'} It could not be started${cause ? `: ${cause.split('\n')[0]}` : ''}. Install Playwright Chromium instead (run the runtime.install action), or set executablePath to a working Chrome / Chromium.`;
|
|
594
|
+
if (plan.source === 'custom-executable')
|
|
595
|
+
return `executablePath does not exist or is not a browser: ${this.config.executablePath}. Fix or clear executablePath in the browser plugin settings.`;
|
|
596
|
+
if (plan.source === 'channel')
|
|
597
|
+
return `channel is ${this.config.channel} but that browser is not installed. Install it, or set channel to chromium (and run runtime.install, or let Google Chrome / Microsoft Edge be picked up automatically).`;
|
|
598
|
+
let cliPath;
|
|
599
|
+
try {
|
|
600
|
+
cliPath = browserRuntimeCliPath(this.config.browserRuntime);
|
|
601
|
+
}
|
|
602
|
+
catch { /* runtime not installed */ }
|
|
603
|
+
return browserInstallAdvice({ runtime: this.config.browserRuntime, ...cliPath ? { cliPath } : {}, ...process.env.PLAYWRIGHT_BROWSERS_PATH ? { browsersPath: process.env.PLAYWRIGHT_BROWSERS_PATH } : {} });
|
|
604
|
+
}
|
|
441
605
|
/** Run `playwright install chromium` from the bundled playwright CLI. */
|
|
442
606
|
installChromium() {
|
|
443
607
|
this.assertEnabled();
|
|
@@ -445,7 +609,7 @@ export class BrowserService {
|
|
|
445
609
|
}
|
|
446
610
|
async navigate(page, url, options, signal) {
|
|
447
611
|
let response;
|
|
448
|
-
for (let attempt = 0; attempt <= this.
|
|
612
|
+
for (let attempt = 0; attempt <= this.usageGovernor.policy.retryLimit; attempt++) {
|
|
449
613
|
response = await this.usageGovernor.run(url, () => page.goto(url, options), signal);
|
|
450
614
|
const status = Number(response?.status?.() ?? 0);
|
|
451
615
|
if (status >= 200 && status < 400)
|
|
@@ -457,14 +621,27 @@ export class BrowserService {
|
|
|
457
621
|
? Number(rawRetryAfter) * 1000
|
|
458
622
|
: (Number.isFinite(Date.parse(rawRetryAfter)) ? Math.max(Date.parse(rawRetryAfter) - Date.now(), 0) : undefined);
|
|
459
623
|
this.usageGovernor.noteResponse(url, status, retryAfterMs);
|
|
460
|
-
if (attempt === this.
|
|
624
|
+
if (attempt === this.usageGovernor.policy.retryLimit)
|
|
461
625
|
return response;
|
|
462
626
|
}
|
|
463
627
|
return response;
|
|
464
628
|
}
|
|
629
|
+
/**
|
|
630
|
+
* `newContext()` arguments that do not depend on the login state: the configured viewport, user agent, locale,
|
|
631
|
+
* time zone and proxy (or the per-call override that replaced them), plus downloads for interactive contexts.
|
|
632
|
+
*
|
|
633
|
+
* Proxy: Playwright applies `proxy` at context level on its own. Measured on Chromium 1243 (headless shell) with
|
|
634
|
+
* Playwright 1.62 and Patchright 1.62, a context created with `proxy` and a browser launched WITHOUT any launch
|
|
635
|
+
* `proxy` sent both page navigation and `context.request` through the proxy, so no `proxy: { server: 'per-context' }`
|
|
636
|
+
* launch option is set. (Chromium bypasses the proxy for loopback hosts regardless; that is the browser's rule.)
|
|
637
|
+
*/
|
|
638
|
+
contextCreationOptions(options, interactive) {
|
|
639
|
+
return { ...toPlaywrightOptions(options), ...interactive ? { acceptDownloads: true } : {} };
|
|
640
|
+
}
|
|
465
641
|
async transientContext(url, opts = {}) {
|
|
466
642
|
if (!['http:', 'https:'].includes(new URL(url).protocol))
|
|
467
643
|
throw new Error('browser navigation requires an HTTP(S) URL');
|
|
644
|
+
this.refreshPolicy();
|
|
468
645
|
const profileId = opts.anonymous ? undefined : (opts.authProfile ?? this.config.defaultAuthProfile);
|
|
469
646
|
const profile = profileId ? this.authProfiles.resolve(profileId, url) : undefined;
|
|
470
647
|
const rulePack = resolveRulePack(this.config.rulePacks, opts.rulePack, url);
|
|
@@ -479,7 +656,8 @@ export class BrowserService {
|
|
|
479
656
|
origins: (state.origins ?? []).filter((origin) => hostAllowed(new URL(origin.origin).hostname, profile.allowedDomains)),
|
|
480
657
|
};
|
|
481
658
|
}
|
|
482
|
-
const
|
|
659
|
+
const applied = opts.context ?? this.config.contextDefaults;
|
|
660
|
+
const contextOptions = { ...stateOptions, ...(scopedState ? { storageState: scopedState } : {}), ...this.contextCreationOptions(applied, opts.interactive === true) };
|
|
483
661
|
let context;
|
|
484
662
|
for (let attempt = 0; attempt < 2; attempt++) {
|
|
485
663
|
const browser = await this.ensure();
|
|
@@ -503,7 +681,7 @@ export class BrowserService {
|
|
|
503
681
|
await context.close().catch(() => { });
|
|
504
682
|
throw error;
|
|
505
683
|
}
|
|
506
|
-
return { context, ...profile ? { profile } : {}, ...rulePack ? { rulePack } : {} };
|
|
684
|
+
return { context, contextOptions: applied, ...profile ? { profile } : {}, ...rulePack ? { rulePack } : {} };
|
|
507
685
|
}
|
|
508
686
|
async persistAndClose(session) {
|
|
509
687
|
try {
|
|
@@ -640,16 +818,11 @@ export class BrowserService {
|
|
|
640
818
|
await this.persistAndClose(session);
|
|
641
819
|
}
|
|
642
820
|
}
|
|
643
|
-
// ──
|
|
821
|
+
// ── opencli (optional) ───────────────────────────────────────────────────────
|
|
644
822
|
opencliAvailable() {
|
|
645
823
|
if (!this.config.enabled || !this.config.opencliEnabled)
|
|
646
824
|
return false;
|
|
647
|
-
|
|
648
|
-
return fs.existsSync(opencliEntryPath());
|
|
649
|
-
}
|
|
650
|
-
catch {
|
|
651
|
-
return false;
|
|
652
|
-
}
|
|
825
|
+
return opencliLocation() !== undefined;
|
|
653
826
|
}
|
|
654
827
|
opencli(args, opts = {}) {
|
|
655
828
|
if (!this.config.enabled)
|
|
@@ -657,7 +830,7 @@ export class BrowserService {
|
|
|
657
830
|
if (!this.config.opencliEnabled)
|
|
658
831
|
return Promise.resolve({ code: -1, stdout: '', stderr: 'dsh-browser: OpenCLI is disabled', timedOut: false });
|
|
659
832
|
if (!this.opencliAvailable())
|
|
660
|
-
return Promise.resolve({ code: -1, stdout: '', stderr: 'dsh-browser:
|
|
833
|
+
return Promise.resolve({ code: -1, stdout: '', stderr: 'dsh-browser: ' + opencliMissingMessage(), timedOut: false });
|
|
661
834
|
if (args.length < 1 || args.length > 40 || args.some(arg => typeof arg !== 'string' || arg.length > 2_000)) {
|
|
662
835
|
return Promise.resolve({ code: -1, stdout: '', stderr: 'dsh-browser: OpenCLI requires 1 to 40 arguments, each at most 2000 characters', timedOut: false });
|
|
663
836
|
}
|
|
@@ -689,13 +862,15 @@ export class BrowserService {
|
|
|
689
862
|
url.hash = '';
|
|
690
863
|
return url.href;
|
|
691
864
|
});
|
|
692
|
-
|
|
693
|
-
const
|
|
694
|
-
|
|
695
|
-
|
|
865
|
+
this.refreshPolicy();
|
|
866
|
+
const limits = this.usageGovernor.policy;
|
|
867
|
+
const maxPages = opts.maxPages ?? limits.maxPagesPerRun;
|
|
868
|
+
const maxDepth = opts.maxDepth ?? limits.maxDepth;
|
|
869
|
+
if (!Number.isInteger(maxPages) || maxPages < 1 || maxPages > limits.maxPagesPerRun) {
|
|
870
|
+
throw new Error('browser crawl maxPages must be from 1 to configured usagePolicy.maxPagesPerRun (' + limits.maxPagesPerRun + ')');
|
|
696
871
|
}
|
|
697
|
-
if (!Number.isInteger(maxDepth) || maxDepth < 0 || maxDepth >
|
|
698
|
-
throw new Error('browser crawl maxDepth must be from 0 to configured usagePolicy.maxDepth (' +
|
|
872
|
+
if (!Number.isInteger(maxDepth) || maxDepth < 0 || maxDepth > limits.maxDepth) {
|
|
873
|
+
throw new Error('browser crawl maxDepth must be from 0 to configured usagePolicy.maxDepth (' + limits.maxDepth + ')');
|
|
699
874
|
}
|
|
700
875
|
const maxCharsPerPage = Math.min(Math.max(opts.maxCharsPerPage ?? 20_000, 1_000), 50_000);
|
|
701
876
|
const sameOrigin = opts.sameOrigin ?? true;
|
|
@@ -847,39 +1022,73 @@ export class BrowserService {
|
|
|
847
1022
|
* or mutating another's page, cookies, console or network log.
|
|
848
1023
|
*/
|
|
849
1024
|
async ensureActivePage(targetUrl, opts = {}) {
|
|
1025
|
+
return (await this.ensurePage(targetUrl, opts)).page;
|
|
1026
|
+
}
|
|
1027
|
+
/**
|
|
1028
|
+
* {@link ensureActivePage} plus what it had to do to the session's context. `context` carries the per-call creation
|
|
1029
|
+
* options of `target.open`: an option the call leaves out keeps what the live context has (a context is only
|
|
1030
|
+
* rebuilt for a value that is different, never because a later call said nothing), and a new context starts from
|
|
1031
|
+
* the configured defaults. `rebuilt` names what differed when a live page was closed to make room for the new context.
|
|
1032
|
+
*/
|
|
1033
|
+
async ensurePage(targetUrl, opts = {}) {
|
|
850
1034
|
if (targetUrl && !['http:', 'https:'].includes(new URL(targetUrl).protocol))
|
|
851
1035
|
throw new Error('browser navigation requires an HTTP(S) URL');
|
|
1036
|
+
this.refreshPolicy();
|
|
852
1037
|
const state = this.state(opts.session);
|
|
853
1038
|
if (state.page && (state.page.isClosed() || !this.browserConnected()))
|
|
854
1039
|
await this.closePage(state);
|
|
1040
|
+
let rebuilt;
|
|
1041
|
+
let wanted = effectiveContextOptions(this.config.contextDefaults, opts.context);
|
|
855
1042
|
if (state.page && !state.page.isClosed()) {
|
|
856
1043
|
if (!targetUrl)
|
|
857
|
-
return state.page;
|
|
858
|
-
|
|
1044
|
+
return { page: state.page };
|
|
1045
|
+
const applied = state.contextOptions ?? this.config.contextDefaults;
|
|
1046
|
+
wanted = { ...applied, ...opts.context };
|
|
1047
|
+
// The login state a context holds was cut to the profile's domains when the context was created. A policy
|
|
1048
|
+
// change that widened the profile (or moved its state file) afterwards is not in that context, so the next open
|
|
1049
|
+
// starts a new one; narrowing already closed the session in changePolicy.
|
|
1050
|
+
const liveProfile = state.profile ? this.authProfiles.get(state.profile.id) : undefined;
|
|
1051
|
+
const scopeChanged = !!state.profile && !!liveProfile
|
|
1052
|
+
&& (liveProfile.storageStatePath !== state.profile.storageStatePath
|
|
1053
|
+
|| [...liveProfile.allowedDomains ?? []].sort().join('\n') !== [...state.profile.allowedDomains ?? []].sort().join('\n'));
|
|
1054
|
+
const sameLogin = (opts.authProfile ?? this.config.defaultAuthProfile) === state.profile?.id && opts.rulePack === state.rulePack?.id;
|
|
1055
|
+
const changedOptions = differingContextOptions(applied, wanted);
|
|
1056
|
+
if (sameLogin && !scopeChanged && changedOptions.length === 0) {
|
|
859
1057
|
if (state.profile)
|
|
860
1058
|
this.authProfiles.resolve(state.profile.id, targetUrl);
|
|
861
1059
|
if (state.rulePack)
|
|
862
1060
|
resolveRulePack(this.config.rulePacks, state.rulePack.id, targetUrl);
|
|
863
|
-
return state.page;
|
|
1061
|
+
return { page: state.page };
|
|
864
1062
|
}
|
|
1063
|
+
rebuilt = [
|
|
1064
|
+
...changedOptions,
|
|
1065
|
+
...(opts.authProfile ?? this.config.defaultAuthProfile) === state.profile?.id ? [] : ['authProfile'],
|
|
1066
|
+
...scopeChanged && (opts.authProfile ?? this.config.defaultAuthProfile) === state.profile?.id ? ['authProfileScope'] : [],
|
|
1067
|
+
...opts.rulePack === state.rulePack?.id ? [] : ['rulePack'],
|
|
1068
|
+
];
|
|
865
1069
|
await this.closePage(state);
|
|
866
1070
|
}
|
|
867
1071
|
if (targetUrl) {
|
|
868
|
-
const session = await this.transientContext(targetUrl, opts);
|
|
1072
|
+
const session = await this.transientContext(targetUrl, { ...opts, context: wanted, interactive: true });
|
|
869
1073
|
state.context = session.context;
|
|
870
1074
|
state.profile = session.profile;
|
|
871
1075
|
state.rulePack = session.rulePack;
|
|
1076
|
+
state.contextOptions = session.contextOptions;
|
|
872
1077
|
}
|
|
873
1078
|
else {
|
|
874
1079
|
const browser = await this.ensure();
|
|
875
|
-
state.context = await browser.newContext(
|
|
1080
|
+
state.context = await browser.newContext({
|
|
1081
|
+
...storageStateOptions(this.config.storageStatePath, 'global'),
|
|
1082
|
+
...this.contextCreationOptions(wanted, true),
|
|
1083
|
+
});
|
|
1084
|
+
state.contextOptions = wanted;
|
|
876
1085
|
}
|
|
877
1086
|
// A popup (window.open, target=_blank) joins this session's page list; it never takes over the active page by itself.
|
|
878
1087
|
state.context.on('page', (opened) => { if (state.context)
|
|
879
1088
|
this.trackPage(state, opened); });
|
|
880
1089
|
state.page = await state.context.newPage();
|
|
881
1090
|
this.trackPage(state, state.page);
|
|
882
|
-
return state.page;
|
|
1091
|
+
return { page: state.page, ...rebuilt ? { rebuilt } : {} };
|
|
883
1092
|
}
|
|
884
1093
|
/**
|
|
885
1094
|
* Add a page to the session's list and wire its capture and close handling.
|
|
@@ -914,11 +1123,14 @@ export class BrowserService {
|
|
|
914
1123
|
state.context = undefined;
|
|
915
1124
|
state.profile = undefined;
|
|
916
1125
|
state.rulePack = undefined;
|
|
1126
|
+
state.contextOptions = undefined;
|
|
917
1127
|
if (context)
|
|
918
1128
|
void this.persistAndClose({ context, ...profile ? { profile } : {} }).catch(() => { });
|
|
919
1129
|
};
|
|
920
1130
|
page.on('close', gone);
|
|
921
1131
|
page.on('crash', gone);
|
|
1132
|
+
// The listener lives and dies with the page (and so with the context); the record goes to this session's own list.
|
|
1133
|
+
page.on('download', (download) => { state.downloadJobs.set(download, this.saveBrowserDownload(state, download)); });
|
|
922
1134
|
page.on('console', (message) => {
|
|
923
1135
|
if (!state.captureConsole)
|
|
924
1136
|
return;
|
|
@@ -1054,7 +1266,9 @@ export class BrowserService {
|
|
|
1054
1266
|
return state;
|
|
1055
1267
|
}
|
|
1056
1268
|
async open(url, opts = {}) {
|
|
1057
|
-
|
|
1269
|
+
// Checked before anything is created or closed: a bad option must not cost the session its page.
|
|
1270
|
+
const wanted = resolveCallOptions(opts);
|
|
1271
|
+
const { page, rebuilt } = await this.ensurePage(url, { ...opts, context: wanted });
|
|
1058
1272
|
const state = this.state(opts.session);
|
|
1059
1273
|
this.resetCapture(state, opts.capture);
|
|
1060
1274
|
page.setDefaultTimeout(30_000);
|
|
@@ -1063,7 +1277,19 @@ export class BrowserService {
|
|
|
1063
1277
|
await applyRuleSteps(page, state.rulePack);
|
|
1064
1278
|
if (opts.waitMs)
|
|
1065
1279
|
await page.waitForTimeout(opts.waitMs);
|
|
1066
|
-
|
|
1280
|
+
const result = await this.readState(page, true);
|
|
1281
|
+
const applied = describeContextOptions(state.contextOptions ?? {});
|
|
1282
|
+
return {
|
|
1283
|
+
...result,
|
|
1284
|
+
// The whole parameter set in effect, always: the configured defaults with what this call (or the context from an
|
|
1285
|
+
// earlier call) changed, each value tagged with where it came from.
|
|
1286
|
+
contextOptions: applied,
|
|
1287
|
+
contextSources: contextSources(state.contextOptions ?? {}, this.config.contextDefaults, wanted),
|
|
1288
|
+
...rebuilt ? { contextRebuilt: {
|
|
1289
|
+
changed: rebuilt,
|
|
1290
|
+
note: 'The session\'s previous context was closed and a new one created because ' + rebuilt.join(', ') + ' changed; its pages (popups included) are gone and cookies not kept in an auth profile were dropped.',
|
|
1291
|
+
} } : {},
|
|
1292
|
+
};
|
|
1067
1293
|
}
|
|
1068
1294
|
async click(target, opts = {}) {
|
|
1069
1295
|
const page = await this.ensureActivePage(undefined, { session: opts.session });
|
|
@@ -1365,6 +1591,211 @@ export class BrowserService {
|
|
|
1365
1591
|
}, request.maxBytes, raw?.counts ?? {}, request.maxItems);
|
|
1366
1592
|
return { ...fitted.record, ...fitted.truncation ? { truncation: fitted.truncation } : {} };
|
|
1367
1593
|
}
|
|
1594
|
+
// ── downloads (state is per session; files go only to config.downloadDir) ─────────────────
|
|
1595
|
+
pushDownload(state, record) {
|
|
1596
|
+
state.downloads.push(record);
|
|
1597
|
+
if (state.downloads.length > DOWNLOAD_HISTORY_LIMIT)
|
|
1598
|
+
state.downloads.splice(0, state.downloads.length - DOWNLOAD_HISTORY_LIMIT);
|
|
1599
|
+
return record;
|
|
1600
|
+
}
|
|
1601
|
+
failedDownload(state, url, suggestedFilename, source, error) {
|
|
1602
|
+
return this.pushDownload(state, {
|
|
1603
|
+
id: 'd' + state.nextDownload++, url: redactCaptureUrl(url), status: 'failed', source, suggestedFilename,
|
|
1604
|
+
error: redactCaptureText(String(error instanceof Error ? error.message : error), 500), timestamp: new Date().toISOString(),
|
|
1605
|
+
});
|
|
1606
|
+
}
|
|
1607
|
+
/**
|
|
1608
|
+
* Save one browser download event (navigation to an attachment, a click, a script-created link) into the
|
|
1609
|
+
* download directory and record the outcome in the session that owns the page. Never rejects: a failure is a
|
|
1610
|
+
* `failed` record with the partial file removed, so nothing half-written is ever reported as a result.
|
|
1611
|
+
*/
|
|
1612
|
+
async saveBrowserDownload(state, download) {
|
|
1613
|
+
const requested = state.downloadFilename;
|
|
1614
|
+
state.downloadFilename = undefined;
|
|
1615
|
+
const url = String(download.url?.() ?? '');
|
|
1616
|
+
const suggested = String(download.suggestedFilename?.() ?? '');
|
|
1617
|
+
const dir = this.config.downloadDir;
|
|
1618
|
+
let temporary;
|
|
1619
|
+
try {
|
|
1620
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
1621
|
+
const name = resolveDownloadFilename({ ...requested !== undefined ? { requested } : {}, suggested, url });
|
|
1622
|
+
temporary = temporaryDownloadPath(dir);
|
|
1623
|
+
const source = await download.createReadStream();
|
|
1624
|
+
if (!source)
|
|
1625
|
+
throw new Error('the browser did not provide the downloaded bytes: ' + String(await download.failure?.() ?? 'unknown reason'));
|
|
1626
|
+
const { bytes, sha256 } = await streamToFile(source, temporary, this.config.downloadMaxBytes);
|
|
1627
|
+
const failure = await download.failure?.();
|
|
1628
|
+
if (failure)
|
|
1629
|
+
throw new Error('the download failed: ' + String(failure));
|
|
1630
|
+
if (bytes === 0)
|
|
1631
|
+
throw new Error('the download produced zero bytes; nothing was saved');
|
|
1632
|
+
const saved = claimDownloadPath(dir, name, temporary);
|
|
1633
|
+
temporary = undefined;
|
|
1634
|
+
return this.pushDownload(state, {
|
|
1635
|
+
id: 'd' + state.nextDownload++, url: redactCaptureUrl(url), status: 'saved', source: 'browser', suggestedFilename: suggested || name,
|
|
1636
|
+
path: saved, bytes, sha256, timestamp: new Date().toISOString(),
|
|
1637
|
+
});
|
|
1638
|
+
}
|
|
1639
|
+
catch (error) {
|
|
1640
|
+
removeQuietly(temporary);
|
|
1641
|
+
if (error instanceof DownloadLimitError)
|
|
1642
|
+
await download.cancel?.().catch(() => { });
|
|
1643
|
+
return this.failedDownload(state, url, suggested, 'browser', error);
|
|
1644
|
+
}
|
|
1645
|
+
}
|
|
1646
|
+
/**
|
|
1647
|
+
* `target.download`: save a URL into the download directory.
|
|
1648
|
+
*
|
|
1649
|
+
* First an HTTP request made through the session's own context (its cookies and login state, its proxy and
|
|
1650
|
+
* user agent); if that does not produce bytes, the URL is opened in a throwaway page of the same context and the
|
|
1651
|
+
* attachment download that navigation triggers is captured. A request that returns nothing is an error, never a
|
|
1652
|
+
* zero-byte file. The session's own page is not navigated.
|
|
1653
|
+
*/
|
|
1654
|
+
async download(url, opts = {}) {
|
|
1655
|
+
this.assertEnabled();
|
|
1656
|
+
let parsed;
|
|
1657
|
+
try {
|
|
1658
|
+
parsed = new URL(url);
|
|
1659
|
+
}
|
|
1660
|
+
catch {
|
|
1661
|
+
throw new Error('target.download requires an HTTP(S) URL');
|
|
1662
|
+
}
|
|
1663
|
+
if (!['http:', 'https:'].includes(parsed.protocol))
|
|
1664
|
+
throw new Error('target.download requires an HTTP(S) URL');
|
|
1665
|
+
// Both are checked before anything is fetched or created.
|
|
1666
|
+
const requested = opts.filename !== undefined ? sanitizeDownloadName(opts.filename, true) : undefined;
|
|
1667
|
+
const timeoutMs = opts.timeoutMs ?? 60_000;
|
|
1668
|
+
if (!Number.isFinite(timeoutMs) || timeoutMs < 1_000 || timeoutMs > 150_000)
|
|
1669
|
+
throw new Error('target.download timeoutMs must be between 1,000 and 150,000 ms');
|
|
1670
|
+
const maxBytes = this.config.downloadMaxBytes;
|
|
1671
|
+
const dir = this.config.downloadDir;
|
|
1672
|
+
this.refreshPolicy();
|
|
1673
|
+
const state = this.state(opts.session);
|
|
1674
|
+
if (state.page && (state.page.isClosed() || !this.browserConnected()))
|
|
1675
|
+
await this.closePage(state);
|
|
1676
|
+
if (state.page && !state.page.isClosed()) {
|
|
1677
|
+
// The session's login state may only be sent where its auth profile is allowed to go; open() applies the same rule.
|
|
1678
|
+
if (state.profile)
|
|
1679
|
+
this.authProfiles.resolve(state.profile.id, url);
|
|
1680
|
+
}
|
|
1681
|
+
else {
|
|
1682
|
+
await this.ensureActivePage(url, { session: opts.session });
|
|
1683
|
+
}
|
|
1684
|
+
const context = state.context;
|
|
1685
|
+
if (!context)
|
|
1686
|
+
throw new Error('dsh-browser: no browser context is available for the download');
|
|
1687
|
+
const deadline = Date.now() + timeoutMs;
|
|
1688
|
+
const signal = opts.signal;
|
|
1689
|
+
let requestFailure = '';
|
|
1690
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
1691
|
+
// 1. A request with the session's cookies.
|
|
1692
|
+
let temporary;
|
|
1693
|
+
try {
|
|
1694
|
+
const response = await this.usageGovernor.run(url, () => context.request.get(url, { timeout: timeoutMs, failOnStatusCode: false }), signal);
|
|
1695
|
+
const status = Number(response.status?.() ?? 0);
|
|
1696
|
+
if (status >= 200 && status < 400)
|
|
1697
|
+
this.usageGovernor.noteResponse(url, status);
|
|
1698
|
+
else if ([429, 502, 503, 504].includes(status)) {
|
|
1699
|
+
const retryAfter = String(response.headers?.()?.['retry-after'] ?? '');
|
|
1700
|
+
this.usageGovernor.noteResponse(url, status, /^\d+$/.test(retryAfter) ? Number(retryAfter) * 1000 : undefined);
|
|
1701
|
+
}
|
|
1702
|
+
if (!response.ok())
|
|
1703
|
+
throw new Error(`the server answered HTTP ${status}`);
|
|
1704
|
+
const finalUrl = String(response.url?.() ?? url);
|
|
1705
|
+
const final = new URL(finalUrl);
|
|
1706
|
+
if (!['http:', 'https:'].includes(final.protocol))
|
|
1707
|
+
throw new Error('the request ended on a non-HTTP(S) URL');
|
|
1708
|
+
if (state.profile)
|
|
1709
|
+
this.authProfiles.resolve(state.profile.id, finalUrl);
|
|
1710
|
+
const headers = response.headers?.() ?? {};
|
|
1711
|
+
const declared = Number(headers['content-length']);
|
|
1712
|
+
if (Number.isFinite(declared) && declared > maxBytes)
|
|
1713
|
+
throw new DownloadLimitError(maxBytes);
|
|
1714
|
+
const body = await response.body();
|
|
1715
|
+
if (body.length === 0)
|
|
1716
|
+
throw new Error('the response body was empty (zero bytes)');
|
|
1717
|
+
const suggested = contentDispositionFilename(headers['content-disposition']);
|
|
1718
|
+
const name = resolveDownloadFilename({ ...requested !== undefined ? { requested } : {}, suggested, url: finalUrl });
|
|
1719
|
+
temporary = temporaryDownloadPath(dir);
|
|
1720
|
+
const { bytes, sha256 } = writeBufferToFile(body, temporary, maxBytes);
|
|
1721
|
+
const saved = claimDownloadPath(dir, name, temporary);
|
|
1722
|
+
temporary = undefined;
|
|
1723
|
+
const record = this.pushDownload(state, {
|
|
1724
|
+
id: 'd' + state.nextDownload++, url: redactCaptureUrl(finalUrl), status: 'saved', source: 'request',
|
|
1725
|
+
suggestedFilename: suggested || name, path: saved, bytes, sha256, timestamp: new Date().toISOString(),
|
|
1726
|
+
});
|
|
1727
|
+
return downloadResult(record, url);
|
|
1728
|
+
}
|
|
1729
|
+
catch (error) {
|
|
1730
|
+
removeQuietly(temporary);
|
|
1731
|
+
if (signal?.aborted)
|
|
1732
|
+
throw error;
|
|
1733
|
+
// A size or policy failure is final: another route would meet the same limit or send the login state where it must not go.
|
|
1734
|
+
if (error instanceof DownloadLimitError) {
|
|
1735
|
+
this.failedDownload(state, url, '', 'request', error);
|
|
1736
|
+
throw error;
|
|
1737
|
+
}
|
|
1738
|
+
if (/auth profile .* is not allowed/.test(String(error))) {
|
|
1739
|
+
this.failedDownload(state, url, '', 'request', error);
|
|
1740
|
+
throw error;
|
|
1741
|
+
}
|
|
1742
|
+
requestFailure = String(error instanceof Error ? error.message : error).split('\n')[0].slice(0, 300);
|
|
1743
|
+
}
|
|
1744
|
+
// 2. Navigation: the attachment download the URL triggers, captured from a page of the same context.
|
|
1745
|
+
const remaining = Math.max(deadline - Date.now(), 5_000);
|
|
1746
|
+
const page = await context.newPage();
|
|
1747
|
+
this.trackPage(state, page);
|
|
1748
|
+
state.downloadFilename = requested;
|
|
1749
|
+
let recorded = false;
|
|
1750
|
+
try {
|
|
1751
|
+
page.setDefaultTimeout(remaining);
|
|
1752
|
+
const waiting = page.waitForEvent('download', { timeout: remaining }).then((value) => value, () => undefined);
|
|
1753
|
+
let navigation = {};
|
|
1754
|
+
const navigated = this.usageGovernor.run(url, () => page.goto(url, { waitUntil: 'commit', timeout: remaining }), signal)
|
|
1755
|
+
.then((response) => { navigation = { response }; }, (error) => { navigation = { error }; });
|
|
1756
|
+
// A download makes goto fail with "Download is starting"; a page that simply loaded leaves a short grace for the event.
|
|
1757
|
+
const download = await Promise.race([
|
|
1758
|
+
waiting,
|
|
1759
|
+
navigated.then(() => Promise.race([waiting, new Promise(resolve => setTimeout(() => resolve(undefined), 1_500))])),
|
|
1760
|
+
]);
|
|
1761
|
+
if (signal?.aborted)
|
|
1762
|
+
throw new Error('target.download aborted');
|
|
1763
|
+
if (!download) {
|
|
1764
|
+
const reason = navigation.error
|
|
1765
|
+
? 'navigation failed: ' + String(navigation.error?.message ?? navigation.error).split('\n')[0].slice(0, 200)
|
|
1766
|
+
: `the URL loaded as a page (HTTP ${Number(navigation.response?.status?.() ?? 0)}) and did not start a download`;
|
|
1767
|
+
throw new Error(`could not download ${redactCaptureUrl(url)}: request failed (${requestFailure}); ${reason}`);
|
|
1768
|
+
}
|
|
1769
|
+
const job = state.downloadJobs.get(download);
|
|
1770
|
+
const record = job ? await job : undefined;
|
|
1771
|
+
// The download listener records what it saw, failures included; only a failure it never saw is recorded below.
|
|
1772
|
+
recorded = !!record;
|
|
1773
|
+
if (!record)
|
|
1774
|
+
throw new Error(`could not download ${redactCaptureUrl(url)}: the browser download was not captured`);
|
|
1775
|
+
if (record.status !== 'saved')
|
|
1776
|
+
throw new Error(`could not download ${redactCaptureUrl(url)}: request failed (${requestFailure}); ${record.error ?? 'download failed'}`);
|
|
1777
|
+
return downloadResult(record, url);
|
|
1778
|
+
}
|
|
1779
|
+
catch (error) {
|
|
1780
|
+
if (!recorded)
|
|
1781
|
+
this.failedDownload(state, url, '', 'request', error);
|
|
1782
|
+
throw error;
|
|
1783
|
+
}
|
|
1784
|
+
finally {
|
|
1785
|
+
state.downloadFilename = undefined;
|
|
1786
|
+
await page.close().catch(() => { });
|
|
1787
|
+
}
|
|
1788
|
+
}
|
|
1789
|
+
/** `inspect.downloads`: this session's download records (newest last); reading never creates a session bucket. */
|
|
1790
|
+
downloadRecords(opts = {}) {
|
|
1791
|
+
const limit = Math.min(Math.max(Math.trunc(opts.limit ?? 20), 1), DOWNLOAD_HISTORY_LIMIT);
|
|
1792
|
+
const state = this.peek(opts.session);
|
|
1793
|
+
const all = state?.downloads ?? [];
|
|
1794
|
+
const records = all.slice(-limit).map(record => ({ ...record }));
|
|
1795
|
+
if (opts.clear && state)
|
|
1796
|
+
state.downloads = [];
|
|
1797
|
+
return { downloadDir: this.config.downloadDir, records, total: all.length };
|
|
1798
|
+
}
|
|
1368
1799
|
consoleMessages(opts = {}) {
|
|
1369
1800
|
const severities = ['debug', 'log', 'info', 'warning', 'error'];
|
|
1370
1801
|
const threshold = opts.level ? severities.indexOf(opts.level) : 0;
|
|
@@ -1488,6 +1919,7 @@ export class BrowserService {
|
|
|
1488
1919
|
session.context = undefined;
|
|
1489
1920
|
session.profile = undefined;
|
|
1490
1921
|
session.rulePack = undefined;
|
|
1922
|
+
session.contextOptions = undefined;
|
|
1491
1923
|
session.capturedConsole = [];
|
|
1492
1924
|
session.capturedRequests = [];
|
|
1493
1925
|
if (page)
|
|
@@ -1509,15 +1941,9 @@ export class BrowserService {
|
|
|
1509
1941
|
catch {
|
|
1510
1942
|
chromiumInstalled = false;
|
|
1511
1943
|
}
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
resolvedOpencliEntryPath = path.resolve(opencliEntryPath());
|
|
1516
|
-
opencliInstalled = fs.existsSync(resolvedOpencliEntryPath);
|
|
1517
|
-
}
|
|
1518
|
-
catch {
|
|
1519
|
-
opencliInstalled = false;
|
|
1520
|
-
}
|
|
1944
|
+
const opencli = opencliLocation();
|
|
1945
|
+
const opencliInstalled = opencli !== undefined;
|
|
1946
|
+
const resolvedOpencliEntryPath = opencli ? path.resolve(opencli.entry) : undefined;
|
|
1521
1947
|
const runtimeWarnings = this.config.browserRuntime === 'patchright'
|
|
1522
1948
|
? [
|
|
1523
1949
|
'Patchright is Chromium-only and disables Playwright console APIs to avoid Runtime.enable detection.',
|
|
@@ -1526,11 +1952,21 @@ export class BrowserService {
|
|
|
1526
1952
|
: []),
|
|
1527
1953
|
]
|
|
1528
1954
|
: [];
|
|
1529
|
-
|
|
1530
|
-
|
|
1955
|
+
let plan;
|
|
1956
|
+
try {
|
|
1957
|
+
plan = this.browserPlan(loadBrowserRuntime(this.config.browserRuntime));
|
|
1531
1958
|
}
|
|
1532
|
-
|
|
1533
|
-
|
|
1959
|
+
catch {
|
|
1960
|
+
plan = this.browserPlan(undefined);
|
|
1961
|
+
}
|
|
1962
|
+
if (!plan.ready && (this.config.channel === 'chromium' || !!this.config.executablePath)) {
|
|
1963
|
+
runtimeWarnings.push(`Expected Chromium executable is missing: ${chromiumExecutablePath ?? '(unknown)'}. ${this.launchFailure(plan)}`);
|
|
1964
|
+
}
|
|
1965
|
+
// OpenCLI is optional (it is not a dependency of this package), so its absence is reported as a hint, not a warning.
|
|
1966
|
+
runtimeWarnings.push(...this.config.configWarnings);
|
|
1967
|
+
this.refreshPolicy();
|
|
1968
|
+
if (this.policyStore.warning)
|
|
1969
|
+
runtimeWarnings.push(this.policyStore.warning);
|
|
1534
1970
|
return {
|
|
1535
1971
|
enabled: this.config.enabled,
|
|
1536
1972
|
channel: this.config.channel,
|
|
@@ -1540,6 +1976,8 @@ export class BrowserService {
|
|
|
1540
1976
|
opencliEnabled: this.config.opencliEnabled,
|
|
1541
1977
|
opencliInstalled,
|
|
1542
1978
|
...resolvedOpencliEntryPath ? { opencliEntryPath: resolvedOpencliEntryPath } : {},
|
|
1979
|
+
...opencli ? { opencliSource: opencli.source } : {},
|
|
1980
|
+
...this.config.opencliEnabled && !opencliInstalled ? { opencliInstallHint: opencliMissingMessage() } : {},
|
|
1543
1981
|
automationMode: this.config.automationMode,
|
|
1544
1982
|
exposedTools: configuredBrowserTools(this.config.automationMode, this.config.automationAssets, this.config.enabled, this.config.toolSurface),
|
|
1545
1983
|
exposedActions: configuredBrowserActions(this.config.automationMode, this.config.automationAssets, this.config.enabled),
|
|
@@ -1548,10 +1986,22 @@ export class BrowserService {
|
|
|
1548
1986
|
externalUserscriptPolicy: !this.config.enabled || this.config.automationMode === 'read-only' ? 'deny' : this.config.automationMode === 'unrestricted' ? 'allow' : 'ask',
|
|
1549
1987
|
pageEvaluatePolicy: !this.config.enabled || this.config.automationMode === 'read-only' ? 'deny' : this.config.automationMode === 'unrestricted' ? 'allow' : 'ask',
|
|
1550
1988
|
fileUploadPolicy: !this.config.enabled || this.config.automationMode === 'read-only' ? 'deny' : this.config.automationMode === 'unrestricted' ? 'allow' : 'ask',
|
|
1989
|
+
downloadPolicy: !this.config.enabled || this.config.automationMode === 'read-only' ? 'deny' : this.config.automationMode === 'unrestricted' ? 'allow' : 'ask',
|
|
1990
|
+
downloadDir: this.config.downloadDir,
|
|
1991
|
+
downloadMaxBytes: this.config.downloadMaxBytes,
|
|
1992
|
+
contextDefaults: describeContextOptions(this.config.contextDefaults),
|
|
1551
1993
|
opencliRunPolicy: !this.config.enabled || this.config.automationMode === 'read-only' ? 'deny' : this.config.automationMode === 'unrestricted' ? 'allow' : 'ask',
|
|
1552
1994
|
chromiumInstalled,
|
|
1553
1995
|
...chromiumExecutablePath ? { chromiumExecutablePath } : {},
|
|
1554
|
-
|
|
1996
|
+
browser: {
|
|
1997
|
+
ready: plan.ready,
|
|
1998
|
+
source: plan.source,
|
|
1999
|
+
...plan.path ? { path: plan.path } : {},
|
|
2000
|
+
...plan.note ? { note: plan.note } : {},
|
|
2001
|
+
...!plan.ready ? { installAdvice: this.launchFailure(plan) } : {},
|
|
2002
|
+
},
|
|
2003
|
+
systemBrowsers: this.systemBrowsers().map(({ channel, name, path: file }) => ({ channel, name, path: file })),
|
|
2004
|
+
usagePolicy: this.usageGovernor.policy,
|
|
1555
2005
|
usageGovernor: this.usageGovernor.snapshot(),
|
|
1556
2006
|
authProfiles: this.authProfiles.list(),
|
|
1557
2007
|
rulePacks: Object.keys(this.config.rulePacks).sort(),
|