@procedures/browser 0.55.1 → 0.56.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,215 @@
1
+ /**
2
+ * The bookmarks files of Chrome. Edge and Chromium write the same format. This file reads the JSON
3
+ * of a bookmarks file, and gives a tree of folders with their sites. It also reads the names of the
4
+ * profiles from the file `Local State`.
5
+ *
6
+ * This file imports no Node built-in, so a page can use it: the entry `@procedures/browser/bookmarks`.
7
+ * The search for the files on the disk is in `profiles.ts`, on the root entry.
8
+ */
9
+
10
+ /** The deepest folder that the parser reads. A deeper folder gives a problem, and the parser does not read it. */
11
+ export const MAX_BOOKMARK_DEPTH = 100;
12
+
13
+ /** The largest bookmarks file that a reader accepts, in bytes. A larger file is not a bookmarks file. */
14
+ export const MAX_BOOKMARKS_BYTES: number = 64 * 1024 * 1024;
15
+
16
+ /**
17
+ * The kind of a bookmarks file. `profile` is the file `Bookmarks` of a profile. `account` is the
18
+ * file `AccountBookmarks`: the bookmarks of a Google account with no sync.
19
+ */
20
+ export type BookmarkFileKind = "profile" | "account";
21
+
22
+ /** The bookmarks files of a profile, by the name of the file, with their kinds. */
23
+ export const BOOKMARK_FILES: Readonly<Record<string, BookmarkFileKind>> = { Bookmarks: "profile", AccountBookmarks: "account" };
24
+
25
+ /** A bookmarks file, with no path. A host can give this to a page and keep the path. */
26
+ export interface BookmarkSource {
27
+ /** The id of the file. The search gives `<browser>:<profile folder>:<file name>`, for example `chrome:Default:Bookmarks`. */
28
+ readonly id: string;
29
+ /** The name of the profile, as the browser shows it. */
30
+ readonly name: string;
31
+ readonly kind: BookmarkFileKind;
32
+ }
33
+
34
+ /** A bookmark of a web site. */
35
+ export interface Bookmark {
36
+ /** The address of the site: an `http` or `https` address with a host. */
37
+ readonly url: string;
38
+ /** The title of the bookmark. A bookmark with an empty title has no title. */
39
+ readonly title?: string;
40
+ /** The ISO 8601 time when the user added the bookmark. */
41
+ readonly added?: string;
42
+ /** The names of the folders from the root to the folder of the bookmark. */
43
+ readonly path: readonly string[];
44
+ }
45
+
46
+ /** A folder of the bookmarks, with its sites and the folders in it. */
47
+ export interface BookmarkFolder {
48
+ /**
49
+ * The key of the folder, after the prefix of the parse: its `guid`, else its `id`, else its place in the file. The place is
50
+ * the name of the root, then the index of each child, for example `bookmark_bar/2/0`. Each parse of a file gives the same keys.
51
+ */
52
+ readonly key: string;
53
+ readonly name: string;
54
+ /** The names of the folders from the root to this folder. The last name is the name of this folder. */
55
+ readonly path: readonly string[];
56
+ readonly folders: readonly BookmarkFolder[];
57
+ /** The sites directly in this folder. Each site has the path of this folder. */
58
+ readonly sites: readonly Bookmark[];
59
+ /** The count of the sites in this folder and in each folder in it. */
60
+ readonly total: number;
61
+ }
62
+
63
+ /** A part of a bookmarks file that the parser cannot read. */
64
+ export interface BookmarkProblem {
65
+ /** The names of the folders to the part. */
66
+ readonly path: readonly string[];
67
+ readonly message: string;
68
+ }
69
+
70
+ /** The result of the parse of a bookmarks file. */
71
+ export interface ChromeBookmarks {
72
+ /** The root folders, for example "Bookmarks bar" and "Other bookmarks". */
73
+ readonly roots: readonly BookmarkFolder[];
74
+ /** The count of the bookmarks that are not web sites, for example `javascript:` or `chrome://` addresses. */
75
+ readonly skipped: number;
76
+ readonly problems: readonly BookmarkProblem[];
77
+ }
78
+
79
+ /** The options of `parseChromeBookmarks`. */
80
+ export interface ParseChromeBookmarksOptions {
81
+ /** The text before each key. Give two files of one profile different prefixes, so that their keys are different. */
82
+ readonly prefix?: string | undefined;
83
+ }
84
+
85
+ /** The order of the roots that Chrome writes. The roots with other names come after them, in the order of their code units. */
86
+ const ROOT_ORDER = ["bookmark_bar", "other", "synced"];
87
+
88
+ /** The time from 1601-01-01 (the start of the Chrome times) to 1970-01-01, in milliseconds. */
89
+ const WINDOWS_EPOCH_MS = 11_644_473_600_000n;
90
+
91
+ /**
92
+ * A web address has the scheme `http` or `https`, a host, and no spaces. The test does not use `URL`, so it is the same in each
93
+ * runtime. The first character of the host is one class, and the rest is `\S*`. Thus no two quantifiers agree on the same
94
+ * character, and the time of the test is linear in the length of the address.
95
+ */
96
+ const WEB = /^https?:\/\/[^\s/?#]\S*$/i;
97
+
98
+ /**
99
+ * This function gives the ISO 8601 time of a Chrome time, or `undefined` for no time. A Chrome time
100
+ * is a text of the microseconds from 1601-01-01 UTC. The function uses `BigInt`, because the value
101
+ * is larger than a safe integer.
102
+ */
103
+ export function chromeTime(value: unknown): string | undefined {
104
+ if (typeof value !== "string" || !/^\d{1,20}$/.test(value)) return undefined;
105
+ const ms = BigInt(value) / 1000n - WINDOWS_EPOCH_MS;
106
+ // The time 0 means no time. A time before 1990 or after 9999 is not a date of a bookmark.
107
+ if (ms < 631_152_000_000n || ms > 253_402_300_799_999n) return undefined;
108
+ return new Date(Number(ms)).toISOString();
109
+ }
110
+
111
+ interface Node {
112
+ readonly type?: unknown;
113
+ readonly name?: unknown;
114
+ readonly url?: unknown;
115
+ readonly children?: unknown;
116
+ readonly guid?: unknown;
117
+ readonly id?: unknown;
118
+ readonly date_added?: unknown;
119
+ }
120
+
121
+ function isNode(value: unknown): value is Node {
122
+ return typeof value === "object" && value !== null && !Array.isArray(value);
123
+ }
124
+
125
+ /** This function gives the `guid` of a folder, else its `id`. An empty text or a value of another type is not a key. */
126
+ function ownKey(node: Node): string | undefined {
127
+ if (typeof node.guid === "string" && node.guid !== "") return node.guid;
128
+ if (typeof node.id === "string" && node.id !== "") return node.id;
129
+ if (typeof node.id === "number" && Number.isFinite(node.id)) return String(node.id);
130
+ return undefined;
131
+ }
132
+
133
+ function rank(name: string): number {
134
+ const index = ROOT_ORDER.indexOf(name);
135
+ return index === -1 ? ROOT_ORDER.length : index;
136
+ }
137
+
138
+ /**
139
+ * This function reads the JSON value of a Chrome bookmarks file: the file `Bookmarks` or
140
+ * `AccountBookmarks` of a profile. It gives a problem for each part that it cannot read, and it
141
+ * reads the other parts. It keeps only the bookmarks of web sites, and counts the others.
142
+ *
143
+ * @throws Error when the value has no roots, so it is not a bookmarks file
144
+ */
145
+ export function parseChromeBookmarks(value: unknown, options: ParseChromeBookmarksOptions = {}): ChromeBookmarks {
146
+ const prefix = options.prefix ?? "";
147
+ if (!isNode(value) || !isNode((value as { roots?: unknown }).roots)) throw new Error("A Chrome bookmarks file has an object roots.");
148
+ const rootNodes = (value as { roots: Record<string, unknown> }).roots;
149
+ const problems: BookmarkProblem[] = [];
150
+ let skipped = 0;
151
+
152
+ // The place of a folder is the name of its root, then the index of each child to it. It is unique in a file.
153
+ const folderOf = (node: Node, parent: readonly string[], fallback: string, depth: number, place: string): BookmarkFolder => {
154
+ const name = typeof node.name === "string" && node.name !== "" ? node.name : fallback;
155
+ const path = [...parent, name];
156
+ const own = ownKey(node) ?? place;
157
+ const folders: BookmarkFolder[] = [];
158
+ const sites: Bookmark[] = [];
159
+ let total = 0;
160
+ const children = Array.isArray(node.children) ? (node.children as unknown[]) : [];
161
+ if (node.children !== undefined && !Array.isArray(node.children)) problems.push({ path, message: "The children of a folder must be a list." });
162
+ for (const [index, child] of children.entries()) {
163
+ if (!isNode(child)) {
164
+ problems.push({ path, message: "A bookmark must be an object." });
165
+ } else if (child.type === "url") {
166
+ if (typeof child.url !== "string" || !WEB.test(child.url)) {
167
+ skipped++;
168
+ continue;
169
+ }
170
+ const title = typeof child.name === "string" && child.name.trim() !== "" ? child.name : undefined;
171
+ const added = chromeTime(child.date_added);
172
+ sites.push({ url: child.url, ...(title === undefined ? {} : { title }), ...(added === undefined ? {} : { added }), path });
173
+ total++;
174
+ } else if (child.type === "folder") {
175
+ if (depth >= MAX_BOOKMARK_DEPTH) {
176
+ problems.push({ path, message: `A folder is deeper than ${MAX_BOOKMARK_DEPTH} folders. The parser does not read it.` });
177
+ continue;
178
+ }
179
+ const folder = folderOf(child, path, "Folder", depth + 1, `${place}/${index}`);
180
+ folders.push(folder);
181
+ total += folder.total;
182
+ } else {
183
+ problems.push({ path, message: `A bookmark has the type ${String(child.type)}, not url or folder.` });
184
+ }
185
+ }
186
+ return { key: `${prefix}${own}`, name, path, folders, sites, total };
187
+ };
188
+
189
+ // The order of the code units, not of a locale, so each runtime gives the same parse. The names are the keys of one
190
+ // object, so no two names are equal.
191
+ const names = Object.keys(rootNodes).sort((a, b) => rank(a) - rank(b) || (a < b ? -1 : 1));
192
+ const roots: BookmarkFolder[] = [];
193
+ for (const name of names) {
194
+ const node = rootNodes[name];
195
+ // The roots object of an old file also has values that are not folders, for example a sync transaction version
196
+ if (!isNode(node) || node.type !== "folder") continue;
197
+ roots.push(folderOf(node, [], name, 1, name));
198
+ }
199
+ return { roots, skipped, problems };
200
+ }
201
+
202
+ /**
203
+ * This function gives the name of each profile from the JSON of the file `Local State` of a
204
+ * Chromium browser, by the folder of the profile. A value that it cannot read gives no names.
205
+ */
206
+ export function profileNames(localState: unknown): Map<string, string> {
207
+ const names = new Map<string, string>();
208
+ const cache = (localState as { profile?: { info_cache?: unknown } } | null)?.profile?.info_cache;
209
+ if (typeof cache !== "object" || cache === null) return names;
210
+ for (const [folder, info] of Object.entries(cache)) {
211
+ const name = (info as { name?: unknown } | null)?.name;
212
+ if (typeof name === "string" && name.trim() !== "") names.set(folder, name);
213
+ }
214
+ return names;
215
+ }
package/src/index.ts CHANGED
@@ -1,5 +1,8 @@
1
1
  // The package has no procedures. It gives functions only: the search, the run and the renders.
2
2
  export * from "./browser.js";
3
- export { DEFAULT_TIMEOUT_MS, type CommandRunner, type HeadlessOptions, type RenderResult } from "./run.js";
3
+ export { DEFAULT_TIMEOUT_MS, policyProxyArgs, type CommandRunner, type HeadlessOptions, type RenderResult } from "./run.js";
4
4
  export { isPdfFile, pdfArgs, renderPdf, type RenderPdfOptions, type RenderPdfResult } from "./pdf.js";
5
5
  export * from "./screenshot.js";
6
+ // The bookmarks files of Chrome, Edge and Chromium: the parse (also on the browser entry ./bookmarks) and the search on the disk
7
+ export * from "./bookmarks.js";
8
+ export * from "./profiles.js";
@@ -0,0 +1,176 @@
1
+ /**
2
+ * The profiles of the Chromium browsers on the disk: Google Chrome, Microsoft Edge and Chromium.
3
+ * This file finds the folders of the user data of each browser. Then it finds the bookmarks files of
4
+ * each profile in them, with the names of the profiles, and reads a file. The parse is in `bookmarks.ts`.
5
+ * This file uses Node.js, so only the root entry exports it.
6
+ */
7
+
8
+ import { readFile, readdir, stat } from "node:fs/promises";
9
+ import { join, posix, win32 } from "node:path";
10
+ import {
11
+ BOOKMARK_FILES,
12
+ MAX_BOOKMARKS_BYTES,
13
+ parseChromeBookmarks,
14
+ profileNames,
15
+ type BookmarkSource,
16
+ type ChromeBookmarks,
17
+ type ParseChromeBookmarksOptions,
18
+ } from "./bookmarks.js";
19
+
20
+ /** The file operations of the search and of the read. A test gives files in memory. */
21
+ export interface FileAccess {
22
+ /** This function gives the names in a folder, or no names when the folder does not exist. */
23
+ list(folder: string): Promise<readonly string[]>;
24
+ /** This function gives the size of a file in bytes, or `undefined` when the file does not exist. */
25
+ size(file: string): Promise<number | undefined>;
26
+ /** This function gives the text of a file in UTF-8. */
27
+ read(file: string): Promise<string>;
28
+ }
29
+
30
+ /**
31
+ * The file operations on the disk. `list` gives no names for each error, not only for a missing
32
+ * folder: also for a folder that it cannot read (`EACCES`). `size` gives `undefined` for each error
33
+ * and for a path that is not a file. `read` throws the error of the disk.
34
+ */
35
+ export const diskFileAccess: FileAccess = {
36
+ list: async (folder) => {
37
+ try {
38
+ return await readdir(folder);
39
+ } catch {
40
+ return [];
41
+ }
42
+ },
43
+ size: async (file) => {
44
+ try {
45
+ const found = await stat(file);
46
+ return found.isFile() ? found.size : undefined;
47
+ } catch {
48
+ return undefined;
49
+ }
50
+ },
51
+ read: (file) => readFile(file, "utf8"),
52
+ };
53
+
54
+ /**
55
+ * The key of a Chromium browser that keeps its profiles in a folder of user data. These are the
56
+ * browsers that `findBrowser` finds. The key is the first part of the id of a bookmarks file.
57
+ */
58
+ export type ProfileBrowser = "chrome" | "edge" | "chromium";
59
+
60
+ /** The browsers of `browserDataFolders`, in the order of its result. */
61
+ export const PROFILE_BROWSERS: readonly ProfileBrowser[] = ["chrome", "edge", "chromium"];
62
+
63
+ /** A folder of the user data of a browser. Each profile of the browser is a folder in it. */
64
+ export interface BrowserDataFolder {
65
+ /** The key of the browser, for example `chrome`. Another key, for example `brave`, is also correct. */
66
+ readonly browser: string;
67
+ /** The path of the folder. */
68
+ readonly folder: string;
69
+ }
70
+
71
+ /** The parts of the path of the user data of each browser, on each platform. */
72
+ const DATA_PATHS: Readonly<Record<ProfileBrowser, { readonly windows: readonly string[]; readonly mac: readonly string[]; readonly linux: string }>> = {
73
+ chrome: { windows: ["Google", "Chrome", "User Data"], mac: ["Google", "Chrome"], linux: "google-chrome" },
74
+ edge: { windows: ["Microsoft", "Edge", "User Data"], mac: ["Microsoft Edge"], linux: "microsoft-edge" },
75
+ chromium: { windows: ["Chromium", "User Data"], mac: ["Chromium"], linux: "chromium" },
76
+ };
77
+
78
+ /**
79
+ * This function gives the folders of the user data of the browsers for a platform. Windows uses
80
+ * `LOCALAPPDATA`, macOS uses `~/Library/Application Support`, and Linux uses `XDG_CONFIG_HOME` or `~/.config`.
81
+ * The function reads no file, so a folder in the result can be absent.
82
+ */
83
+ export function browserDataFolders(
84
+ platform: NodeJS.Platform,
85
+ env: Readonly<Record<string, string | undefined>>,
86
+ home: string,
87
+ browsers: readonly ProfileBrowser[] = PROFILE_BROWSERS,
88
+ ): BrowserDataFolder[] {
89
+ if (platform === "win32") {
90
+ const local = env["LOCALAPPDATA"] ?? win32.join(home, "AppData", "Local");
91
+ return browsers.map((browser) => ({ browser, folder: win32.join(local, ...DATA_PATHS[browser].windows) }));
92
+ }
93
+ if (platform === "darwin") {
94
+ const support = posix.join(home, "Library", "Application Support");
95
+ return browsers.map((browser) => ({ browser, folder: posix.join(support, ...DATA_PATHS[browser].mac) }));
96
+ }
97
+ const config = env["XDG_CONFIG_HOME"] ?? posix.join(home, ".config");
98
+ return browsers.map((browser) => ({ browser, folder: posix.join(config, DATA_PATHS[browser].linux) }));
99
+ }
100
+
101
+ /** A profile folder has the name `Default` or `Profile <n>`. The guest and system profiles have no bookmarks. */
102
+ const PROFILE = /^(Default|Profile \d+)$/;
103
+
104
+ /** This function gives the order of two profile folders: `Default` first, then `Profile <n>` by the number. */
105
+ function profileOrder(a: string, b: string): number {
106
+ const number = (name: string): number => (name === "Default" ? 0 : Number(name.slice("Profile ".length)));
107
+ return number(a) - number(b);
108
+ }
109
+
110
+ /** A bookmarks file that the search found, with its browser, its profile folder and its path. */
111
+ export interface BookmarkFile extends BookmarkSource {
112
+ /** The key of the browser of the profile, for example `chrome` or `edge`. */
113
+ readonly browser: string;
114
+ /** The name of the folder of the profile, for example `Default` or `Profile 2`. */
115
+ readonly profile: string;
116
+ /** The path of the file. */
117
+ readonly path: string;
118
+ /** The size of the file in bytes, at the time of the search. A file over `MAX_BOOKMARKS_BYTES` is not a bookmarks file. */
119
+ readonly size: number;
120
+ }
121
+
122
+ /**
123
+ * This function finds the bookmarks files of the profiles in the folders of the user data, in the
124
+ * order of the folders. In a folder, it gives `Default` first, then `Profile <n>` by the number. The
125
+ * file `Local State` gives the names of the profiles. Without it, a profile has the name of its folder.
126
+ * The id of a file starts with the key of its browser, so the files of two browsers have different ids.
127
+ */
128
+ export async function findBookmarkFiles(folders: readonly BrowserDataFolder[], files: FileAccess = diskFileAccess): Promise<BookmarkFile[]> {
129
+ const found: BookmarkFile[] = [];
130
+ for (const { browser, folder } of folders) {
131
+ let names = new Map<string, string>();
132
+ try {
133
+ names = profileNames(JSON.parse(await files.read(join(folder, "Local State"))));
134
+ } catch {
135
+ // A missing or broken file Local State gives the names of the folders
136
+ }
137
+ const profiles = new Set([...names.keys(), ...(await files.list(folder))].filter((name) => PROFILE.test(name)));
138
+ for (const profile of [...profiles].sort(profileOrder)) {
139
+ for (const [file, kind] of Object.entries(BOOKMARK_FILES)) {
140
+ const path = join(folder, profile, file);
141
+ const size = await files.size(path);
142
+ if (size === undefined) continue;
143
+ found.push({ id: `${browser}:${profile}:${file}`, name: names.get(profile) ?? profile, kind, browser, profile, path, size });
144
+ }
145
+ }
146
+ }
147
+ return found;
148
+ }
149
+
150
+ /**
151
+ * This function reads and parses a bookmarks file that `findBookmarkFiles` found. It reads the size
152
+ * again, because the file can change after the search. It refuses a file over `MAX_BOOKMARKS_BYTES`
153
+ * before the read. The default prefix of the keys is the id of the file and a colon, so the keys of
154
+ * two files are different.
155
+ *
156
+ * @throws Error when the file is not there, is too large, is not JSON or has no roots
157
+ */
158
+ export async function readBookmarkFile(
159
+ file: Pick<BookmarkFile, "id" | "path">,
160
+ files: FileAccess = diskFileAccess,
161
+ options: ParseChromeBookmarksOptions = {},
162
+ ): Promise<ChromeBookmarks> {
163
+ const size = await files.size(file.path);
164
+ if (size === undefined) throw new Error(`The file ${file.path} is not there now.`);
165
+ if (size > MAX_BOOKMARKS_BYTES) {
166
+ throw new Error(`The file ${file.path} has ${size} bytes. A bookmarks file has ${MAX_BOOKMARKS_BYTES} bytes at most.`);
167
+ }
168
+ const text = await files.read(file.path);
169
+ let value: unknown;
170
+ try {
171
+ value = JSON.parse(text);
172
+ } catch (error) {
173
+ throw new Error(`The file ${file.path} is not JSON, so it is not a bookmarks file.`, { cause: error });
174
+ }
175
+ return parseChromeBookmarks(value, { prefix: options.prefix ?? `${file.id}:` });
176
+ }
package/src/run.ts CHANGED
@@ -5,12 +5,17 @@
5
5
  *
6
6
  * The browser starts through `runCommand` of `@procedures/shell`, not through a shell. A time limit
7
7
  * and the signal of the call stop its process tree.
8
+ *
9
+ * With the option `hostPolicy`, the run starts a policy proxy of `@procedures/http`. The browser
10
+ * sends each request through the proxy, and the proxy applies the host policy to each request.
11
+ * When the page is an `http:` or `https:` URL and the proxy gets no request, the run fails.
8
12
  */
9
13
 
10
14
  import { closeSync, copyFileSync, mkdirSync, mkdtempSync, openSync, readSync, rmSync, statSync, writeFileSync } from "node:fs";
11
15
  import { tmpdir } from "node:os";
12
16
  import { dirname, join, resolve } from "node:path";
13
17
  import { pathToFileURL } from "node:url";
18
+ import { startPolicyProxy, type HostPolicy, type ProxyRefusal } from "@procedures/http";
14
19
  import { runCommand, type CommandOptions, type CommandResult } from "@procedures/shell/command";
15
20
  import { BROWSER_ENV, findBrowser, type FindBrowserOptions } from "./browser.js";
16
21
 
@@ -32,6 +37,82 @@ export interface HeadlessOptions extends FindBrowserOptions {
32
37
  readonly signal?: AbortSignal | undefined;
33
38
  /** The function that runs the browser (default: `runCommand`). The tests give their own. */
34
39
  readonly runner?: CommandRunner | undefined;
40
+ /**
41
+ * The host policy of each request of the browser. With it, each request goes through a policy
42
+ * proxy, also each redirect and each resource of the page. The proxy gives the status 403 to
43
+ * each host that the policy refuses.
44
+ */
45
+ readonly hostPolicy?: HostPolicy | undefined;
46
+ /** This function gets each request that the host policy refused. It needs `hostPolicy`. */
47
+ readonly onHostRefusal?: ((refusal: ProxyRefusal) => void) | undefined;
48
+ }
49
+
50
+ /**
51
+ * The switches that the flags of the policy proxy set, and the switches that can undo these flags.
52
+ * With a host policy, the run refuses each of them in `args`.
53
+ */
54
+ const POLICY_SWITCHES: ReadonlySet<string> = new Set([
55
+ "proxy-server",
56
+ "proxy-bypass-list",
57
+ "no-proxy-server",
58
+ "proxy-pac-url",
59
+ "proxy-auto-detect",
60
+ "use-system-proxy-resolver",
61
+ "winhttp-proxy-resolver",
62
+ "enable-quic",
63
+ "origin-to-force-quic-on",
64
+ "force-webrtc-ip-handling-policy",
65
+ "host-resolver-rules",
66
+ "host-rules",
67
+ ]);
68
+
69
+ /**
70
+ * This function gives the flags that send each request of the browser through the proxy at
71
+ * `proxyUrl`, and that close the paths around the proxy.
72
+ *
73
+ * - `--proxy-server` sends each `http:`, `https:`, `ws:` and `wss:` request to the proxy. The proxy
74
+ * resolves each name, not the browser.
75
+ * - `--proxy-bypass-list=<-loopback>` removes the default bypass of `localhost`, of the loopback
76
+ * addresses and of the link-local addresses. Thus these requests also go to the proxy.
77
+ * - `--disable-quic` stops the QUIC protocol, so no request goes over UDP around the proxy.
78
+ * - `--force-webrtc-ip-handling-policy=disable_non_proxied_udp` stops the UDP of WebRTC, and shows no
79
+ * local address.
80
+ * - `--host-resolver-rules` stops each DNS look-up of the browser. The only exception is the address
81
+ * of the proxy.
82
+ */
83
+ export function policyProxyArgs(proxyUrl: string): string[] {
84
+ const { hostname } = new URL(proxyUrl);
85
+ return [
86
+ `--proxy-server=${proxyUrl}`,
87
+ "--proxy-bypass-list=<-loopback>",
88
+ "--disable-quic",
89
+ "--force-webrtc-ip-handling-policy=disable_non_proxied_udp",
90
+ `--host-resolver-rules=MAP * ~NOTFOUND , EXCLUDE ${hostname}`,
91
+ ];
92
+ }
93
+
94
+ /**
95
+ * This function gives the name of the switch of a flag, or undefined for a value that is not a
96
+ * switch. The browser takes the prefixes `--`, `-` and, on Windows, `/`. Windows ignores the case of
97
+ * the name. Thus the function takes each prefix, and gives the name in lower case.
98
+ */
99
+ function switchName(arg: string): string | undefined {
100
+ const match = /^(?:--|-|\/)([^=]+)/.exec(arg);
101
+ return match?.[1]?.toLowerCase();
102
+ }
103
+
104
+ /** This function throws when the options of the host policy do not agree with the other options. */
105
+ function checkPolicyOptions(options: HeadlessOptions): void {
106
+ if (options.hostPolicy === undefined) {
107
+ if (options.onHostRefusal !== undefined) throw new Error("The option onHostRefusal needs the option hostPolicy");
108
+ return;
109
+ }
110
+ for (const arg of options.args ?? []) {
111
+ const name = switchName(arg);
112
+ if (name !== undefined && POLICY_SWITCHES.has(name)) {
113
+ throw new Error(`The option hostPolicy sets the proxy flags of the browser, so args cannot have the flag --${name}`);
114
+ }
115
+ }
35
116
  }
36
117
 
37
118
  /** The result of a render. */
@@ -85,27 +166,69 @@ function lastLines(text: string, count = 5): string {
85
166
  return text.trim().split(/\r?\n/).slice(-count).join("\n");
86
167
  }
87
168
 
169
+ /**
170
+ * This function removes a folder and its content. It tries again 3 times, 100 ms apart, because
171
+ * Windows can hold a file for a short time after the browser stops. A removal that fails leaves the
172
+ * folder and does not throw. Thus it does not hide the result or the error of the caller.
173
+ */
174
+ function removeFolder(path: string): void {
175
+ try {
176
+ rmSync(path, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 });
177
+ } catch {
178
+ // The folder stays in the temporary folder of the system
179
+ }
180
+ }
181
+
182
+ /**
183
+ * This function makes a new temporary folder, gives its path to `use`, and then removes the folder,
184
+ * also when `use` throws. The name of the folder starts with `prefix`. The folder is in `folder`,
185
+ * else in the temporary folder of the system. Each temporary folder of this package uses this function.
186
+ */
187
+ export async function withTemporaryFolder<T>(prefix: string, folder: string | undefined, use: (path: string) => Promise<T>): Promise<T> {
188
+ const path = mkdtempSync(join(folder ?? tmpdir(), prefix));
189
+ try {
190
+ return await use(path);
191
+ } finally {
192
+ removeFolder(path);
193
+ }
194
+ }
195
+
88
196
  /**
89
197
  * This function starts the browser one time for a render, and gives the path and the size of the
90
198
  * file. It throws when it finds no browser, at the time limit and after a cancel. It also throws
91
199
  * when the file is missing, empty or not of the kind of the render.
92
200
  *
93
201
  * An existing output file changes only after the check of the new file. Thus a failed render keeps
94
- * an old output file.
202
+ * an old output file. Without the option `output`, the file goes in a new temporary folder. This
203
+ * folder stays after a success, and goes after a failure.
204
+ *
205
+ * With the option `hostPolicy`, the browser gets the flags of `policyProxyArgs` before the flags of
206
+ * `args`. The proxy stops at the end of the run, also after an error. When the page is an `http:`
207
+ * or `https:` URL and the proxy gets no request, the render fails. Then the requests of the page
208
+ * did not go through the policy.
95
209
  */
96
210
  export async function headlessRender(render: HeadlessRender, source: PageSource, options: HeadlessOptions): Promise<RenderResult> {
211
+ checkPolicyOptions(options);
97
212
  const browser = findBrowser(options);
98
213
  if (browser === undefined) {
99
214
  throw new Error(`No browser found. Install Microsoft Edge, Google Chrome or Chromium, or set ${BROWSER_ENV} to the browser program.`);
100
215
  }
101
-
102
- const work = mkdtempSync(join(tmpdir(), render.prefix));
103
- const profile = join(work, "profile");
104
- const htmlFile = join(work, "page.html");
105
- const output = resolve(options.output ?? join(work, render.file));
106
- const written = join(work, render.file);
107
- let made = false;
216
+ if (options.output !== undefined) return renderOnce(render, source, options, browser, resolve(options.output));
217
+ const kept = mkdtempSync(join(tmpdir(), render.prefix));
108
218
  try {
219
+ return await renderOnce(render, source, options, browser, join(kept, render.file));
220
+ } catch (error) {
221
+ removeFolder(kept);
222
+ throw error;
223
+ }
224
+ }
225
+
226
+ /** This function starts the browser in a temporary folder that always goes, and copies the checked file to `output`. */
227
+ function renderOnce(render: HeadlessRender, source: PageSource, options: HeadlessOptions, browser: string, output: string): Promise<RenderResult> {
228
+ return withTemporaryFolder(render.prefix, undefined, async (work) => {
229
+ const profile = join(work, "profile");
230
+ const htmlFile = join(work, "page.html");
231
+ const written = join(work, render.file);
109
232
  let page: string;
110
233
  if ("html" in source) {
111
234
  writeFileSync(htmlFile, source.html, "utf8");
@@ -119,13 +242,21 @@ export async function headlessRender(render: HeadlessRender, source: PageSource,
119
242
 
120
243
  const run = options.runner ?? runCommand;
121
244
  const timeout = options.timeout ?? DEFAULT_TIMEOUT_MS;
122
- const result = await run(browser, {
123
- args: render.argv(page, written, profile, options.args ?? []),
124
- timeout,
125
- signal: options.signal,
126
- maxOutputBytes: 64 * 1024,
127
- keep: "tail",
128
- });
245
+ const proxy = options.hostPolicy === undefined ? undefined : await startPolicyProxy({ policy: options.hostPolicy, onRefusal: options.onHostRefusal });
246
+ let result: CommandResult;
247
+ try {
248
+ // The flags of the proxy come before the extra flags. A "--" in the extra flags ends the switches of the browser
249
+ const extra = proxy === undefined ? (options.args ?? []) : [...policyProxyArgs(proxy.url), ...(options.args ?? [])];
250
+ result = await run(browser, {
251
+ args: render.argv(page, written, profile, extra),
252
+ timeout,
253
+ signal: options.signal,
254
+ maxOutputBytes: 64 * 1024,
255
+ keep: "tail",
256
+ });
257
+ } finally {
258
+ await proxy?.close();
259
+ }
129
260
  if (result.error !== undefined) throw new Error(`The browser did not start: ${result.error}`);
130
261
  if (result.timedOut === true) throw new Error(`The browser did not finish in ${timeout} ms`);
131
262
  if (result.aborted === true) throw new Error("The render was cancelled");
@@ -135,18 +266,16 @@ export async function headlessRender(render: HeadlessRender, source: PageSource,
135
266
  const detail = lastLines(result.stderr || result.stdout);
136
267
  throw new Error(`The browser made no ${render.noun} (exit code ${result.exitCode})${detail ? `: ${detail}` : ""}`);
137
268
  }
269
+ // A Chrome or Edge policy (ProxySettings or ProxyMode) replaces the flag --proxy-server, and the browser gives no error.
270
+ // A web page sends at least one request, so a run with no request through the proxy is not a pass
271
+ if (proxy !== undefined && /^https?:/i.test(page) && proxy.requests === 0) {
272
+ throw new Error(
273
+ "The browser sent no request through the policy proxy, so the host policy did not apply. A Chrome or Edge policy (ProxySettings or ProxyMode) can replace the flag --proxy-server.",
274
+ );
275
+ }
138
276
  if (!render.isValid(written)) throw new Error(`The file ${output} is not a ${render.noun}`);
139
- // The temporary folder goes in `finally`, so a copy is enough, also across two drives
140
- if (written !== output) copyFileSync(written, output);
141
- made = true;
277
+ // The temporary folder goes after this function, so a copy is enough, also across two drives
278
+ copyFileSync(written, output);
142
279
  return { path: output, bytes, browser };
143
- } finally {
144
- // The temporary page and profile go. A default output stays in its folder after a success.
145
- if (made && options.output === undefined) {
146
- rmSync(profile, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 });
147
- rmSync(htmlFile, { force: true });
148
- } else {
149
- rmSync(work, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 });
150
- }
151
- }
280
+ });
152
281
  }