@procedures/browser 0.55.1 → 0.56.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +36 -0
- package/README.md +102 -1
- package/dist/bookmarks.d.ts +93 -0
- package/dist/bookmarks.d.ts.map +1 -0
- package/dist/bookmarks.js +139 -0
- package/dist/bookmarks.js.map +1 -0
- package/dist/index.d.ts +3 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -1
- package/dist/index.js.map +1 -1
- package/dist/profiles.d.ts +70 -0
- package/dist/profiles.d.ts.map +1 -0
- package/dist/profiles.js +121 -0
- package/dist/profiles.js.map +1 -0
- package/dist/run.d.ts +41 -1
- package/dist/run.d.ts.map +1 -1
- package/dist/run.js +145 -29
- package/dist/run.js.map +1 -1
- package/dist/screenshot.d.ts +21 -0
- package/dist/screenshot.d.ts.map +1 -1
- package/dist/screenshot.js +18 -2
- package/dist/screenshot.js.map +1 -1
- package/package.json +19 -6
- package/src/bookmarks.ts +215 -0
- package/src/index.ts +4 -1
- package/src/profiles.ts +176 -0
- package/src/run.ts +156 -27
- package/src/screenshot.ts +34 -2
package/src/bookmarks.ts
ADDED
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The bookmarks files of Chrome. Edge and Chromium write the same format. This file reads the JSON
|
|
3
|
+
* of a bookmarks file, and gives a tree of folders with their sites. It also reads the names of the
|
|
4
|
+
* profiles from the file `Local State`.
|
|
5
|
+
*
|
|
6
|
+
* This file imports no Node built-in, so a page can use it: the entry `@procedures/browser/bookmarks`.
|
|
7
|
+
* The search for the files on the disk is in `profiles.ts`, on the root entry.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
/** The deepest folder that the parser reads. A deeper folder gives a problem, and the parser does not read it. */
|
|
11
|
+
export const MAX_BOOKMARK_DEPTH = 100;
|
|
12
|
+
|
|
13
|
+
/** The largest bookmarks file that a reader accepts, in bytes. A larger file is not a bookmarks file. */
|
|
14
|
+
export const MAX_BOOKMARKS_BYTES: number = 64 * 1024 * 1024;
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* The kind of a bookmarks file. `profile` is the file `Bookmarks` of a profile. `account` is the
|
|
18
|
+
* file `AccountBookmarks`: the bookmarks of a Google account with no sync.
|
|
19
|
+
*/
|
|
20
|
+
export type BookmarkFileKind = "profile" | "account";
|
|
21
|
+
|
|
22
|
+
/** The bookmarks files of a profile, by the name of the file, with their kinds. */
|
|
23
|
+
export const BOOKMARK_FILES: Readonly<Record<string, BookmarkFileKind>> = { Bookmarks: "profile", AccountBookmarks: "account" };
|
|
24
|
+
|
|
25
|
+
/** A bookmarks file, with no path. A host can give this to a page and keep the path. */
|
|
26
|
+
export interface BookmarkSource {
|
|
27
|
+
/** The id of the file. The search gives `<browser>:<profile folder>:<file name>`, for example `chrome:Default:Bookmarks`. */
|
|
28
|
+
readonly id: string;
|
|
29
|
+
/** The name of the profile, as the browser shows it. */
|
|
30
|
+
readonly name: string;
|
|
31
|
+
readonly kind: BookmarkFileKind;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/** A bookmark of a web site. */
|
|
35
|
+
export interface Bookmark {
|
|
36
|
+
/** The address of the site: an `http` or `https` address with a host. */
|
|
37
|
+
readonly url: string;
|
|
38
|
+
/** The title of the bookmark. A bookmark with an empty title has no title. */
|
|
39
|
+
readonly title?: string;
|
|
40
|
+
/** The ISO 8601 time when the user added the bookmark. */
|
|
41
|
+
readonly added?: string;
|
|
42
|
+
/** The names of the folders from the root to the folder of the bookmark. */
|
|
43
|
+
readonly path: readonly string[];
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** A folder of the bookmarks, with its sites and the folders in it. */
|
|
47
|
+
export interface BookmarkFolder {
|
|
48
|
+
/**
|
|
49
|
+
* The key of the folder, after the prefix of the parse: its `guid`, else its `id`, else its place in the file. The place is
|
|
50
|
+
* the name of the root, then the index of each child, for example `bookmark_bar/2/0`. Each parse of a file gives the same keys.
|
|
51
|
+
*/
|
|
52
|
+
readonly key: string;
|
|
53
|
+
readonly name: string;
|
|
54
|
+
/** The names of the folders from the root to this folder. The last name is the name of this folder. */
|
|
55
|
+
readonly path: readonly string[];
|
|
56
|
+
readonly folders: readonly BookmarkFolder[];
|
|
57
|
+
/** The sites directly in this folder. Each site has the path of this folder. */
|
|
58
|
+
readonly sites: readonly Bookmark[];
|
|
59
|
+
/** The count of the sites in this folder and in each folder in it. */
|
|
60
|
+
readonly total: number;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/** A part of a bookmarks file that the parser cannot read. */
|
|
64
|
+
export interface BookmarkProblem {
|
|
65
|
+
/** The names of the folders to the part. */
|
|
66
|
+
readonly path: readonly string[];
|
|
67
|
+
readonly message: string;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** The result of the parse of a bookmarks file. */
|
|
71
|
+
export interface ChromeBookmarks {
|
|
72
|
+
/** The root folders, for example "Bookmarks bar" and "Other bookmarks". */
|
|
73
|
+
readonly roots: readonly BookmarkFolder[];
|
|
74
|
+
/** The count of the bookmarks that are not web sites, for example `javascript:` or `chrome://` addresses. */
|
|
75
|
+
readonly skipped: number;
|
|
76
|
+
readonly problems: readonly BookmarkProblem[];
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** The options of `parseChromeBookmarks`. */
|
|
80
|
+
export interface ParseChromeBookmarksOptions {
|
|
81
|
+
/** The text before each key. Give two files of one profile different prefixes, so that their keys are different. */
|
|
82
|
+
readonly prefix?: string | undefined;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/** The order of the roots that Chrome writes. The roots with other names come after them, in the order of their code units. */
|
|
86
|
+
const ROOT_ORDER = ["bookmark_bar", "other", "synced"];
|
|
87
|
+
|
|
88
|
+
/** The time from 1601-01-01 (the start of the Chrome times) to 1970-01-01, in milliseconds. */
|
|
89
|
+
const WINDOWS_EPOCH_MS = 11_644_473_600_000n;
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* A web address has the scheme `http` or `https`, a host, and no spaces. The test does not use `URL`, so it is the same in each
|
|
93
|
+
* runtime. The first character of the host is one class, and the rest is `\S*`. Thus no two quantifiers agree on the same
|
|
94
|
+
* character, and the time of the test is linear in the length of the address.
|
|
95
|
+
*/
|
|
96
|
+
const WEB = /^https?:\/\/[^\s/?#]\S*$/i;
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* This function gives the ISO 8601 time of a Chrome time, or `undefined` for no time. A Chrome time
|
|
100
|
+
* is a text of the microseconds from 1601-01-01 UTC. The function uses `BigInt`, because the value
|
|
101
|
+
* is larger than a safe integer.
|
|
102
|
+
*/
|
|
103
|
+
export function chromeTime(value: unknown): string | undefined {
|
|
104
|
+
if (typeof value !== "string" || !/^\d{1,20}$/.test(value)) return undefined;
|
|
105
|
+
const ms = BigInt(value) / 1000n - WINDOWS_EPOCH_MS;
|
|
106
|
+
// The time 0 means no time. A time before 1990 or after 9999 is not a date of a bookmark.
|
|
107
|
+
if (ms < 631_152_000_000n || ms > 253_402_300_799_999n) return undefined;
|
|
108
|
+
return new Date(Number(ms)).toISOString();
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
interface Node {
|
|
112
|
+
readonly type?: unknown;
|
|
113
|
+
readonly name?: unknown;
|
|
114
|
+
readonly url?: unknown;
|
|
115
|
+
readonly children?: unknown;
|
|
116
|
+
readonly guid?: unknown;
|
|
117
|
+
readonly id?: unknown;
|
|
118
|
+
readonly date_added?: unknown;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
function isNode(value: unknown): value is Node {
|
|
122
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/** This function gives the `guid` of a folder, else its `id`. An empty text or a value of another type is not a key. */
|
|
126
|
+
function ownKey(node: Node): string | undefined {
|
|
127
|
+
if (typeof node.guid === "string" && node.guid !== "") return node.guid;
|
|
128
|
+
if (typeof node.id === "string" && node.id !== "") return node.id;
|
|
129
|
+
if (typeof node.id === "number" && Number.isFinite(node.id)) return String(node.id);
|
|
130
|
+
return undefined;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
function rank(name: string): number {
|
|
134
|
+
const index = ROOT_ORDER.indexOf(name);
|
|
135
|
+
return index === -1 ? ROOT_ORDER.length : index;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* This function reads the JSON value of a Chrome bookmarks file: the file `Bookmarks` or
|
|
140
|
+
* `AccountBookmarks` of a profile. It gives a problem for each part that it cannot read, and it
|
|
141
|
+
* reads the other parts. It keeps only the bookmarks of web sites, and counts the others.
|
|
142
|
+
*
|
|
143
|
+
* @throws Error when the value has no roots, so it is not a bookmarks file
|
|
144
|
+
*/
|
|
145
|
+
export function parseChromeBookmarks(value: unknown, options: ParseChromeBookmarksOptions = {}): ChromeBookmarks {
|
|
146
|
+
const prefix = options.prefix ?? "";
|
|
147
|
+
if (!isNode(value) || !isNode((value as { roots?: unknown }).roots)) throw new Error("A Chrome bookmarks file has an object roots.");
|
|
148
|
+
const rootNodes = (value as { roots: Record<string, unknown> }).roots;
|
|
149
|
+
const problems: BookmarkProblem[] = [];
|
|
150
|
+
let skipped = 0;
|
|
151
|
+
|
|
152
|
+
// The place of a folder is the name of its root, then the index of each child to it. It is unique in a file.
|
|
153
|
+
const folderOf = (node: Node, parent: readonly string[], fallback: string, depth: number, place: string): BookmarkFolder => {
|
|
154
|
+
const name = typeof node.name === "string" && node.name !== "" ? node.name : fallback;
|
|
155
|
+
const path = [...parent, name];
|
|
156
|
+
const own = ownKey(node) ?? place;
|
|
157
|
+
const folders: BookmarkFolder[] = [];
|
|
158
|
+
const sites: Bookmark[] = [];
|
|
159
|
+
let total = 0;
|
|
160
|
+
const children = Array.isArray(node.children) ? (node.children as unknown[]) : [];
|
|
161
|
+
if (node.children !== undefined && !Array.isArray(node.children)) problems.push({ path, message: "The children of a folder must be a list." });
|
|
162
|
+
for (const [index, child] of children.entries()) {
|
|
163
|
+
if (!isNode(child)) {
|
|
164
|
+
problems.push({ path, message: "A bookmark must be an object." });
|
|
165
|
+
} else if (child.type === "url") {
|
|
166
|
+
if (typeof child.url !== "string" || !WEB.test(child.url)) {
|
|
167
|
+
skipped++;
|
|
168
|
+
continue;
|
|
169
|
+
}
|
|
170
|
+
const title = typeof child.name === "string" && child.name.trim() !== "" ? child.name : undefined;
|
|
171
|
+
const added = chromeTime(child.date_added);
|
|
172
|
+
sites.push({ url: child.url, ...(title === undefined ? {} : { title }), ...(added === undefined ? {} : { added }), path });
|
|
173
|
+
total++;
|
|
174
|
+
} else if (child.type === "folder") {
|
|
175
|
+
if (depth >= MAX_BOOKMARK_DEPTH) {
|
|
176
|
+
problems.push({ path, message: `A folder is deeper than ${MAX_BOOKMARK_DEPTH} folders. The parser does not read it.` });
|
|
177
|
+
continue;
|
|
178
|
+
}
|
|
179
|
+
const folder = folderOf(child, path, "Folder", depth + 1, `${place}/${index}`);
|
|
180
|
+
folders.push(folder);
|
|
181
|
+
total += folder.total;
|
|
182
|
+
} else {
|
|
183
|
+
problems.push({ path, message: `A bookmark has the type ${String(child.type)}, not url or folder.` });
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
return { key: `${prefix}${own}`, name, path, folders, sites, total };
|
|
187
|
+
};
|
|
188
|
+
|
|
189
|
+
// The order of the code units, not of a locale, so each runtime gives the same parse. The names are the keys of one
|
|
190
|
+
// object, so no two names are equal.
|
|
191
|
+
const names = Object.keys(rootNodes).sort((a, b) => rank(a) - rank(b) || (a < b ? -1 : 1));
|
|
192
|
+
const roots: BookmarkFolder[] = [];
|
|
193
|
+
for (const name of names) {
|
|
194
|
+
const node = rootNodes[name];
|
|
195
|
+
// The roots object of an old file also has values that are not folders, for example a sync transaction version
|
|
196
|
+
if (!isNode(node) || node.type !== "folder") continue;
|
|
197
|
+
roots.push(folderOf(node, [], name, 1, name));
|
|
198
|
+
}
|
|
199
|
+
return { roots, skipped, problems };
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/**
|
|
203
|
+
* This function gives the name of each profile from the JSON of the file `Local State` of a
|
|
204
|
+
* Chromium browser, by the folder of the profile. A value that it cannot read gives no names.
|
|
205
|
+
*/
|
|
206
|
+
export function profileNames(localState: unknown): Map<string, string> {
|
|
207
|
+
const names = new Map<string, string>();
|
|
208
|
+
const cache = (localState as { profile?: { info_cache?: unknown } } | null)?.profile?.info_cache;
|
|
209
|
+
if (typeof cache !== "object" || cache === null) return names;
|
|
210
|
+
for (const [folder, info] of Object.entries(cache)) {
|
|
211
|
+
const name = (info as { name?: unknown } | null)?.name;
|
|
212
|
+
if (typeof name === "string" && name.trim() !== "") names.set(folder, name);
|
|
213
|
+
}
|
|
214
|
+
return names;
|
|
215
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
// The package has no procedures. It gives functions only: the search, the run and the renders.
|
|
2
2
|
export * from "./browser.js";
|
|
3
|
-
export { DEFAULT_TIMEOUT_MS, type CommandRunner, type HeadlessOptions, type RenderResult } from "./run.js";
|
|
3
|
+
export { DEFAULT_TIMEOUT_MS, policyProxyArgs, type CommandRunner, type HeadlessOptions, type RenderResult } from "./run.js";
|
|
4
4
|
export { isPdfFile, pdfArgs, renderPdf, type RenderPdfOptions, type RenderPdfResult } from "./pdf.js";
|
|
5
5
|
export * from "./screenshot.js";
|
|
6
|
+
// The bookmarks files of Chrome, Edge and Chromium: the parse (also on the browser entry ./bookmarks) and the search on the disk
|
|
7
|
+
export * from "./bookmarks.js";
|
|
8
|
+
export * from "./profiles.js";
|
package/src/profiles.ts
ADDED
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The profiles of the Chromium browsers on the disk: Google Chrome, Microsoft Edge and Chromium.
|
|
3
|
+
* This file finds the folders of the user data of each browser. Then it finds the bookmarks files of
|
|
4
|
+
* each profile in them, with the names of the profiles, and reads a file. The parse is in `bookmarks.ts`.
|
|
5
|
+
* This file uses Node.js, so only the root entry exports it.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import { readFile, readdir, stat } from "node:fs/promises";
|
|
9
|
+
import { join, posix, win32 } from "node:path";
|
|
10
|
+
import {
|
|
11
|
+
BOOKMARK_FILES,
|
|
12
|
+
MAX_BOOKMARKS_BYTES,
|
|
13
|
+
parseChromeBookmarks,
|
|
14
|
+
profileNames,
|
|
15
|
+
type BookmarkSource,
|
|
16
|
+
type ChromeBookmarks,
|
|
17
|
+
type ParseChromeBookmarksOptions,
|
|
18
|
+
} from "./bookmarks.js";
|
|
19
|
+
|
|
20
|
+
/** The file operations of the search and of the read. A test gives files in memory. */
|
|
21
|
+
export interface FileAccess {
|
|
22
|
+
/** This function gives the names in a folder, or no names when the folder does not exist. */
|
|
23
|
+
list(folder: string): Promise<readonly string[]>;
|
|
24
|
+
/** This function gives the size of a file in bytes, or `undefined` when the file does not exist. */
|
|
25
|
+
size(file: string): Promise<number | undefined>;
|
|
26
|
+
/** This function gives the text of a file in UTF-8. */
|
|
27
|
+
read(file: string): Promise<string>;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* The file operations on the disk. `list` gives no names for each error, not only for a missing
|
|
32
|
+
* folder: also for a folder that it cannot read (`EACCES`). `size` gives `undefined` for each error
|
|
33
|
+
* and for a path that is not a file. `read` throws the error of the disk.
|
|
34
|
+
*/
|
|
35
|
+
export const diskFileAccess: FileAccess = {
|
|
36
|
+
list: async (folder) => {
|
|
37
|
+
try {
|
|
38
|
+
return await readdir(folder);
|
|
39
|
+
} catch {
|
|
40
|
+
return [];
|
|
41
|
+
}
|
|
42
|
+
},
|
|
43
|
+
size: async (file) => {
|
|
44
|
+
try {
|
|
45
|
+
const found = await stat(file);
|
|
46
|
+
return found.isFile() ? found.size : undefined;
|
|
47
|
+
} catch {
|
|
48
|
+
return undefined;
|
|
49
|
+
}
|
|
50
|
+
},
|
|
51
|
+
read: (file) => readFile(file, "utf8"),
|
|
52
|
+
};
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* The key of a Chromium browser that keeps its profiles in a folder of user data. These are the
|
|
56
|
+
* browsers that `findBrowser` finds. The key is the first part of the id of a bookmarks file.
|
|
57
|
+
*/
|
|
58
|
+
export type ProfileBrowser = "chrome" | "edge" | "chromium";
|
|
59
|
+
|
|
60
|
+
/** The browsers of `browserDataFolders`, in the order of its result. */
|
|
61
|
+
export const PROFILE_BROWSERS: readonly ProfileBrowser[] = ["chrome", "edge", "chromium"];
|
|
62
|
+
|
|
63
|
+
/** A folder of the user data of a browser. Each profile of the browser is a folder in it. */
|
|
64
|
+
export interface BrowserDataFolder {
|
|
65
|
+
/** The key of the browser, for example `chrome`. Another key, for example `brave`, is also correct. */
|
|
66
|
+
readonly browser: string;
|
|
67
|
+
/** The path of the folder. */
|
|
68
|
+
readonly folder: string;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** The parts of the path of the user data of each browser, on each platform. */
|
|
72
|
+
const DATA_PATHS: Readonly<Record<ProfileBrowser, { readonly windows: readonly string[]; readonly mac: readonly string[]; readonly linux: string }>> = {
|
|
73
|
+
chrome: { windows: ["Google", "Chrome", "User Data"], mac: ["Google", "Chrome"], linux: "google-chrome" },
|
|
74
|
+
edge: { windows: ["Microsoft", "Edge", "User Data"], mac: ["Microsoft Edge"], linux: "microsoft-edge" },
|
|
75
|
+
chromium: { windows: ["Chromium", "User Data"], mac: ["Chromium"], linux: "chromium" },
|
|
76
|
+
};
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* This function gives the folders of the user data of the browsers for a platform. Windows uses
|
|
80
|
+
* `LOCALAPPDATA`, macOS uses `~/Library/Application Support`, and Linux uses `XDG_CONFIG_HOME` or `~/.config`.
|
|
81
|
+
* The function reads no file, so a folder in the result can be absent.
|
|
82
|
+
*/
|
|
83
|
+
export function browserDataFolders(
|
|
84
|
+
platform: NodeJS.Platform,
|
|
85
|
+
env: Readonly<Record<string, string | undefined>>,
|
|
86
|
+
home: string,
|
|
87
|
+
browsers: readonly ProfileBrowser[] = PROFILE_BROWSERS,
|
|
88
|
+
): BrowserDataFolder[] {
|
|
89
|
+
if (platform === "win32") {
|
|
90
|
+
const local = env["LOCALAPPDATA"] ?? win32.join(home, "AppData", "Local");
|
|
91
|
+
return browsers.map((browser) => ({ browser, folder: win32.join(local, ...DATA_PATHS[browser].windows) }));
|
|
92
|
+
}
|
|
93
|
+
if (platform === "darwin") {
|
|
94
|
+
const support = posix.join(home, "Library", "Application Support");
|
|
95
|
+
return browsers.map((browser) => ({ browser, folder: posix.join(support, ...DATA_PATHS[browser].mac) }));
|
|
96
|
+
}
|
|
97
|
+
const config = env["XDG_CONFIG_HOME"] ?? posix.join(home, ".config");
|
|
98
|
+
return browsers.map((browser) => ({ browser, folder: posix.join(config, DATA_PATHS[browser].linux) }));
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/** A profile folder has the name `Default` or `Profile <n>`. The guest and system profiles have no bookmarks. */
|
|
102
|
+
const PROFILE = /^(Default|Profile \d+)$/;
|
|
103
|
+
|
|
104
|
+
/** This function gives the order of two profile folders: `Default` first, then `Profile <n>` by the number. */
|
|
105
|
+
function profileOrder(a: string, b: string): number {
|
|
106
|
+
const number = (name: string): number => (name === "Default" ? 0 : Number(name.slice("Profile ".length)));
|
|
107
|
+
return number(a) - number(b);
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/** A bookmarks file that the search found, with its browser, its profile folder and its path. */
|
|
111
|
+
export interface BookmarkFile extends BookmarkSource {
|
|
112
|
+
/** The key of the browser of the profile, for example `chrome` or `edge`. */
|
|
113
|
+
readonly browser: string;
|
|
114
|
+
/** The name of the folder of the profile, for example `Default` or `Profile 2`. */
|
|
115
|
+
readonly profile: string;
|
|
116
|
+
/** The path of the file. */
|
|
117
|
+
readonly path: string;
|
|
118
|
+
/** The size of the file in bytes, at the time of the search. A file over `MAX_BOOKMARKS_BYTES` is not a bookmarks file. */
|
|
119
|
+
readonly size: number;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* This function finds the bookmarks files of the profiles in the folders of the user data, in the
|
|
124
|
+
* order of the folders. In a folder, it gives `Default` first, then `Profile <n>` by the number. The
|
|
125
|
+
* file `Local State` gives the names of the profiles. Without it, a profile has the name of its folder.
|
|
126
|
+
* The id of a file starts with the key of its browser, so the files of two browsers have different ids.
|
|
127
|
+
*/
|
|
128
|
+
export async function findBookmarkFiles(folders: readonly BrowserDataFolder[], files: FileAccess = diskFileAccess): Promise<BookmarkFile[]> {
|
|
129
|
+
const found: BookmarkFile[] = [];
|
|
130
|
+
for (const { browser, folder } of folders) {
|
|
131
|
+
let names = new Map<string, string>();
|
|
132
|
+
try {
|
|
133
|
+
names = profileNames(JSON.parse(await files.read(join(folder, "Local State"))));
|
|
134
|
+
} catch {
|
|
135
|
+
// A missing or broken file Local State gives the names of the folders
|
|
136
|
+
}
|
|
137
|
+
const profiles = new Set([...names.keys(), ...(await files.list(folder))].filter((name) => PROFILE.test(name)));
|
|
138
|
+
for (const profile of [...profiles].sort(profileOrder)) {
|
|
139
|
+
for (const [file, kind] of Object.entries(BOOKMARK_FILES)) {
|
|
140
|
+
const path = join(folder, profile, file);
|
|
141
|
+
const size = await files.size(path);
|
|
142
|
+
if (size === undefined) continue;
|
|
143
|
+
found.push({ id: `${browser}:${profile}:${file}`, name: names.get(profile) ?? profile, kind, browser, profile, path, size });
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
return found;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/**
|
|
151
|
+
* This function reads and parses a bookmarks file that `findBookmarkFiles` found. It reads the size
|
|
152
|
+
* again, because the file can change after the search. It refuses a file over `MAX_BOOKMARKS_BYTES`
|
|
153
|
+
* before the read. The default prefix of the keys is the id of the file and a colon, so the keys of
|
|
154
|
+
* two files are different.
|
|
155
|
+
*
|
|
156
|
+
* @throws Error when the file is not there, is too large, is not JSON or has no roots
|
|
157
|
+
*/
|
|
158
|
+
export async function readBookmarkFile(
|
|
159
|
+
file: Pick<BookmarkFile, "id" | "path">,
|
|
160
|
+
files: FileAccess = diskFileAccess,
|
|
161
|
+
options: ParseChromeBookmarksOptions = {},
|
|
162
|
+
): Promise<ChromeBookmarks> {
|
|
163
|
+
const size = await files.size(file.path);
|
|
164
|
+
if (size === undefined) throw new Error(`The file ${file.path} is not there now.`);
|
|
165
|
+
if (size > MAX_BOOKMARKS_BYTES) {
|
|
166
|
+
throw new Error(`The file ${file.path} has ${size} bytes. A bookmarks file has ${MAX_BOOKMARKS_BYTES} bytes at most.`);
|
|
167
|
+
}
|
|
168
|
+
const text = await files.read(file.path);
|
|
169
|
+
let value: unknown;
|
|
170
|
+
try {
|
|
171
|
+
value = JSON.parse(text);
|
|
172
|
+
} catch (error) {
|
|
173
|
+
throw new Error(`The file ${file.path} is not JSON, so it is not a bookmarks file.`, { cause: error });
|
|
174
|
+
}
|
|
175
|
+
return parseChromeBookmarks(value, { prefix: options.prefix ?? `${file.id}:` });
|
|
176
|
+
}
|
package/src/run.ts
CHANGED
|
@@ -5,12 +5,17 @@
|
|
|
5
5
|
*
|
|
6
6
|
* The browser starts through `runCommand` of `@procedures/shell`, not through a shell. A time limit
|
|
7
7
|
* and the signal of the call stop its process tree.
|
|
8
|
+
*
|
|
9
|
+
* With the option `hostPolicy`, the run starts a policy proxy of `@procedures/http`. The browser
|
|
10
|
+
* sends each request through the proxy, and the proxy applies the host policy to each request.
|
|
11
|
+
* When the page is an `http:` or `https:` URL and the proxy gets no request, the run fails.
|
|
8
12
|
*/
|
|
9
13
|
|
|
10
14
|
import { closeSync, copyFileSync, mkdirSync, mkdtempSync, openSync, readSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
11
15
|
import { tmpdir } from "node:os";
|
|
12
16
|
import { dirname, join, resolve } from "node:path";
|
|
13
17
|
import { pathToFileURL } from "node:url";
|
|
18
|
+
import { startPolicyProxy, type HostPolicy, type ProxyRefusal } from "@procedures/http";
|
|
14
19
|
import { runCommand, type CommandOptions, type CommandResult } from "@procedures/shell/command";
|
|
15
20
|
import { BROWSER_ENV, findBrowser, type FindBrowserOptions } from "./browser.js";
|
|
16
21
|
|
|
@@ -32,6 +37,82 @@ export interface HeadlessOptions extends FindBrowserOptions {
|
|
|
32
37
|
readonly signal?: AbortSignal | undefined;
|
|
33
38
|
/** The function that runs the browser (default: `runCommand`). The tests give their own. */
|
|
34
39
|
readonly runner?: CommandRunner | undefined;
|
|
40
|
+
/**
|
|
41
|
+
* The host policy of each request of the browser. With it, each request goes through a policy
|
|
42
|
+
* proxy, also each redirect and each resource of the page. The proxy gives the status 403 to
|
|
43
|
+
* each host that the policy refuses.
|
|
44
|
+
*/
|
|
45
|
+
readonly hostPolicy?: HostPolicy | undefined;
|
|
46
|
+
/** This function gets each request that the host policy refused. It needs `hostPolicy`. */
|
|
47
|
+
readonly onHostRefusal?: ((refusal: ProxyRefusal) => void) | undefined;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* The switches that the flags of the policy proxy set, and the switches that can undo these flags.
|
|
52
|
+
* With a host policy, the run refuses each of them in `args`.
|
|
53
|
+
*/
|
|
54
|
+
const POLICY_SWITCHES: ReadonlySet<string> = new Set([
|
|
55
|
+
"proxy-server",
|
|
56
|
+
"proxy-bypass-list",
|
|
57
|
+
"no-proxy-server",
|
|
58
|
+
"proxy-pac-url",
|
|
59
|
+
"proxy-auto-detect",
|
|
60
|
+
"use-system-proxy-resolver",
|
|
61
|
+
"winhttp-proxy-resolver",
|
|
62
|
+
"enable-quic",
|
|
63
|
+
"origin-to-force-quic-on",
|
|
64
|
+
"force-webrtc-ip-handling-policy",
|
|
65
|
+
"host-resolver-rules",
|
|
66
|
+
"host-rules",
|
|
67
|
+
]);
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* This function gives the flags that send each request of the browser through the proxy at
|
|
71
|
+
* `proxyUrl`, and that close the paths around the proxy.
|
|
72
|
+
*
|
|
73
|
+
* - `--proxy-server` sends each `http:`, `https:`, `ws:` and `wss:` request to the proxy. The proxy
|
|
74
|
+
* resolves each name, not the browser.
|
|
75
|
+
* - `--proxy-bypass-list=<-loopback>` removes the default bypass of `localhost`, of the loopback
|
|
76
|
+
* addresses and of the link-local addresses. Thus these requests also go to the proxy.
|
|
77
|
+
* - `--disable-quic` stops the QUIC protocol, so no request goes over UDP around the proxy.
|
|
78
|
+
* - `--force-webrtc-ip-handling-policy=disable_non_proxied_udp` stops the UDP of WebRTC, and shows no
|
|
79
|
+
* local address.
|
|
80
|
+
* - `--host-resolver-rules` stops each DNS look-up of the browser. The only exception is the address
|
|
81
|
+
* of the proxy.
|
|
82
|
+
*/
|
|
83
|
+
export function policyProxyArgs(proxyUrl: string): string[] {
|
|
84
|
+
const { hostname } = new URL(proxyUrl);
|
|
85
|
+
return [
|
|
86
|
+
`--proxy-server=${proxyUrl}`,
|
|
87
|
+
"--proxy-bypass-list=<-loopback>",
|
|
88
|
+
"--disable-quic",
|
|
89
|
+
"--force-webrtc-ip-handling-policy=disable_non_proxied_udp",
|
|
90
|
+
`--host-resolver-rules=MAP * ~NOTFOUND , EXCLUDE ${hostname}`,
|
|
91
|
+
];
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* This function gives the name of the switch of a flag, or undefined for a value that is not a
|
|
96
|
+
* switch. The browser takes the prefixes `--`, `-` and, on Windows, `/`. Windows ignores the case of
|
|
97
|
+
* the name. Thus the function takes each prefix, and gives the name in lower case.
|
|
98
|
+
*/
|
|
99
|
+
function switchName(arg: string): string | undefined {
|
|
100
|
+
const match = /^(?:--|-|\/)([^=]+)/.exec(arg);
|
|
101
|
+
return match?.[1]?.toLowerCase();
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/** This function throws when the options of the host policy do not agree with the other options. */
|
|
105
|
+
function checkPolicyOptions(options: HeadlessOptions): void {
|
|
106
|
+
if (options.hostPolicy === undefined) {
|
|
107
|
+
if (options.onHostRefusal !== undefined) throw new Error("The option onHostRefusal needs the option hostPolicy");
|
|
108
|
+
return;
|
|
109
|
+
}
|
|
110
|
+
for (const arg of options.args ?? []) {
|
|
111
|
+
const name = switchName(arg);
|
|
112
|
+
if (name !== undefined && POLICY_SWITCHES.has(name)) {
|
|
113
|
+
throw new Error(`The option hostPolicy sets the proxy flags of the browser, so args cannot have the flag --${name}`);
|
|
114
|
+
}
|
|
115
|
+
}
|
|
35
116
|
}
|
|
36
117
|
|
|
37
118
|
/** The result of a render. */
|
|
@@ -85,27 +166,69 @@ function lastLines(text: string, count = 5): string {
|
|
|
85
166
|
return text.trim().split(/\r?\n/).slice(-count).join("\n");
|
|
86
167
|
}
|
|
87
168
|
|
|
169
|
+
/**
|
|
170
|
+
* This function removes a folder and its content. It tries again 3 times, 100 ms apart, because
|
|
171
|
+
* Windows can hold a file for a short time after the browser stops. A removal that fails leaves the
|
|
172
|
+
* folder and does not throw. Thus it does not hide the result or the error of the caller.
|
|
173
|
+
*/
|
|
174
|
+
function removeFolder(path: string): void {
|
|
175
|
+
try {
|
|
176
|
+
rmSync(path, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 });
|
|
177
|
+
} catch {
|
|
178
|
+
// The folder stays in the temporary folder of the system
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/**
|
|
183
|
+
* This function makes a new temporary folder, gives its path to `use`, and then removes the folder,
|
|
184
|
+
* also when `use` throws. The name of the folder starts with `prefix`. The folder is in `folder`,
|
|
185
|
+
* else in the temporary folder of the system. Each temporary folder of this package uses this function.
|
|
186
|
+
*/
|
|
187
|
+
export async function withTemporaryFolder<T>(prefix: string, folder: string | undefined, use: (path: string) => Promise<T>): Promise<T> {
|
|
188
|
+
const path = mkdtempSync(join(folder ?? tmpdir(), prefix));
|
|
189
|
+
try {
|
|
190
|
+
return await use(path);
|
|
191
|
+
} finally {
|
|
192
|
+
removeFolder(path);
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
|
|
88
196
|
/**
|
|
89
197
|
* This function starts the browser one time for a render, and gives the path and the size of the
|
|
90
198
|
* file. It throws when it finds no browser, at the time limit and after a cancel. It also throws
|
|
91
199
|
* when the file is missing, empty or not of the kind of the render.
|
|
92
200
|
*
|
|
93
201
|
* An existing output file changes only after the check of the new file. Thus a failed render keeps
|
|
94
|
-
* an old output file.
|
|
202
|
+
* an old output file. Without the option `output`, the file goes in a new temporary folder. This
|
|
203
|
+
* folder stays after a success, and goes after a failure.
|
|
204
|
+
*
|
|
205
|
+
* With the option `hostPolicy`, the browser gets the flags of `policyProxyArgs` before the flags of
|
|
206
|
+
* `args`. The proxy stops at the end of the run, also after an error. When the page is an `http:`
|
|
207
|
+
* or `https:` URL and the proxy gets no request, the render fails. Then the requests of the page
|
|
208
|
+
* did not go through the policy.
|
|
95
209
|
*/
|
|
96
210
|
export async function headlessRender(render: HeadlessRender, source: PageSource, options: HeadlessOptions): Promise<RenderResult> {
|
|
211
|
+
checkPolicyOptions(options);
|
|
97
212
|
const browser = findBrowser(options);
|
|
98
213
|
if (browser === undefined) {
|
|
99
214
|
throw new Error(`No browser found. Install Microsoft Edge, Google Chrome or Chromium, or set ${BROWSER_ENV} to the browser program.`);
|
|
100
215
|
}
|
|
101
|
-
|
|
102
|
-
const
|
|
103
|
-
const profile = join(work, "profile");
|
|
104
|
-
const htmlFile = join(work, "page.html");
|
|
105
|
-
const output = resolve(options.output ?? join(work, render.file));
|
|
106
|
-
const written = join(work, render.file);
|
|
107
|
-
let made = false;
|
|
216
|
+
if (options.output !== undefined) return renderOnce(render, source, options, browser, resolve(options.output));
|
|
217
|
+
const kept = mkdtempSync(join(tmpdir(), render.prefix));
|
|
108
218
|
try {
|
|
219
|
+
return await renderOnce(render, source, options, browser, join(kept, render.file));
|
|
220
|
+
} catch (error) {
|
|
221
|
+
removeFolder(kept);
|
|
222
|
+
throw error;
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/** This function starts the browser in a temporary folder that always goes, and copies the checked file to `output`. */
|
|
227
|
+
function renderOnce(render: HeadlessRender, source: PageSource, options: HeadlessOptions, browser: string, output: string): Promise<RenderResult> {
|
|
228
|
+
return withTemporaryFolder(render.prefix, undefined, async (work) => {
|
|
229
|
+
const profile = join(work, "profile");
|
|
230
|
+
const htmlFile = join(work, "page.html");
|
|
231
|
+
const written = join(work, render.file);
|
|
109
232
|
let page: string;
|
|
110
233
|
if ("html" in source) {
|
|
111
234
|
writeFileSync(htmlFile, source.html, "utf8");
|
|
@@ -119,13 +242,21 @@ export async function headlessRender(render: HeadlessRender, source: PageSource,
|
|
|
119
242
|
|
|
120
243
|
const run = options.runner ?? runCommand;
|
|
121
244
|
const timeout = options.timeout ?? DEFAULT_TIMEOUT_MS;
|
|
122
|
-
const
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
245
|
+
const proxy = options.hostPolicy === undefined ? undefined : await startPolicyProxy({ policy: options.hostPolicy, onRefusal: options.onHostRefusal });
|
|
246
|
+
let result: CommandResult;
|
|
247
|
+
try {
|
|
248
|
+
// The flags of the proxy come before the extra flags. A "--" in the extra flags ends the switches of the browser
|
|
249
|
+
const extra = proxy === undefined ? (options.args ?? []) : [...policyProxyArgs(proxy.url), ...(options.args ?? [])];
|
|
250
|
+
result = await run(browser, {
|
|
251
|
+
args: render.argv(page, written, profile, extra),
|
|
252
|
+
timeout,
|
|
253
|
+
signal: options.signal,
|
|
254
|
+
maxOutputBytes: 64 * 1024,
|
|
255
|
+
keep: "tail",
|
|
256
|
+
});
|
|
257
|
+
} finally {
|
|
258
|
+
await proxy?.close();
|
|
259
|
+
}
|
|
129
260
|
if (result.error !== undefined) throw new Error(`The browser did not start: ${result.error}`);
|
|
130
261
|
if (result.timedOut === true) throw new Error(`The browser did not finish in ${timeout} ms`);
|
|
131
262
|
if (result.aborted === true) throw new Error("The render was cancelled");
|
|
@@ -135,18 +266,16 @@ export async function headlessRender(render: HeadlessRender, source: PageSource,
|
|
|
135
266
|
const detail = lastLines(result.stderr || result.stdout);
|
|
136
267
|
throw new Error(`The browser made no ${render.noun} (exit code ${result.exitCode})${detail ? `: ${detail}` : ""}`);
|
|
137
268
|
}
|
|
269
|
+
// A Chrome or Edge policy (ProxySettings or ProxyMode) replaces the flag --proxy-server, and the browser gives no error.
|
|
270
|
+
// A web page sends at least one request, so a run with no request through the proxy is not a pass
|
|
271
|
+
if (proxy !== undefined && /^https?:/i.test(page) && proxy.requests === 0) {
|
|
272
|
+
throw new Error(
|
|
273
|
+
"The browser sent no request through the policy proxy, so the host policy did not apply. A Chrome or Edge policy (ProxySettings or ProxyMode) can replace the flag --proxy-server.",
|
|
274
|
+
);
|
|
275
|
+
}
|
|
138
276
|
if (!render.isValid(written)) throw new Error(`The file ${output} is not a ${render.noun}`);
|
|
139
|
-
// The temporary folder goes
|
|
140
|
-
|
|
141
|
-
made = true;
|
|
277
|
+
// The temporary folder goes after this function, so a copy is enough, also across two drives
|
|
278
|
+
copyFileSync(written, output);
|
|
142
279
|
return { path: output, bytes, browser };
|
|
143
|
-
}
|
|
144
|
-
// The temporary page and profile go. A default output stays in its folder after a success.
|
|
145
|
-
if (made && options.output === undefined) {
|
|
146
|
-
rmSync(profile, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 });
|
|
147
|
-
rmSync(htmlFile, { force: true });
|
|
148
|
-
} else {
|
|
149
|
-
rmSync(work, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 });
|
|
150
|
-
}
|
|
151
|
-
}
|
|
280
|
+
});
|
|
152
281
|
}
|