html-snapshots 5.11.0 → 6.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/HISTORY.md +6 -0
- package/README.md +66 -106
- package/lib/async/exists.js +1 -1
- package/lib/async/index.js +4 -3
- package/lib/common/index.js +31 -0
- package/lib/common/sitemap/sitemap-collection.js +2 -2
- package/lib/common/sitemap/sitemap.js +5 -6
- package/lib/html-snapshots.js +41 -78
- package/lib/input-generators/_base.js +12 -11
- package/lib/input-generators/array.js +1 -2
- package/lib/input-generators/index.js +2 -2
- package/lib/input-generators/robots.js +1 -1
- package/lib/input-generators/textfile.js +2 -2
- package/lib/playwright/index.js +125 -0
- package/lib/playwright/removeScripts.js +11 -0
- package/lib/puppeteer/index.js +7 -7
- package/lib/puppeteer/removeScripts.js +1 -1
- package/package.json +34 -31
- package/eslint.config.js +0 -23
- package/lib/phantom/customFilter.js +0 -24
- package/lib/phantom/default.js +0 -21
- package/lib/phantom/modules/cli.js +0 -100
- package/lib/phantom/modules/detectors.js +0 -48
- package/lib/phantom/modules/globals.js +0 -43
- package/lib/phantom/modules/selectorDetect.js +0 -108
- package/lib/phantom/modules/verbose.js +0 -71
- package/lib/phantom/removeScripts.js +0 -30
package/lib/html-snapshots.js
CHANGED
|
@@ -10,52 +10,19 @@
|
|
|
10
10
|
* Licensed under the MIT license.
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
|
-
const spawn = require("child_process")
|
|
14
|
-
const path = require("path");
|
|
15
|
-
const EventEmitter = require("events")
|
|
16
|
-
const fs = require("fs");
|
|
13
|
+
const { spawn } = require("node:child_process");
|
|
14
|
+
const path = require("node:path");
|
|
15
|
+
const { EventEmitter } = require("node:events");
|
|
16
|
+
const fs = require("node:fs");
|
|
17
17
|
const asyncLib = require("async");
|
|
18
18
|
const common = require("./common");
|
|
19
19
|
const inputFactory = require("./input-generators");
|
|
20
|
-
const Notifier = require("./async")
|
|
20
|
+
const { Notifier } = require("./async");
|
|
21
21
|
|
|
22
|
-
/**
|
|
23
|
-
* Determine the default phantomJS module path. This is overridden by the
|
|
24
|
-
* phatomjs option.
|
|
25
|
-
*
|
|
26
|
-
* This function, (technique and concerns) originated from karma-phantomjs-launcher:
|
|
27
|
-
*
|
|
28
|
-
* The MIT License
|
|
29
|
-
* Copyright (C) 2011-2013 Google, Inc.
|
|
30
|
-
*
|
|
31
|
-
* Permission is hereby granted, free of charge, to any person obtaining a copy of
|
|
32
|
-
* this software and associated documentation files (the "Software"), to deal in
|
|
33
|
-
* the Software without restriction, including without limitation the rights to
|
|
34
|
-
* use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
|
|
35
|
-
* of the Software, and to permit persons to whom the Software is furnished to do
|
|
36
|
-
* so, subject to the following conditions:
|
|
37
|
-
*
|
|
38
|
-
* The above copyright notice and this permission notice shall be included in all
|
|
39
|
-
* copies or substantial portions of the Software.
|
|
40
|
-
*/
|
|
41
|
-
function defaultPhantomJSExePath () {
|
|
42
|
-
// If the path we're given by phantomjs is to a .cmd, it is pointing to a global copy.
|
|
43
|
-
// Using the cmd as the process to execute causes problems cleaning up the processes
|
|
44
|
-
// so we walk from the cmd to the phantomjs.exe and use that instead.
|
|
45
|
-
|
|
46
|
-
const phantomSource = require("phantomjs-prebuilt").path;
|
|
47
|
-
|
|
48
|
-
if (path.extname(phantomSource).toLowerCase() === ".cmd") {
|
|
49
|
-
return path.join(path.dirname( phantomSource ), "//node_modules//phantomjs-prebuilt//lib//phantom//bin//phantomjs.exe");
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
return phantomSource;
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
const phantomDir = "./phantom";
|
|
56
|
-
const phantomSnapshotScript = path.join(phantomDir, "default.js");
|
|
57
22
|
const puppeteerDir = "./puppeteer";
|
|
58
23
|
const puppeteerSnapshotScript = path.join(puppeteerDir, "index.js");
|
|
24
|
+
const playwrightDir = "./playwright";
|
|
25
|
+
const playwrightSnapshotScript = path.join(playwrightDir, "index.js");
|
|
59
26
|
|
|
60
27
|
/**
|
|
61
28
|
* This module's defaults
|
|
@@ -63,9 +30,8 @@ const puppeteerSnapshotScript = path.join(puppeteerDir, "index.js");
|
|
|
63
30
|
const defaults = Object.freeze({
|
|
64
31
|
input: "robots",
|
|
65
32
|
browser: "puppeteer",
|
|
66
|
-
phantomjs: defaultPhantomJSExePath(),
|
|
67
|
-
phantomSnapshotScript: path.join(__dirname, phantomSnapshotScript),
|
|
68
33
|
puppeteer: path.join(__dirname, puppeteerSnapshotScript),
|
|
34
|
+
playwright: path.join(__dirname, playwrightSnapshotScript),
|
|
69
35
|
snapshotScript: path.join(__dirname, puppeteerSnapshotScript),
|
|
70
36
|
outputDirClean: false,
|
|
71
37
|
pollInterval: 500,
|
|
@@ -73,22 +39,18 @@ const defaults = Object.freeze({
|
|
|
73
39
|
});
|
|
74
40
|
|
|
75
41
|
/**
|
|
76
|
-
* The worker task that launches phantomjs.
|
|
77
|
-
*
|
|
78
42
|
* @param {Object} input - An input object from an input generator.
|
|
79
|
-
* @param {Array} input.phantomjsOptions - Actual Phantomjs options.
|
|
80
43
|
* @param {String} input.outputFile - The output file that the html is written to.
|
|
81
44
|
* @param {String} input.url - The url to get the html from.
|
|
82
45
|
* @param {String} input.selector - The selector to wait for.
|
|
83
|
-
* @param {String} input.timeout - The
|
|
84
|
-
* @param {
|
|
85
|
-
* @param {
|
|
86
|
-
* @param {Boolean} input.verbose - True if verbose output is desired.
|
|
46
|
+
* @param {String} input.timeout - The entire process timeout.
|
|
47
|
+
* @param {Object} input.debug - The debug object.
|
|
48
|
+
* @param {Object} input.puppeteerLaunchOptions - overriding puppeteer launch options.
|
|
87
49
|
* @param {Object} options - Input options.
|
|
88
50
|
* @param {Object} options.snapshotScript - Snapshot script options.
|
|
89
51
|
* @param {String} options.snapshotScript.script - Snapshot script name.
|
|
90
52
|
* @param {String} options.snapshotScript.module - Snapshot script module.
|
|
91
|
-
* @param {String} options.
|
|
53
|
+
* @param {String} options.puppeteer - Full path to the puppeteer exec.
|
|
92
54
|
* @param {Object} notifier - The async Notifier instance for this run.
|
|
93
55
|
* @param {Function} notifier.remove - Remove an outputFile from notifier.
|
|
94
56
|
* @param {Function} notifier.setError - Set an error on the notifier.
|
|
@@ -96,34 +58,34 @@ const defaults = Object.freeze({
|
|
|
96
58
|
* @param {Function} notifier.known - Check if a file is or has been processed.
|
|
97
59
|
* @param {Function} qcb - async.queue callback function.
|
|
98
60
|
*/
|
|
99
|
-
function
|
|
100
|
-
let
|
|
61
|
+
function puppeteerWorker (input, options, notifier, qcb) {
|
|
62
|
+
let filter = "false";
|
|
101
63
|
let snapshotScript = options.snapshotScript;
|
|
102
|
-
const phantomjsOptions =
|
|
103
|
-
Array.isArray(input.phantomjsOptions) ? input.phantomjsOptions : [input.phantomjsOptions];
|
|
104
64
|
|
|
105
65
|
// If the outputFile has NOT already been seen by the notifier, process.
|
|
106
66
|
if (!notifier.known(input.outputFile)) {
|
|
107
67
|
|
|
108
|
-
// map snapshotScript object script
|
|
68
|
+
// map snapshotScript object script
|
|
109
69
|
if (common.isObject(options.snapshotScript)) {
|
|
110
|
-
snapshotScript =
|
|
111
|
-
|
|
70
|
+
snapshotScript = options.puppeteer;
|
|
71
|
+
if (options.snapshotScript.script === "removeScripts") {
|
|
72
|
+
filter = "./removeScripts";
|
|
73
|
+
} else {
|
|
74
|
+
filter = options.snapshotScript.module;
|
|
75
|
+
}
|
|
112
76
|
}
|
|
113
77
|
|
|
114
|
-
cp = spawn(
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
snapshotScript,
|
|
118
|
-
input.outputFile,
|
|
78
|
+
const cp = spawn(
|
|
79
|
+
snapshotScript,
|
|
80
|
+
[ input.outputFile,
|
|
119
81
|
input.url,
|
|
120
82
|
input.selector,
|
|
121
83
|
input.timeout,
|
|
122
|
-
|
|
123
|
-
input.
|
|
124
|
-
input.
|
|
125
|
-
|
|
126
|
-
]
|
|
84
|
+
filter,
|
|
85
|
+
input.debug.flag,
|
|
86
|
+
input.debug.slowMo,
|
|
87
|
+
JSON.stringify(input.puppeteerLaunchOptions)
|
|
88
|
+
], { cwd: process.cwd(), stdio: "inherit", detached: true }
|
|
127
89
|
);
|
|
128
90
|
|
|
129
91
|
cp.on("error", function (e) {
|
|
@@ -139,8 +101,7 @@ function phantomjsWorker (input, options, notifier, qcb) {
|
|
|
139
101
|
|
|
140
102
|
// start counting
|
|
141
103
|
notifier.add(input.outputFile, input.timeout);
|
|
142
|
-
}
|
|
143
|
-
else {
|
|
104
|
+
} else {
|
|
144
105
|
// The input.outputFile is being or has been processed this run.
|
|
145
106
|
qcb(0);
|
|
146
107
|
}
|
|
@@ -153,12 +114,14 @@ function phantomjsWorker (input, options, notifier, qcb) {
|
|
|
153
114
|
* @param {String} input.selector - The selector to wait for.
|
|
154
115
|
* @param {String} input.timeout - The entire process timeout.
|
|
155
116
|
* @param {Object} input.debug - The debug object.
|
|
156
|
-
* @param {Object} input.
|
|
117
|
+
* @param {Object} input.playwrightLaunchOptions - overriding playwright launch options.
|
|
118
|
+
* May include a `browserType` key ("chromium" | "firefox" | "webkit") to select
|
|
119
|
+
* the browser engine. Defaults to "chromium".
|
|
157
120
|
* @param {Object} options - Input options.
|
|
158
121
|
* @param {Object} options.snapshotScript - Snapshot script options.
|
|
159
122
|
* @param {String} options.snapshotScript.script - Snapshot script name.
|
|
160
123
|
* @param {String} options.snapshotScript.module - Snapshot script module.
|
|
161
|
-
* @param {String} options.
|
|
124
|
+
* @param {String} options.playwright - Full path to the playwright exec.
|
|
162
125
|
* @param {Object} notifier - The async Notifier instance for this run.
|
|
163
126
|
* @param {Function} notifier.remove - Remove an outputFile from notifier.
|
|
164
127
|
* @param {Function} notifier.setError - Set an error on the notifier.
|
|
@@ -166,7 +129,7 @@ function phantomjsWorker (input, options, notifier, qcb) {
|
|
|
166
129
|
* @param {Function} notifier.known - Check if a file is or has been processed.
|
|
167
130
|
* @param {Function} qcb - async.queue callback function.
|
|
168
131
|
*/
|
|
169
|
-
function
|
|
132
|
+
function playwrightWorker (input, options, notifier, qcb) {
|
|
170
133
|
let filter = "false";
|
|
171
134
|
let snapshotScript = options.snapshotScript;
|
|
172
135
|
|
|
@@ -175,7 +138,7 @@ function puppeteerWorker (input, options, notifier, qcb) {
|
|
|
175
138
|
|
|
176
139
|
// map snapshotScript object script
|
|
177
140
|
if (common.isObject(options.snapshotScript)) {
|
|
178
|
-
snapshotScript = options.
|
|
141
|
+
snapshotScript = options.playwright;
|
|
179
142
|
if (options.snapshotScript.script === "removeScripts") {
|
|
180
143
|
filter = "./removeScripts";
|
|
181
144
|
} else {
|
|
@@ -192,7 +155,7 @@ function puppeteerWorker (input, options, notifier, qcb) {
|
|
|
192
155
|
filter,
|
|
193
156
|
input.debug.flag,
|
|
194
157
|
input.debug.slowMo,
|
|
195
|
-
JSON.stringify(input.
|
|
158
|
+
JSON.stringify(input.playwrightLaunchOptions)
|
|
196
159
|
], { cwd: process.cwd(), stdio: "inherit", detached: true }
|
|
197
160
|
);
|
|
198
161
|
|
|
@@ -230,10 +193,10 @@ function prepOptions (options) {
|
|
|
230
193
|
options.input = "array";
|
|
231
194
|
}
|
|
232
195
|
|
|
233
|
-
// If
|
|
234
|
-
if (options.browser ===
|
|
196
|
+
// If playwright browser, make sure the default snapshotScript is correctly set.
|
|
197
|
+
if (options.browser === "playwright") {
|
|
235
198
|
if (!options.snapshotScript || options.snapshotScript === options.puppeteer) {
|
|
236
|
-
options.snapshotScript = options.
|
|
199
|
+
options.snapshotScript = options.playwright;
|
|
237
200
|
}
|
|
238
201
|
}
|
|
239
202
|
}
|
|
@@ -306,7 +269,7 @@ function run (options, listener) {
|
|
|
306
269
|
options._abort = notifier.abort.bind(notifier, q);
|
|
307
270
|
|
|
308
271
|
const worker =
|
|
309
|
-
options.browser ===
|
|
272
|
+
options.browser === "playwright" ? playwrightWorker : puppeteerWorker;
|
|
310
273
|
|
|
311
274
|
// generate input for the snapshots.
|
|
312
275
|
return inputGenerator.run(options, input => {
|
|
@@ -7,9 +7,9 @@
|
|
|
7
7
|
* Licensed under the MIT license.
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
|
-
const EventEmitter = require("events")
|
|
11
|
-
const path = require("path");
|
|
12
|
-
const urlm = require("url");
|
|
10
|
+
const { EventEmitter } = require("node:events");
|
|
11
|
+
const path = require("node:path");
|
|
12
|
+
const urlm = require("node:url");
|
|
13
13
|
const common = require("../common");
|
|
14
14
|
|
|
15
15
|
// Defaults for this module
|
|
@@ -25,7 +25,7 @@ const defaults = Object.freeze({
|
|
|
25
25
|
checkInterval: 250,
|
|
26
26
|
useJQuery: false,
|
|
27
27
|
verbose: false,
|
|
28
|
-
|
|
28
|
+
playwrightLaunchOptions: {},
|
|
29
29
|
puppeteerLaunchOptions: {},
|
|
30
30
|
debug: {
|
|
31
31
|
flag: "false",
|
|
@@ -116,7 +116,8 @@ function prepOptions (options, listener) {
|
|
|
116
116
|
|
|
117
117
|
// Prep single, global object options for normalization.
|
|
118
118
|
[
|
|
119
|
-
"puppeteerLaunchOptions"
|
|
119
|
+
"puppeteerLaunchOptions",
|
|
120
|
+
"playwrightLaunchOptions"
|
|
120
121
|
]
|
|
121
122
|
.forEach((optionName) => {
|
|
122
123
|
prepSingleObjectOptions(optionName, options);
|
|
@@ -126,7 +127,7 @@ function prepOptions (options, listener) {
|
|
|
126
127
|
// The certain arguments are per-page options.
|
|
127
128
|
// However, outputPath is a special case
|
|
128
129
|
[
|
|
129
|
-
"selector", "timeout", "useJQuery", "verbose", "
|
|
130
|
+
"selector", "timeout", "useJQuery", "verbose", "playwrightLaunchOptions",
|
|
130
131
|
"puppeteerLaunchOptions"
|
|
131
132
|
]
|
|
132
133
|
.forEach((perPageOption) => {
|
|
@@ -152,7 +153,7 @@ function prepOptions (options, listener) {
|
|
|
152
153
|
* @returns {String} The 'path' part of the file output path.
|
|
153
154
|
*/
|
|
154
155
|
function getOutputPath (options, page, parse) {
|
|
155
|
-
const pagePart =
|
|
156
|
+
const pagePart = common.parseUrl(page);
|
|
156
157
|
// if outputPath was normalized with an object, let the key passthru
|
|
157
158
|
let outputPath = options.outputPath(page, true);
|
|
158
159
|
|
|
@@ -163,7 +164,7 @@ function getOutputPath (options, page, parse) {
|
|
|
163
164
|
|
|
164
165
|
// if the outputPath is really still a url, fix it to path+hash
|
|
165
166
|
if (common.isUrl(outputPath)) {
|
|
166
|
-
outputPath = pagePart.
|
|
167
|
+
outputPath = `${pagePart.pathname}${pagePart.search}${pagePart.hash ? pagePart.hash : ""}`;
|
|
167
168
|
}
|
|
168
169
|
|
|
169
170
|
// if the caller wants the url parse output, return it
|
|
@@ -254,7 +255,7 @@ module.exports = {
|
|
|
254
255
|
* @param {Number} options.checkInterval - The millisecond resolute poll interval.
|
|
255
256
|
* @param {Function} options.userJQuery - Produces a jquery bool given a page url.
|
|
256
257
|
* @param {Function} options.verbose - Produces a verbose bool given a page url.
|
|
257
|
-
* @param {Function} options.
|
|
258
|
+
* @param {Function} options.playwrightLaunchOptions - Produces playwright launch options given a page url.
|
|
258
259
|
* @param {Function} options.puppeteerLaunchOptions - Produces puppeteer launch options given a page url.
|
|
259
260
|
* @param {Object} options.debug - Produces debug options given a page url.
|
|
260
261
|
* @param {Function} options._inputEmitter - The EventEmitter to generate input.
|
|
@@ -288,8 +289,8 @@ module.exports = {
|
|
|
288
289
|
useJQuery: options.useJQuery(page),
|
|
289
290
|
// map the input page to a verbose flag
|
|
290
291
|
verbose: options.verbose(page),
|
|
291
|
-
// map the input page to
|
|
292
|
-
|
|
292
|
+
// map the input page to playwright launch options
|
|
293
|
+
playwrightLaunchOptions: options.playwrightLaunchOptions(page),
|
|
293
294
|
// map the input page to puppeteer launch options
|
|
294
295
|
puppeteerLaunchOptions: options.puppeteerLaunchOptions(page),
|
|
295
296
|
// useful for puppeteer testing, debugging
|
|
@@ -8,7 +8,6 @@
|
|
|
8
8
|
* Licensed under the MIT license.
|
|
9
9
|
*/
|
|
10
10
|
|
|
11
|
-
const urlm = require("url");
|
|
12
11
|
const common = require("../common");
|
|
13
12
|
const base = require("./_base");
|
|
14
13
|
|
|
@@ -38,7 +37,7 @@ function generateInput (options) {
|
|
|
38
37
|
const result = new Promise((resolve, reject) => {
|
|
39
38
|
if (Array.isArray(options.source)) {
|
|
40
39
|
const all = options.source.every(sourceUrl => {
|
|
41
|
-
const url =
|
|
40
|
+
const url = common.parseUrl(sourceUrl);
|
|
42
41
|
const opts = Object.assign({}, options, {
|
|
43
42
|
protocol: url.protocol,
|
|
44
43
|
auth: url.auth,
|
|
@@ -28,7 +28,7 @@ module.exports = {
|
|
|
28
28
|
|
|
29
29
|
if (input) {
|
|
30
30
|
input = (""+input).replace(" ", "").toLowerCase();
|
|
31
|
-
hasInput = input && input.charAt(0) !==
|
|
31
|
+
hasInput = input && input.charAt(0) !== "_" && input !== "index";
|
|
32
32
|
}
|
|
33
33
|
|
|
34
34
|
try {
|
|
@@ -36,7 +36,7 @@ module.exports = {
|
|
|
36
36
|
result = require( "./" + input);
|
|
37
37
|
}
|
|
38
38
|
} catch (e) {
|
|
39
|
-
console.error(
|
|
39
|
+
console.error(`Input generator load failed '${input}'`, e);
|
|
40
40
|
/* return the "null" generator on error */
|
|
41
41
|
}
|
|
42
42
|
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
* Licensed under the MIT license.
|
|
9
9
|
*/
|
|
10
10
|
|
|
11
|
-
const fs = require("fs")
|
|
11
|
+
const { promises: fs } = require("node:fs");
|
|
12
12
|
const common = require("../common");
|
|
13
13
|
const { sitemapCollection: smc } = require("../common/sitemap");
|
|
14
14
|
const base = require("./_base");
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
* Licensed under the MIT license.
|
|
9
9
|
*/
|
|
10
10
|
|
|
11
|
-
const fs = require("fs")
|
|
11
|
+
const { promises: fs } = require("node:fs");
|
|
12
12
|
const base = require("./_base");
|
|
13
13
|
const common = require("../common");
|
|
14
14
|
|
|
@@ -50,7 +50,7 @@ function generateInput (options) {
|
|
|
50
50
|
.then(data => {
|
|
51
51
|
if (data) {
|
|
52
52
|
let error;
|
|
53
|
-
data.toString().split(
|
|
53
|
+
data.toString().split("\n").every(line => {
|
|
54
54
|
const page = line.replace(/^\s+|\s+$/g, "");
|
|
55
55
|
|
|
56
56
|
if (!base.input(options, page)) {
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
#!/usr/bin/env -S node --unhandled-rejections=strict
|
|
2
|
+
/**
|
|
3
|
+
* playwright process entry point
|
|
4
|
+
*
|
|
5
|
+
* Command line arguments:
|
|
6
|
+
* 0 - /path/to/node
|
|
7
|
+
* 1 - *this script*
|
|
8
|
+
* 2 - outputFile
|
|
9
|
+
* 3 - url
|
|
10
|
+
* 4 - selector
|
|
11
|
+
* [5] - timeout
|
|
12
|
+
* [6] - filter
|
|
13
|
+
* [7] - debug
|
|
14
|
+
* [8] - slowMo
|
|
15
|
+
* [9] - arbitrary launchOptions (JSON)
|
|
16
|
+
*
|
|
17
|
+
* launchOptions may include a `browserType` key ("chromium" | "firefox" | "webkit")
|
|
18
|
+
* to select the browser engine. Defaults to "chromium".
|
|
19
|
+
*
|
|
20
|
+
* Copyright (c) 2013 - 2025 Alex Grant, LocalNerve, contributors
|
|
21
|
+
* Licensed under the MIT license.
|
|
22
|
+
*/
|
|
23
|
+
const path = require("node:path");
|
|
24
|
+
const { promises: fs } = require("node:fs");
|
|
25
|
+
const playwright = require("playwright");
|
|
26
|
+
|
|
27
|
+
if (process.argv.length < 5) {
|
|
28
|
+
throw Error(`playwright script '${
|
|
29
|
+
process.argv[1]
|
|
30
|
+
}' expected these arguments:\n\
|
|
31
|
+
OUTPUTFILE URL SELECTOR [TIMEOUT FILTER DEBUG SLOWMO LAUNCHOPTIONS]`);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
const launchOpts = process.argv[9] ? JSON.parse(process.argv[9]) : null;
|
|
35
|
+
|
|
36
|
+
// browserType is a playwright concern, not a launch option — extract it before
|
|
37
|
+
// passing the rest through to playwright.launch()
|
|
38
|
+
const { browserType: browserTypeKey = "chromium", ...extraLaunchOpts } = launchOpts ?? {};
|
|
39
|
+
|
|
40
|
+
if (!["chromium", "firefox", "webkit"].includes(browserTypeKey)) {
|
|
41
|
+
throw Error(`playwright script: unsupported browserType '${browserTypeKey}'. \
|
|
42
|
+
Must be one of: chromium, firefox, webkit`);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
const options = {
|
|
46
|
+
outputFile: process.argv[2],
|
|
47
|
+
url: process.argv[3],
|
|
48
|
+
selector: process.argv[4],
|
|
49
|
+
timeout: process.argv[5] ? parseInt(process.argv[5], 10) : 10000,
|
|
50
|
+
filter: process.argv[6] && process.argv[6] !== "false" ? process.argv[6] : null,
|
|
51
|
+
debug: process.argv[7] && process.argv[7].toLowerCase() === "true",
|
|
52
|
+
slowMo: process.argv[8] ? parseInt(process.argv[8], 10) : 500,
|
|
53
|
+
};
|
|
54
|
+
|
|
55
|
+
const launchOptions = {
|
|
56
|
+
...(options.debug ? {
|
|
57
|
+
headless: false,
|
|
58
|
+
slowMo: options.slowMo,
|
|
59
|
+
devtools: browserTypeKey === "chromium" // devtools only supported in chromium
|
|
60
|
+
} : {
|
|
61
|
+
headless: true
|
|
62
|
+
}),
|
|
63
|
+
...extraLaunchOpts
|
|
64
|
+
};
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Run the snapshot script.
|
|
68
|
+
*
|
|
69
|
+
* @returns {Promise<string>} on completion
|
|
70
|
+
*/
|
|
71
|
+
async function run () {
|
|
72
|
+
let timeLeft, browser;
|
|
73
|
+
const start = Date.now();
|
|
74
|
+
|
|
75
|
+
timeLeft = options.timeout - 200; // leave some room for reporting
|
|
76
|
+
if (timeLeft < 0) timeLeft = 0;
|
|
77
|
+
|
|
78
|
+
try {
|
|
79
|
+
const filter =
|
|
80
|
+
options.filter ? require(options.filter) : passthru => passthru;
|
|
81
|
+
|
|
82
|
+
const browserType = playwright[browserTypeKey];
|
|
83
|
+
browser = await browserType.launch(launchOptions);
|
|
84
|
+
const page = await browser.newPage();
|
|
85
|
+
|
|
86
|
+
await page.goto(options.url, {
|
|
87
|
+
timeout: timeLeft
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
timeLeft -= (Date.now() - start);
|
|
91
|
+
if (timeLeft < 0) timeLeft = 0;
|
|
92
|
+
|
|
93
|
+
await fs.mkdir(path.dirname(options.outputFile), {
|
|
94
|
+
recursive: true
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
await page.waitForSelector(options.selector, {
|
|
98
|
+
state: "visible",
|
|
99
|
+
timeout: timeLeft
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
const content = await page.content();
|
|
103
|
+
await fs.writeFile(options.outputFile, filter(content));
|
|
104
|
+
const elapsed = Date.now() - start;
|
|
105
|
+
|
|
106
|
+
await browser.close();
|
|
107
|
+
|
|
108
|
+
return `snapshot for ${options.url} finished in ${elapsed} ms\n\
|
|
109
|
+
written to ${options.outputFile}`;
|
|
110
|
+
} catch (e) {
|
|
111
|
+
const err = new Error(`failed to snapshot ${options.url}`, {
|
|
112
|
+
cause: e
|
|
113
|
+
});
|
|
114
|
+
if (browser) {
|
|
115
|
+
browser.close().finally(() => {
|
|
116
|
+
throw err;
|
|
117
|
+
});
|
|
118
|
+
}
|
|
119
|
+
throw err;
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
run().then(console.log).catch(e => {
|
|
124
|
+
throw e;
|
|
125
|
+
});
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Remove all script tags from content
|
|
3
|
+
*
|
|
4
|
+
* Copyright (c) 2013 - 2025 Alex Grant, LocalNerve, contributors
|
|
5
|
+
* Licensed under the MIT license.
|
|
6
|
+
*/
|
|
7
|
+
function removeScriptTags (content) {
|
|
8
|
+
return content.replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, "");
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
module.exports = removeScriptTags;
|
package/lib/puppeteer/index.js
CHANGED
|
@@ -17,9 +17,9 @@
|
|
|
17
17
|
* Copyright (c) 2013 - 2025 Alex Grant, LocalNerve, contributors
|
|
18
18
|
* Licensed under the MIT license.
|
|
19
19
|
*/
|
|
20
|
-
const
|
|
21
|
-
const
|
|
22
|
-
const
|
|
20
|
+
const path = require("node:path");
|
|
21
|
+
const { promises: fs } = require("node:fs");
|
|
22
|
+
const puppeteer = require("puppeteer");
|
|
23
23
|
|
|
24
24
|
if (process.argv.length < 5) {
|
|
25
25
|
throw Error(`puppeteer script '${
|
|
@@ -33,8 +33,8 @@ const options = {
|
|
|
33
33
|
url: process.argv[3],
|
|
34
34
|
selector: process.argv[4],
|
|
35
35
|
timeout: process.argv[5] ? parseInt(process.argv[5], 10) : 10000,
|
|
36
|
-
filter: process.argv[6] && process.argv[6] !==
|
|
37
|
-
debug: process.argv[7] && process.argv[7].toLowerCase() ===
|
|
36
|
+
filter: process.argv[6] && process.argv[6] !== "false" ? process.argv[6] : null,
|
|
37
|
+
debug: process.argv[7] && process.argv[7].toLowerCase() === "true" ? true : false,
|
|
38
38
|
slowMo: process.argv[8] ? parseInt(process.argv[8], 10) : 500,
|
|
39
39
|
launchOpts: process.argv[9] ? JSON.parse(process.argv[9]) : null
|
|
40
40
|
};
|
|
@@ -44,7 +44,7 @@ const launchOptions = {
|
|
|
44
44
|
slowMo: options.slowMo,
|
|
45
45
|
devtools: true
|
|
46
46
|
} : {
|
|
47
|
-
headless:
|
|
47
|
+
headless: true
|
|
48
48
|
}),
|
|
49
49
|
...(options.launchOpts ?? {})
|
|
50
50
|
};
|
|
@@ -79,7 +79,7 @@ async function run () {
|
|
|
79
79
|
});
|
|
80
80
|
|
|
81
81
|
await page.waitForSelector(options.selector, {
|
|
82
|
-
|
|
82
|
+
visible: true,
|
|
83
83
|
timeout: timeLeft
|
|
84
84
|
});
|
|
85
85
|
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
* Licensed under the MIT license.
|
|
6
6
|
*/
|
|
7
7
|
function removeScriptTags (content) {
|
|
8
|
-
return content.replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi,
|
|
8
|
+
return content.replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, "");
|
|
9
9
|
}
|
|
10
10
|
|
|
11
11
|
module.exports = removeScriptTags;
|
package/package.json
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "html-snapshots",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "6.1.0",
|
|
4
4
|
"author": {
|
|
5
5
|
"name": "Alex Grant",
|
|
6
6
|
"email": "alex@localnerve.com",
|
|
7
7
|
"url": "https://www.localnerve.com"
|
|
8
8
|
},
|
|
9
9
|
"homepage": "https://github.com/localnerve/html-snapshots",
|
|
10
|
-
"description": "A selector-based html snapshot tool using Puppeteer or
|
|
10
|
+
"description": "A selector-based html snapshot tool using Puppeteer or Playwright that sources sitemap.xml, robots.txt, or arbitrary input",
|
|
11
11
|
"contributors": [
|
|
12
12
|
{
|
|
13
13
|
"name": "Alex Grant",
|
|
@@ -19,26 +19,38 @@
|
|
|
19
19
|
},
|
|
20
20
|
"scripts": {
|
|
21
21
|
"lint": "eslint .",
|
|
22
|
-
"
|
|
23
|
-
"
|
|
24
|
-
"
|
|
25
|
-
"test
|
|
26
|
-
"
|
|
27
|
-
"test:
|
|
28
|
-
"
|
|
29
|
-
"test:
|
|
30
|
-
"
|
|
31
|
-
"test:
|
|
32
|
-
"
|
|
33
|
-
"test:
|
|
34
|
-
"
|
|
35
|
-
"
|
|
22
|
+
"_cleancov": "node -e \"const fs=require('fs');fs.rmSync('coverage',{recursive:true,force:true});fs.mkdirSync('coverage',{recursive:true})\"",
|
|
23
|
+
"_test": "node --test --experimental-test-coverage --test-reporter=spec --test-reporter-destination=stdout --test-reporter=lcov --test-reporter-destination=\"coverage/lcov.info\"",
|
|
24
|
+
"pretest": "npm run _cleancov",
|
|
25
|
+
"test": "npm run _test -- --test-coverage-exclude=\"test/**\" --test-concurrency=1 \"test/suites/**/*.spec.js\"",
|
|
26
|
+
"pretest:24": "npm run _cleancov",
|
|
27
|
+
"test:24": "npm run _test -- --test-coverage-exclude=\"test/**\" --test-isolation=none \"test/suites/**/*.spec.js\"",
|
|
28
|
+
"pretest:async": "npm run _cleancov",
|
|
29
|
+
"test:async": "npm run _test -- \"test/suites/async/*.spec.js\" --test-coverage-exclude=\"test/**\"",
|
|
30
|
+
"pretest:common": "npm run _cleancov",
|
|
31
|
+
"test:common": "npm run _test -- \"test/suites/common/*.spec.js\" --test-coverage-exclude=\"test/**\"",
|
|
32
|
+
"pretest:browsers": "npm run _cleancov",
|
|
33
|
+
"test:browsers": "npm run _test -- \"test/suites/browsers/*.spec.js\" --test-coverage-exclude=\"test/**\"",
|
|
34
|
+
"pretest:input-generators": "npm run _cleancov",
|
|
35
|
+
"test:input-generators": "npm run _test -- \"test/suites/input-generators/*.spec.js\" --test-coverage-exclude=\"test/**\"",
|
|
36
|
+
"pretest:html-snapshots": "npm run _cleancov",
|
|
37
|
+
"test:html-snapshots": "node --test --experimental-test-coverage --test-reporter=spec --test-reporter-destination=stdout --test-reporter=lcov --test-reporter-destination=\"coverage/lcov.info\" \"test/suites/html-snapshots/*.spec.js\" --test-coverage-exclude=\"test/**\""
|
|
36
38
|
},
|
|
37
39
|
"main": "./lib/html-snapshots",
|
|
38
40
|
"repository": {
|
|
39
41
|
"type": "git",
|
|
40
42
|
"url": "https://github.com/localnerve/html-snapshots.git"
|
|
41
43
|
},
|
|
44
|
+
"funding": [
|
|
45
|
+
{
|
|
46
|
+
"type": "github",
|
|
47
|
+
"url": "https://github.com/sponsors/localnerve"
|
|
48
|
+
},
|
|
49
|
+
{
|
|
50
|
+
"type": "paypal",
|
|
51
|
+
"url": "https://www.paypal.com/ncp/payment/DHCB5GUYMGX5U"
|
|
52
|
+
}
|
|
53
|
+
],
|
|
42
54
|
"keywords": [
|
|
43
55
|
"SEO",
|
|
44
56
|
"html",
|
|
@@ -52,30 +64,21 @@
|
|
|
52
64
|
"dependencies": {
|
|
53
65
|
"async": "3.2.6",
|
|
54
66
|
"async-lock": "1.4.1",
|
|
55
|
-
"
|
|
56
|
-
"
|
|
57
|
-
"puppeteer": "^24.43.0",
|
|
67
|
+
"playwright": "1.61.0",
|
|
68
|
+
"puppeteer": "25.1.0",
|
|
58
69
|
"xml2js": "0.6.2"
|
|
59
70
|
},
|
|
60
71
|
"devDependencies": {
|
|
61
|
-
"eslint": "^10.3.0",
|
|
62
72
|
"@eslint/js": "^10.0.1",
|
|
63
|
-
"
|
|
73
|
+
"eslint": "^10.5.0",
|
|
74
|
+
"eslint-plugin-n": "18.1.0",
|
|
64
75
|
"express": "^5.2.1",
|
|
65
|
-
"
|
|
66
|
-
"c8": "^11.0.0",
|
|
76
|
+
"globals": "^17.6.0",
|
|
67
77
|
"server-destroy": "^1.0.1",
|
|
68
78
|
"sitemap-xml": "^0.1.0"
|
|
69
79
|
},
|
|
70
|
-
"overrides": {
|
|
71
|
-
"request": "npm:@cypress/request@4.0.0",
|
|
72
|
-
"diff": "^9.0.0",
|
|
73
|
-
"glob": "^13.0.6",
|
|
74
|
-
"serialize-javascript": "^7.0.5",
|
|
75
|
-
"brace-expansion": "^5.0.5"
|
|
76
|
-
},
|
|
77
80
|
"license": "MIT",
|
|
78
81
|
"engines": {
|
|
79
|
-
"node": ">=
|
|
82
|
+
"node": ">=22"
|
|
80
83
|
}
|
|
81
84
|
}
|