html-snapshots 0.19.0 → 1.0.1

package/examples/verbose/package.json

@@ -7,6 +7,6 @@
     "start": "node ./snapshot.js"
   },
   "dependencies": {
-    "html-snapshots": "^0.18.3"
+    "html-snapshots": "^1.0.0"
   }
 }

package/examples/verbose/snapshot.js

@@ -23,6 +23,7 @@ htmlSnapshots.run({
   outputDirClean: true,
   selector: "#mainContent",
   timeout: 10000,
+  browser: "phantomjs",
   processLimit: 1, // to clarify output
   verbose: verboseOption
 })

package/lib/html-snapshots.js

@@ -20,8 +20,6 @@ const _ = require("lodash");
 const common = require("./common");
 const inputFactory = require("./input-generators");
 const Notifier = require("./async").Notifier;
-const phantomDir = "./phantom";
-const snapshotScript = path.join(phantomDir, "default.js");
 
 /**
  * Determine the default phantomJS module path. This is overridden by the
@@ -56,13 +54,21 @@ function defaultPhantomJSExePath () {
   return phantomSource;
 }
 
+const phantomDir = "./phantom";
+const phantomSnapshotScript = path.join(phantomDir, "default.js");
+const puppeteerDir = "./puppeteer";
+const puppeteerSnapshotScript = path.join(puppeteerDir, "index.js");
+
 /**
  * This module's defaults
  */
 const defaults = Object.freeze({
   input: "robots",
+  browser: "puppeteer",
   phantomjs: defaultPhantomJSExePath(),
-  snapshotScript: path.join(__dirname, snapshotScript),
+  phantomSnapshotScript: path.join(__dirname, phantomSnapshotScript),
+  puppeteer: path.join(__dirname, puppeteerSnapshotScript),
+  snapshotScript: path.join(__dirname, puppeteerSnapshotScript),
   outputDirClean: false,
   pollInterval: 500,
   processLimit: 4
@@ -92,7 +98,7 @@ const defaults = Object.freeze({
  * @param {Function} notifier.known - Check if a file is or has been processed.
  * @param {Function} qcb - async.queue callback function.
  */
-function worker (input, options, notifier, qcb) {
+function phantomjsWorker (input, options, notifier, qcb) {
   let cp, customModule;
   let snapshotScript = options.snapshotScript;
   const phantomjsOptions =
@@ -142,6 +148,73 @@ function worker (input, options, notifier, qcb) {
   }
 }
 
+/**
+ * @param {Object} input - An input object from an input generator.
+ * @param {String} input.outputFile - The output file that the html is written to.
+ * @param {String} input.url - The url to get the html from.
+ * @param {String} input.selector - The selector to wait for.
+ * @param {String} input.timeout - The entire process timeout.
+ * @param {String} input.debug - The debug object.
+ * @param {Object} options - Input options.
+ * @param {Object} options.snapshotScript - Snapshot script options.
+ * @param {String} options.snapshotScript.script - Snapshot script name.
+ * @param {String} options.snapshotScript.module - Snapshot script module.
+ * @param {String} options.puppeteer - Full path to the puppeteer exec.
+ * @param {Object} notifier - The async Notifier instance for this run.
+ * @param {Function} notifier.remove - Remove an outputFile from notifier.
+ * @param {Function} notifier.setError - Set an error on the notifier.
+ * @param {Function} notifier.add - Add a file, timeout to the notifier.
+ * @param {Function} notifier.known - Check if a file is or has been processed.
+ * @param {Function} qcb - async.queue callback function.
+ */
+function puppeteerWorker (input, options, notifier, qcb) {
+  let filter = "false";
+  let snapshotScript = options.snapshotScript;
+  
+  // If the outputFile has NOT already been seen by the notifier, process.
+  if (!notifier.known(input.outputFile)) {
+
+    // map snapshotScript object script
+    if (_.isObject(options.snapshotScript)) {
+      snapshotScript = options.puppeteer;
+      if (options.snapshotScript.script === "removeScripts") {
+        filter = "./removeScripts";
+      } else {
+        filter = options.snapshotScript.module;
+      }
+    }
+
+    const cp = spawn(
+      snapshotScript,
+      [ input.outputFile,
+        input.url,
+        input.selector,
+        input.timeout,
+        filter,
+        input.debug.flag,
+        input.debug.slowMo
+      ], { cwd: process.cwd(), stdio: "inherit", detached: true }
+    );
+
+    cp.on("error", function (e) {
+      notifier.remove(input.outputFile);
+      notifier.setError(e);
+      console.error(e);
+      qcb(e);
+    });
+
+    cp.on("exit", function (code) {
+      qcb(code);
+    });
+
+    // start counting
+    notifier.add(input.outputFile, input.timeout);
+  } else {
+    // The input.outputFile is being or has been processed this run.
+    qcb(0);
+  }
+}
+
 /**
  * Prepare html snapshots options.
  *
@@ -156,6 +229,13 @@ function prepOptions (options) {
   if (Array.isArray(options.source)) {
     options.input = "array";
   }
+
+  // If phantomjs browser, make sure the default snapshotScript is correctly set.
+  if (options.browser === 'phantomjs') {
+    if (!options.snapshotScript || options.snapshotScript === options.puppeteer) {
+      options.snapshotScript = options.phantomSnapshotScript;
+    }
+  }
 }
 
 module.exports = {
@@ -226,6 +306,9 @@ module.exports = {
       // expose abort callback to input generators via options.
       options._abort = notifier.abort.bind(notifier, q);
 
+      const worker =
+        options.browser === 'puppeteer' ? puppeteerWorker : phantomjsWorker;
+
       // generate input for the snapshots.
       return inputGenerator.run(options, input => {
         q.push(_.partial(worker, input, options, notifier));

package/lib/input-generators/_base.js

@@ -26,7 +26,11 @@ const defaults = Object.freeze({
   checkInterval: 250,
   useJQuery: false,
   verbose: false,
-  phantomjsOptions: ""
+  phantomjsOptions: "",
+  debug: {
+    flag: "false",
+    slowMo: 500
+  }
 });
 
 /**
@@ -224,6 +228,7 @@ module.exports = {
    * @param {Function} options.userJQuery - Produces a jquery bool given a page url.
    * @param {Function} options.verbose - Produces a verbose bool given a page url.
    * @param {Function} options.phantomjsOptions - Produces phantomjs options given a page url.
+   * @param {Function} options.debug - Produces debug options given a page url.
    * @param {Function} options._inputEmitter - The EventEmitter to generate input.
    * @param {String} page - The url to the page resource.
    * @returns {String} The full output file path.
@@ -258,6 +263,7 @@ module.exports = {
         // map the input page to phantomJS options
         phantomjsOptions: options.phantomjsOptions(page),
         // useful for testing, debugging
+        debug: options.debug,
         __page: page
       });
 

package/lib/puppeteer/index.js

@@ -0,0 +1,98 @@
+#!/usr/bin/env -S node --unhandled-rejections=strict
+/**
+ * puppeteer process entry point
+ * 
+ * Command line arguments:
+ * 0   - /path/to/node
+ * 1   - *this script*
+ * 2   - outputFile
+ * 3   - url
+ * 4   - selector
+ * [5] - timeout
+ * [6] - filter
+ * [7] - debug
+ * [8] - slowMo
+ */
+const puppeteer = require('puppeteer');
+const path = require('path');
+const { promises: fs } = require('fs');
+
+if (process.argv.length < 5) {
+  throw Error(`puppeteer script '${
+    process.argv[1]
+  }' expected these arguments:\n\
+  OUTPUTFILE URL SELECTOR [TIMEOUT FILTER DEBUG SLOWMO]`);
+}
+
+const options = {
+  outputFile: process.argv[2],
+  url: process.argv[3],
+  selector: process.argv[4],
+  timeout: process.argv[5] ? parseInt(process.argv[5], 10) : 10000,
+  filter: process.argv[6] && process.argv[6] !== 'false' ? process.argv[6] : null,
+  debug: process.argv[7] && process.argv[7].toLowerCase() === 'true' ? true : false,
+  slowMo: process.argv[8] ? parseInt(process.argv[8], 10) : 500
+};
+const launchOptions = options.debug ? {
+  headless: false,
+  slowMo: options.slowMo,
+  devtools: true
+} : {};
+
+/**
+ * Run the snapshot script.
+ * 
+ * @returns {Promise} on completion
+ */
+async function run () {
+  let timeLeft, browser;
+  const start = Date.now();
+
+  timeLeft = options.timeout - 200; // leave some room for reporting
+  if (timeLeft < 0) timeLeft = 0;
+
+  try {
+    const filter =
+      options.filter ? require(options.filter) : passthru => passthru;
+
+    browser = await puppeteer.launch(launchOptions);
+    const page = await browser.newPage();
+    await page.goto(options.url, {
+      timeout: timeLeft
+    });
+
+    timeLeft -= (Date.now() - start);
+    if (timeLeft < 0) timeLeft = 0;
+
+    await fs.mkdir(path.dirname(options.outputFile), {
+      recursive: true
+    });
+
+    await page.waitForSelector(options.selector, {
+      Visible: true,
+      timeout: timeLeft
+    });
+
+    const content = await page.content();
+    await fs.writeFile(options.outputFile, filter(content));
+    const elapsed = Date.now() - start;
+
+    await browser.close();
+    
+    return `snapshot for ${options.url} finished in ${elapsed} ms\n\
+  written to ${options.outputFile}`;
+    } catch (e) {
+      const err = new Error(`failed to snapshot ${options.url}`, {
+        cause: e
+      });
+      if (browser) {
+        browser.close().finally(() => {
+          throw err;
+        });
+      }
+      throw err;
+    }
+}
+run().then(console.log).catch(e => {
+  throw e;
+});

package/lib/puppeteer/removeScripts.js

@@ -0,0 +1,8 @@
+/**
+ * Remove all script tags from content
+ */
+function removeScriptTags (content) {
+  return content.replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, '');
+}
+
+module.exports = removeScriptTags;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "html-snapshots",
-  "version": "0.19.0",
+  "version": "1.0.1",
   "author": {
     "name": "Alex Grant",
     "email": "alex@localnerve.com",
@@ -22,7 +22,10 @@
     "test": "mocha test/mocha/**/*.js --exit --recursive --reporter spec",
     "test:async": "mocha test/mocha/async/test.js --exit --reporter spec",
     "test:common": "mocha test/mocha/common/*.js --exit --reporter spec",
+    "test:browsers": "mocha test/mocha/browsers/test.js --exit --reporter spec",
+    "test:browsers:cover": "c8 -- npm run test:browsers",
     "test:input-generators": "mocha test/mocha/input-generators/*.js --exit --reporter spec",
+    "test:input-generators:debug": "mocha --inspect-brk test/mocha/input-generators/*.js --exit --reporter spec",
     "test:html-snapshots": "mocha test/mocha/html-snapshots/*.js --exit --reporter spec",
     "test:html-snapshots:debug": "mocha --inspect-brk test/mocha/html-snapshots/*.js --exit --reporter spec",
     "test:cover": "c8 -- npm test",
@@ -50,10 +53,11 @@
     "async": "3.2.4",
     "async-lock": "1.3.2",
     "combine-errors": "3.0.3",
+    "got": "12.5.1",
     "lodash": "4.17.21",
     "mkdirp": "1.0.4",
     "phantomjs-prebuilt": "2.1.16",
-    "got": "12.5.0",
+    "puppeteer": "^18.0.5",
     "rimraf": "3.0.2",
     "xml2js": "0.4.23"
   },

package/test/mocha/browsers/puppeteer.js

@@ -0,0 +1,106 @@
+/**
+ * Tests that target the puppeteer script specifically.
+ * 
+ * Copyright (c) 2013 - 2022, Alex Grant, LocalNerve, contributors
+ */
+/* global before, it */
+const assert = require("assert");
+const path = require("path");
+const rimraf = require("rimraf").sync;
+const spawn = require("child_process").spawn;
+const server = require("../../server");
+
+const outputDir = path.join(__dirname, "tmp");
+const port = 9340;
+
+function puppeteerTests () {
+  const snapshotScript = path.resolve(__dirname, '../../../lib/puppeteer/index.js');
+  const outputFile = path.join(outputDir, "snapshot.html");
+
+  /**
+   * Launch the puppeteer script as a process.
+   *
+   * @param {Array} args - array of puppeteer args
+   */
+  function spawnPuppeteer (args, done) {
+    const cp = spawn(
+      snapshotScript,
+      args
+      , { cwd: process.cwd(), stdio: "inherit", detached: true }
+    );
+    
+    cp.on("exit", code => {
+      if (code !== 0) {
+        return done(new Error(`error exit code: ${code}`));
+      }
+      done();
+    });
+  }
+
+  return function () {
+    before(function() {
+      server.start(path.join(__dirname, "./server"), port);
+    });
+    
+    it("should succeed with minimal args", function (done) {
+      this.timeout(5000);
+
+      rimraf(outputDir);
+
+      spawnPuppeteer([
+        outputFile,
+        `http://localhost:${port}/`,
+        "body"
+      ], done);
+    });
+
+    it("should fail if insufficient input args", function (done) {
+      rimraf(outputDir);
+      
+      spawnPuppeteer([
+        outputFile,
+        "http://someurl.domain/path/resource"
+      ], e => {
+        const msg = "expected process error";
+        assert.ok(e, msg);
+        done(e ? undefined : new Error(msg));
+      });
+    });
+
+    it("should fail with bad url", function (done) {
+      rimraf(outputDir);
+
+      spawnPuppeteer([
+        outputFile,
+        "http://nonexistent.domain/path/resource",
+        "body",
+        1000
+      ], e => {
+        const msg = "expected process error";
+        assert.ok(e, msg);
+        done(e ? undefined : new Error(msg));
+      });
+    });
+
+    it("should launch debugging", function (done) {
+      const timeout = 5000;
+      this.timeout(timeout);
+
+      rimraf(outputDir);
+
+      spawnPuppeteer([
+        outputFile,
+        `http://localhost:${port}/`,
+        "body",
+        timeout - 1000,
+        "false",
+        "true",
+        10
+      ], done);
+    });
+  };
+}
+
+module.exports = {
+  testSuite: puppeteerTests
+};

package/test/mocha/browsers/server/index.html

@@ -0,0 +1,9 @@
+<!DOCTYPE html>
+<html lang="en-US">
+  <body>
+    <h1>Hello World</h1>
+    <p>
+      This is a simple test file, how do you do?
+    </p>
+  </body>
+</html>

package/test/mocha/browsers/test.js

@@ -0,0 +1,11 @@
+/**
+ * browser process specific tests
+ * 
+ * Copyright (c) 2013 - 2022, Alex Grant, LocalNerve, contributors
+ */
+/* global describe */
+const puppeteer = require("./puppeteer");
+
+describe("browser process tests", function () {
+  describe("puppeteer", puppeteer.testSuite());
+});

package/test/mocha/html-snapshots/basics.js

@@ -4,68 +4,81 @@
  * Copyright (c) 2013 - 2022, Alex Grant, LocalNerve, contributors
  */
 /* global it */
-var assert = require("assert");
-var _ = require("lodash");
-var fs = require("fs");
-var path = require("path");
-var ss = require("../../../lib/html-snapshots");
-var utils = require("./utils");
-var optHelp = require("../../helpers/options");
+const assert = require("assert");
+const _ = require("lodash");
+const fs = require("fs");
+const path = require("path");
+const ss = require("../../../lib/html-snapshots");
+const utils = require("./utils");
+const optHelp = require("../../helpers/options");
 
 // missing destructuring, will write postcard...
-var bogusFile = utils.bogusFile;
-var cleanupError = utils.cleanupError;
-var unexpectedSuccess = utils.unexpectedSuccess;
+const {
+  bogusFile,
+  cleanupError,
+  unexpectedSuccess
+} = utils;
 
-function basicTests () {
+function basicTests (options) {
+  const {
+    browsers
+  } = options;
   return function () {
-    it("no arguments should fail", function (done) {
-      var twice = _.after(2, cleanupError.bind(null, done, 0));
+    browsers.forEach(browser => {
+      it(`no arguments should fail - ${browser}`, function (done) {
+        const twice = _.after(2, cleanupError.bind(null, done, 0));
 
-      ss.run(optHelp.decorate({}), twice)
-       .then(unexpectedSuccess.bind(null, done))
-       .catch(twice);
-    });
+        ss.run(optHelp.decorate({
+          browser
+        }), twice)
+        .then(unexpectedSuccess.bind(null, done))
+        .catch(twice);
+      });
 
-    it("invalid source should fail", function (done) {
-      var twice = _.after(2, cleanupError.bind(null, done, 0));
+      it(`invalid source should fail - ${browser}`, function (done) {
+        const twice = _.after(2, cleanupError.bind(null, done, 0));
 
-      ss.run(optHelp.decorate({ source: bogusFile }), twice)
-       .then(unexpectedSuccess.bind(null, done))
-       .catch(twice);
-    });
+        ss.run(optHelp.decorate({
+          source: bogusFile,
+          browser
+        }), twice)
+        .then(unexpectedSuccess.bind(null, done))
+        .catch(twice);
+      });
 
-    it("should clean the output directory when specified", function (done) {
-      var dir = path.join(__dirname, "./tmpdir");
-      var file = path.join(dir, "somefile.txt");
-      var twice = _.after(2, cleanupError.bind(null, done, 0));
+      it(`should clean the output directory when specified - ${browser}`, function (done) {
+        const dir = path.join(__dirname, "./tmpdir");
+        const file = path.join(dir, "somefile.txt");
+        const twice = _.after(2, cleanupError.bind(null, done, 0));
 
-      fs.mkdirSync(dir);
-      fs.writeFileSync(file, "some data");
-      assert.doesNotThrow(fs.accessSync.bind(null, dir));
+        fs.mkdirSync(dir);
+        fs.writeFileSync(file, "some data");
+        assert.doesNotThrow(fs.accessSync.bind(null, dir));
 
-      ss.run(optHelp.decorate({
-       source: bogusFile,
-       outputDir: dir,
-       outputDirClean: true
-      }), twice)
-       .then(unexpectedSuccess.bind(null, done))
-       .catch(twice);
+        ss.run(optHelp.decorate({
+          source: bogusFile,
+          outputDir: dir,
+          outputDirClean: true,
+          browser
+        }), twice)
+        .then(unexpectedSuccess.bind(null, done))
+        .catch(twice);
 
-      assert.throws(fs.accessSync.bind(null, dir));
-    });
+        assert.throws(fs.accessSync.bind(null, dir));
+      });
 
-    it("default snapshot script should exist", function (done) {
-      var options = { source: "./bogus/file.txt" };
-      var twice = _.after(2, cleanupError.bind(null, done, 0));
+      it(`default snapshot script should exist - ${browser}`, function (done) {
+        const options = { source: "./bogus/file.txt", browser };
+        const twice = _.after(2, cleanupError.bind(null, done, 0));
 
-      var result = ss.run(optHelp.decorate(options), twice);
+        const result = ss.run(optHelp.decorate(options), twice);
 
-      assert.doesNotThrow(fs.accessSync.bind(null, options.snapshotScript));
+        assert.doesNotThrow(fs.accessSync.bind(null, options.snapshotScript));
 
-      result
-       .then(unexpectedSuccess.bind(null, done))
-       .catch(twice);
+        result
+        .then(unexpectedSuccess.bind(null, done))
+        .catch(twice);
+      });
     });
   };
 }