html-snapshots 0.18.3 → 1.0.0

package/examples/verbose/package.json

@@ -7,6 +7,6 @@
     "start": "node ./snapshot.js"
   },
   "dependencies": {
-    "html-snapshots": "^0.18.2"
+    "html-snapshots": "^1.0.0"
   }
 }

package/examples/verbose/snapshot.js

@@ -5,32 +5,30 @@
  * Use sitemap.xml to snapshot an entire site.
  * Use the verbose option to debug the loading of one of the pages.
  */
-var htmlSnapshots = require("html-snapshots");
+const htmlSnapshots = require("html-snapshots");
 
-var origin = "http://enigmatic-refuge-9006.herokuapp.com:80";
-
-var pageToDebug = origin + "/sample-page";
+const origin = "https://www.sitemaps.org";
+const pageToDebug = `${origin}/en_GB/`;
 
 // Make the verbose option as an object keyed with the page to debug.
-var verboseOption = {
-  __default: false
+const verboseOption = {
+  __default: false,
+  [pageToDebug]: true
 };
-verboseOption[pageToDebug] = true;
 
 htmlSnapshots.run({
   input: "sitemap",
-  source: origin + "/sitemap.xml",
+  source: `${origin}/sitemap.xml`,
   outputDir: require("path").join(__dirname, "./tmp"),
   outputDirClean: true,
-  selector: ".page-content",
-  timeout: 15000,
+  selector: "#mainContent",
+  timeout: 10000,
+  browser: "phantomjs",
   processLimit: 1, // to clarify output
   verbose: verboseOption
 })
-.then(function (completed) {
+.then(completed => {
   console.log("completed snapshots:");
   console.log(require("util").inspect(completed));
 })
-.catch(function (err) {
-  console.error(err);
-});
+.catch(console.error);

package/lib/async/index.js

@@ -7,7 +7,6 @@
  * Copyright (c) 2013 - 2022, Alex Grant, LocalNerve, contributors
  * Licensed under the MIT license.
  */
-/* global Promise */
 
 const crypto = require("crypto");
 const AsyncLock = require("async-lock");

package/lib/common/index.js

@@ -48,7 +48,7 @@ function isUrl (obj) {
  * If undefined, then pass through undefined.
  * @param {String} [message] - The message to prepend.
  * @param {Boolean} [quoteInput] - True if the message should be quoted.
- * @returns {Error} The updated Error instance, undefined if no error.
+ * @returns {Error|Undefined} The updated Error instance, *undefined if no error*.
  */
 function prependMsgToErr (error, message, quoteInput) {
   let result, prepend;

package/lib/common/sitemap/index.js

@@ -0,0 +1,16 @@
+/**
+ * Reused sitemap modules.
+ * Used by robots, sitemap, and sitemap-index input generators.
+ *
+ * Copyright (c) 2013 - 2022 Alex Grant, LocalNerve, contributors
+ * Licensed under the MIT license.
+ */
+const sitemap = require("./sitemap");
+const smc = require("./sitemap-collection");
+const smi = require("./sitemap-index");
+
+module.exports = {
+  sitemapCollection: { process: smc.processSitemaps.bind(null, sitemap) },
+  sitemapIndex: { process: smi.sitemapIndex.bind(null, sitemap) },
+  sitemap
+};

package/lib/common/sitemap/sitemap-collection.js

@@ -0,0 +1,94 @@
+/**
+ * sitemap-index.js (common functions)
+ * 
+ * Copyright (c) 2013 - 2022 Alex Grant, LocalNerve, contributors
+ * Licensed under the MIT license.
+ */
+const fs = require("fs");
+const pathLib = require("path");
+const mkdirp = require("mkdirp");
+const common = require("../index");
+const base = require("../../input-generators/_base");
+
+/**
+ * Make sure path exists so a sitemap write can succeed.
+ *
+ * @param {String} outputPath - File path and name.
+ * @param {Function} callback - Completion callback.
+ */
+ function prepareWrite (outputPath, callback) {
+  const path = pathLib.parse(outputPath);
+  const dir = pathLib.join(path.root, path.dir);
+  mkdirp(dir).then(() => callback()).catch(callback);
+}
+
+/**
+ * Parse a sitemap, generate input, and optionally update the local file
+ * for sitemapPolicy purposes.
+ * This writes the sitemap files out EVEN IF sitemapPolicy is false.
+ * The thinking is that it's better/cheaper to do this b/c if sitemapPolicy is
+ * enabled later, then there is something to compare against, for more savings.
+ *
+ * @param {Object} options - Sitemap options.
+ * @param {Object} options.__sitemapIndex - Sitemap Index options.
+ * @param {String} document - xml document string.
+ * @param {Function} callback - A completion callback.
+ */
+function processSitemap (smLib, options, document, callback) {
+  smLib.parse(options, document, err => {
+    const sitemapIndexOpts = options.__sitemapIndex;
+    const outputPath = base.outputFile(sitemapIndexOpts, options.source);
+
+    if (!err && sitemapIndexOpts.sitemapOutputDir) {
+      prepareWrite(outputPath, err => {
+        if (!err) {
+          fs.writeFile(outputPath, document, callback);
+        } else {
+          callback(common.prependMsgToErr(err, outputPath, true));
+        }
+      });
+    } else {
+      callback(err);
+    }
+  });
+}
+
+/**
+ * Get and process a sitemaps collection.
+ * 
+ * @param {Object} smLib - Sitemap common module.
+ * @param {Object} options - Sitemap options.
+ * @param {Array} sitemapUrls - collection of sitemap urls to process.
+ * @param {Object} [sitemapIndexOptions] - Sitemap index options.
+ * @returns {Promise} resolves when all sitemaps are processed, reject as soon as one fails
+ */
+function processSitemaps (smLib, options, sitemapUrls, sitemapIndexOptions) {
+  // Edge case message for clarity
+  if (sitemapUrls.length === 0) {
+    console.log("[*] No sitemaps qualified for processing");
+    return Promise.resolve([]);
+  }
+
+  const smiOptions = sitemapIndexOptions || {
+    ...options,
+    ...{ outputPath: undefined }
+  };
+
+  // Get all the sitemaps, parse them, and generate input for each
+  return Promise.all(sitemapUrls.map(sitemapUrl => {
+    const sitemapOptions = Object.assign({}, options, {
+      source: sitemapUrl,
+      input: "sitemap",
+      sitemapOutputDir: false,
+      __sitemapIndex: smiOptions
+    });
+
+    console.log(`[+] Loading the sitemap: '${sitemapUrl}'`);
+
+    return smLib.getUrl(sitemapOptions, processSitemap.bind(null, smLib));
+  }));
+}
+
+module.exports = {
+  processSitemaps
+};

package/lib/common/sitemap/sitemap-index.js

@@ -0,0 +1,68 @@
+/**
+ * sitemap-index.js
+ * 
+ * Generates the urls for a sitemap-index.
+ * 
+ * Copyright (c) 2013 - 2022 Alex Grant, LocalNerve, contributors
+ * Licensed under the MIT license.
+ */
+const _ = require("lodash");
+const smc = require("./sitemap-collection");
+
+/**
+ * Create a collection of sitemap urls to process by policy, form sitemapIndexOptions.
+ *
+ * @param {Object} smLib - Sitemap module
+ * @param {Object} options - sitemap index options
+ * @param {Boolean} options.sitemapPolicy - True if sitemap policy should be used.
+ * @param {Object} parseResult - xml2js sax parse result object
+ * @returns {Object} The sitemapUrls collection and sitemapIndexOptions
+ */
+function sitemapIndexUrls (smLib, options, parseResult) {
+  const sitemapUrls = [];
+  const sitemapIndexOptions = {
+    ...options,
+    ...{ outputPath: undefined }
+  };
+
+  // Check if we should process each sitemap in the index.
+  _.forEach(
+    // if the sitemap index is malformed, just blow up
+    parseResult.sitemapindex.sitemap,
+    sitemapNode => {
+      // optionally ignore current sitemaps by sitemap policy
+      const shouldProcess = !options.sitemapPolicy ||
+        !smLib.stillCurrent(sitemapNode, sitemapIndexOptions);
+
+      if (shouldProcess) {
+        // if sitemap index is malformed, just blow up
+        sitemapUrls.push(sitemapNode.loc[0]);
+      }
+    }
+  );
+
+  return {
+    sitemapUrls,
+    sitemapIndexOptions
+  };
+}
+
+/**
+ * Collect sitemap urls from a sitemap index file for processing.
+ *
+ * @param {Object} smLib - sitemap module
+ * @param {Object} options - sitemapIndex options
+ * @param {Object} parseResult - xml2js sax parse result object
+ * @returns {Promise} resolves when all sitemaps are processed, reject as soon as one fails
+ */
+function sitemapIndex (smLib, options, parseResult) {
+  const {
+    sitemapUrls,
+    sitemapIndexOptions
+  } = sitemapIndexUrls(smLib, options, parseResult);
+  return smc.processSitemaps(smLib, options, sitemapUrls, sitemapIndexOptions);
+}
+
+module.exports = {
+  sitemapIndex
+};

package/lib/common/{sitemap.js → sitemap/sitemap.js}

@@ -9,14 +9,15 @@
  */
 
 const fs = require("fs");
+const util = require("util");
 const path = require("path");
 const urlm = require("url");
 const zlib = require("zlib");
 const _ = require("lodash");
 const xml2js = require("xml2js");
-const request = require("request");
-const common = require("./index");
-const base = require("../input-generators/_base");
+const smi = require("./sitemap-index");
+const common = require("../index");
+const base = require("../../input-generators/_base");
 
 // the out of date fallback
 const unixStart = "1970-01-01T00:00:00.000Z";
@@ -44,6 +45,7 @@ const changeFreq = {
  */
 function stillCurrent (urlNode, options) {
   let lesser, greater, oPath;
+  const statOpts = {throwIfNoEntry: false};
   const now = Date.now();
   const lMod = _.first(urlNode.lastmod);
   const cFreq = _.first(urlNode.changefreq) ? _.first(urlNode.changefreq).toLowerCase() : null;
@@ -53,7 +55,7 @@ function stillCurrent (urlNode, options) {
     // if sitemap is malformed, just blow up
     oPath = base.outputFile(options, urlNode.loc[0]);
 
-    lesser = now - ( (fs.existsSync(oPath) && fs.statSync(oPath).mtime.getTime()) || unixStart );
+    lesser = now - (fs.statSync(oPath, statOpts)?.mtime?.getTime() ?? unixStart);
     greater = now - Date.parse(lMod);
   }
   // only changefreq specified
@@ -61,7 +63,7 @@ function stillCurrent (urlNode, options) {
     // if sitemap is malformed, just blow up
     oPath = base.outputFile(options, urlNode.loc[0]);
 
-    lesser = now - ( (fs.existsSync(oPath) && fs.statSync(oPath).mtime.getTime()) || unixStart );
+    lesser = now - (fs.statSync(oPath, statOpts)?.mtime?.getTime() ?? unixStart);
     greater = changeFreq[cFreq] || changeFreq.always;
   }
   // both or neither were specified
@@ -91,43 +93,49 @@ function parse (options, document, callback) {
     normalize: true
   }, (err, result) => {
     let source = options.source;
-
     if (!err) {
-
-      // Process the url input, but break if base.input returns false.
-      //   In other words, _.find is looking for a non-falsy err.
-      // For now, this can only happen if no outputDir is defined,
-      //   which is a fatal bad option problem and will happen immediately.
-      _.find(
-        // if the sitemap is malformed, just blow up
-        result.urlset.url,
-        urlNode => {
-          // optionally ignore current urls by sitemap policy
-          let url;
-          const process = !options.sitemapPolicy ||
-                !stillCurrent(urlNode, options);
-
-          if (process) {
-            // if sitemap is malformed, just blow up
-            url = urlm.parse(urlNode.loc[0]);
-            if (!base.input(_.extend({}, options, {
-                protocol: url.protocol,
-                auth: url.auth,
-                hostname: url.hostname,
-                port: url.port
-              }),
-              urlNode.loc[0])
-            ) {
-              source = urlNode.loc[0];
-              err = base.generatorError();
+      if (result.sitemapindex) {
+        // actually a sitemap index
+        return smi.sitemapIndex(module.exports, options, result)
+          .then(() => callback())
+          .catch(callback);
+      } else {
+        // Process the url input, but break if base.input returns false.
+        //   In other words, _.find is looking for a non-falsy err.
+        // For now, this can only happen if no outputDir is defined,
+        //   which is a fatal bad option problem and will happen immediately.
+        _.find(
+          // if the sitemap is malformed, just blow up
+          result.urlset.url,
+          urlNode => {
+            // optionally ignore current urls by sitemap policy
+            let url;
+            const process = !options.sitemapPolicy ||
+                  !stillCurrent(urlNode, options);
+
+            if (process) {
+              // if sitemap is malformed, just blow up
+              url = urlm.parse(urlNode.loc[0]);
+              if (!base.input(_.extend({}, options, {
+                  protocol: url.protocol,
+                  auth: url.auth,
+                  hostname: url.hostname,
+                  port: url.port
+                }),
+                urlNode.loc[0])
+              ) {
+                source = urlNode.loc[0];
+                err = base.generatorError();
+              }
             }
-          }
 
-          return err;
-        }
-      );
+            return err;
+          }
+        );
+      }
     }
 
+    // NOTE: if !err, then prependMsgToErr returns `undefined`
     callback(common.prependMsgToErr(err, source, true));
   });
 }
@@ -165,21 +173,25 @@ function convert (options, buffer, next, callback) {
  * @param {String} options.source - Sitemap URL.
  * @param {Function} options.timeout - User supplied run timeout.
  * @param {Function} parseFn - A data parser function.
- * @param {Function} callback - A completion callback function.
+ * @return {Promise} resolves to data on completion.
  */
-function getUrl (options, parseFn, callback) {
-  request({
+async function getUrl (options, parseFn) {
+  const { default:got } = await import("got");
+  return got({
     url: options.source,
-    encoding: null,
-    timeout: options.timeout() // get the default timeout
-  }, (err, res, body) => {
-    var error = err || common.checkResponse(res, ["text/xml", "application/xml"]);
-
+    responseType: "buffer",
+    timeout: {
+      request: options.timeout() // get the default timeout
+    }
+  }).then(res => {
+    let error = common.checkResponse(res, ["text/xml", "application/xml"]);
     if (error) {
-      callback(common.prependMsgToErr(error, options.source, true));
-    } else {
-      convert(options, body, parseFn, callback);
+      throw new Error(error);
     }
+    const conv = util.promisify(convert);
+    return conv(options, res.body, parseFn);
+  }).catch(err => {
+    throw new Error(common.prependMsgToErr(err, options.source, true));
   });
 }
 
@@ -189,16 +201,15 @@ function getUrl (options, parseFn, callback) {
  * @param {Object} options - Sitemap options.
  * @param {String} options.source - Sitemap file path.
  * @param {Function} parseFn - A data parser function.
- * @param {Function} callback - A completion callback.
+ * @return {Promise} resolves to data on completion.
  */
-function getFile (options, parseFn, callback) {
-  fs.readFile(options.source, (err, data) => {
-    if (err) {
-      callback(common.prependMsgToErr(err, options.source, true));
-    } else {
-      convert(options, data, parseFn, callback);
-    }
-  });
+function getFile (options, parseFn) {
+  const readFile = util.promisify(fs.readFile);
+  const conv = util.promisify(convert);
+  return readFile(options.source).then(data => conv(options, data, parseFn))
+    .catch(err => {
+      throw new Error(common.prependMsgToErr(err, options.source, true));
+    });
 }
 
 module.exports = {

package/lib/html-snapshots.js

@@ -9,7 +9,6 @@
  * Copyright (c) 2013 - 2022 Alex Grant, LocalNerve, contributors
  * Licensed under the MIT license.
  */
-/* global Promise */
 
 const spawn = require("child_process").spawn;
 const path = require("path");
@@ -21,8 +20,6 @@ const _ = require("lodash");
 const common = require("./common");
 const inputFactory = require("./input-generators");
 const Notifier = require("./async").Notifier;
-const phantomDir = "./phantom";
-const snapshotScript = path.join(phantomDir, "default.js");
 
 /**
  * Determine the default phantomJS module path. This is overridden by the
@@ -57,13 +54,21 @@ function defaultPhantomJSExePath () {
   return phantomSource;
 }
 
+const phantomDir = "./phantom";
+const phantomSnapshotScript = path.join(phantomDir, "default.js");
+const puppeteerDir = "./puppeteer";
+const puppeteerSnapshotScript = path.join(puppeteerDir, "index.js");
+
 /**
  * This module's defaults
  */
 const defaults = Object.freeze({
   input: "robots",
+  browser: "puppeteer",
   phantomjs: defaultPhantomJSExePath(),
-  snapshotScript: path.join(__dirname, snapshotScript),
+  phantomSnapshotScript: path.join(__dirname, phantomSnapshotScript),
+  puppeteer: path.join(__dirname, puppeteerSnapshotScript),
+  snapshotScript: path.join(__dirname, puppeteerSnapshotScript),
   outputDirClean: false,
   pollInterval: 500,
   processLimit: 4
@@ -93,7 +98,7 @@ const defaults = Object.freeze({
  * @param {Function} notifier.known - Check if a file is or has been processed.
  * @param {Function} qcb - async.queue callback function.
  */
-function worker (input, options, notifier, qcb) {
+function phantomjsWorker (input, options, notifier, qcb) {
   let cp, customModule;
   let snapshotScript = options.snapshotScript;
   const phantomjsOptions =
@@ -143,6 +148,73 @@ function worker (input, options, notifier, qcb) {
   }
 }
 
+/**
+ * @param {Object} input - An input object from an input generator.
+ * @param {String} input.outputFile - The output file that the html is written to.
+ * @param {String} input.url - The url to get the html from.
+ * @param {String} input.selector - The selector to wait for.
+ * @param {String} input.timeout - The entire process timeout.
+ * @param {String} input.debug - The debug object.
+ * @param {Object} options - Input options.
+ * @param {Object} options.snapshotScript - Snapshot script options.
+ * @param {String} options.snapshotScript.script - Snapshot script name.
+ * @param {String} options.snapshotScript.module - Snapshot script module.
+ * @param {String} options.puppeteer - Full path to the puppeteer exec.
+ * @param {Object} notifier - The async Notifier instance for this run.
+ * @param {Function} notifier.remove - Remove an outputFile from notifier.
+ * @param {Function} notifier.setError - Set an error on the notifier.
+ * @param {Function} notifier.add - Add a file, timeout to the notifier.
+ * @param {Function} notifier.known - Check if a file is or has been processed.
+ * @param {Function} qcb - async.queue callback function.
+ */
+function puppeteerWorker (input, options, notifier, qcb) {
+  let filter = "false";
+  let snapshotScript = options.snapshotScript;
+  
+  // If the outputFile has NOT already been seen by the notifier, process.
+  if (!notifier.known(input.outputFile)) {
+
+    // map snapshotScript object script
+    if (_.isObject(options.snapshotScript)) {
+      snapshotScript = options.puppeteer;
+      if (options.snapshotScript.script === "removeScripts") {
+        filter = "./removeScripts";
+      } else {
+        filter = options.snapshotScript.module;
+      }
+    }
+
+    const cp = spawn(
+      snapshotScript,
+      [ input.outputFile,
+        input.url,
+        input.selector,
+        input.timeout,
+        filter,
+        input.debug.flag,
+        input.debug.slowMo
+      ], { cwd: process.cwd(), stdio: "inherit", detached: true }
+    );
+
+    cp.on("error", function (e) {
+      notifier.remove(input.outputFile);
+      notifier.setError(e);
+      console.error(e);
+      qcb(e);
+    });
+
+    cp.on("exit", function (code) {
+      qcb(code);
+    });
+
+    // start counting
+    notifier.add(input.outputFile, input.timeout);
+  } else {
+    // The input.outputFile is being or has been processed this run.
+    qcb(0);
+  }
+}
+
 /**
  * Prepare html snapshots options.
  *
@@ -157,6 +229,13 @@ function prepOptions (options) {
   if (Array.isArray(options.source)) {
     options.input = "array";
   }
+
+  // If phantomjs browser, make sure the default snapshotScript is correctly set.
+  if (options.browser === 'phantomjs') {
+    if (!options.snapshotScript || options.snapshotScript === options.puppeteer) {
+      options.snapshotScript = options.phantomSnapshotScript;
+    }
+  }
 }
 
 module.exports = {
@@ -227,6 +306,9 @@ module.exports = {
       // expose abort callback to input generators via options.
       options._abort = notifier.abort.bind(notifier, q);
 
+      const worker =
+        options.browser === 'puppeteer' ? puppeteerWorker : phantomjsWorker;
+
       // generate input for the snapshots.
       return inputGenerator.run(options, input => {
         q.push(_.partial(worker, input, options, notifier));

package/lib/input-generators/_base.js

@@ -26,7 +26,11 @@ const defaults = Object.freeze({
   checkInterval: 250,
   useJQuery: false,
   verbose: false,
-  phantomjsOptions: ""
+  phantomjsOptions: "",
+  debug: {
+    flag: "false",
+    slowMo: 500
+  }
 });
 
 /**
@@ -224,6 +228,7 @@ module.exports = {
    * @param {Function} options.userJQuery - Produces a jquery bool given a page url.
    * @param {Function} options.verbose - Produces a verbose bool given a page url.
    * @param {Function} options.phantomjsOptions - Produces phantomjs options given a page url.
+   * @param {Function} options.debug - Produces debug options given a page url.
    * @param {Function} options._inputEmitter - The EventEmitter to generate input.
    * @param {String} page - The url to the page resource.
    * @returns {String} The full output file path.
@@ -258,6 +263,7 @@ module.exports = {
         // map the input page to phantomJS options
         phantomjsOptions: options.phantomjsOptions(page),
         // useful for testing, debugging
+        debug: options.debug,
         __page: page
       });
 

package/lib/input-generators/array.js

@@ -7,7 +7,6 @@
  * Copyright (c) 2013 - 2022 Alex Grant, LocalNerve, contributors
  * Licensed under the MIT license.
  */
-/* global Promise */
 
 const urlm = require("url");
 const common = require("../common");