html-snapshots 0.18.3 → 0.19.0

package/examples/verbose/package.json

@@ -7,6 +7,6 @@
     "start": "node ./snapshot.js"
   },
   "dependencies": {
-    "html-snapshots": "^0.18.2"
+    "html-snapshots": "^0.18.3"
   }
 }

package/examples/verbose/snapshot.js

@@ -5,32 +5,29 @@
  * Use sitemap.xml to snapshot an entire site.
  * Use the verbose option to debug the loading of one of the pages.
  */
-var htmlSnapshots = require("html-snapshots");
+const htmlSnapshots = require("html-snapshots");
 
-var origin = "http://enigmatic-refuge-9006.herokuapp.com:80";
-
-var pageToDebug = origin + "/sample-page";
+const origin = "https://www.sitemaps.org";
+const pageToDebug = `${origin}/en_GB/`;
 
 // Make the verbose option as an object keyed with the page to debug.
-var verboseOption = {
-  __default: false
+const verboseOption = {
+  __default: false,
+  [pageToDebug]: true
 };
-verboseOption[pageToDebug] = true;
 
 htmlSnapshots.run({
   input: "sitemap",
-  source: origin + "/sitemap.xml",
+  source: `${origin}/sitemap.xml`,
   outputDir: require("path").join(__dirname, "./tmp"),
   outputDirClean: true,
-  selector: ".page-content",
-  timeout: 15000,
+  selector: "#mainContent",
+  timeout: 10000,
   processLimit: 1, // to clarify output
   verbose: verboseOption
 })
-.then(function (completed) {
+.then(completed => {
   console.log("completed snapshots:");
   console.log(require("util").inspect(completed));
 })
-.catch(function (err) {
-  console.error(err);
-});
+.catch(console.error);

package/lib/async/index.js

@@ -7,7 +7,6 @@
  * Copyright (c) 2013 - 2022, Alex Grant, LocalNerve, contributors
  * Licensed under the MIT license.
  */
-/* global Promise */
 
 const crypto = require("crypto");
 const AsyncLock = require("async-lock");

package/lib/common/index.js

@@ -48,7 +48,7 @@ function isUrl (obj) {
  * If undefined, then pass through undefined.
  * @param {String} [message] - The message to prepend.
  * @param {Boolean} [quoteInput] - True if the message should be quoted.
- * @returns {Error} The updated Error instance, undefined if no error.
+ * @returns {Error|Undefined} The updated Error instance, *undefined if no error*.
  */
 function prependMsgToErr (error, message, quoteInput) {
   let result, prepend;

package/lib/common/sitemap/index.js

@@ -0,0 +1,16 @@
+/**
+ * Reused sitemap modules.
+ * Used by robots, sitemap, and sitemap-index input generators.
+ *
+ * Copyright (c) 2013 - 2022 Alex Grant, LocalNerve, contributors
+ * Licensed under the MIT license.
+ */
+const sitemap = require("./sitemap");
+const smc = require("./sitemap-collection");
+const smi = require("./sitemap-index");
+
+module.exports = {
+  sitemapCollection: { process: smc.processSitemaps.bind(null, sitemap) },
+  sitemapIndex: { process: smi.sitemapIndex.bind(null, sitemap) },
+  sitemap
+};

package/lib/common/sitemap/sitemap-collection.js

@@ -0,0 +1,94 @@
+/**
+ * sitemap-index.js (common functions)
+ * 
+ * Copyright (c) 2013 - 2022 Alex Grant, LocalNerve, contributors
+ * Licensed under the MIT license.
+ */
+const fs = require("fs");
+const pathLib = require("path");
+const mkdirp = require("mkdirp");
+const common = require("../index");
+const base = require("../../input-generators/_base");
+
+/**
+ * Make sure path exists so a sitemap write can succeed.
+ *
+ * @param {String} outputPath - File path and name.
+ * @param {Function} callback - Completion callback.
+ */
+ function prepareWrite (outputPath, callback) {
+  const path = pathLib.parse(outputPath);
+  const dir = pathLib.join(path.root, path.dir);
+  mkdirp(dir).then(() => callback()).catch(callback);
+}
+
+/**
+ * Parse a sitemap, generate input, and optionally update the local file
+ * for sitemapPolicy purposes.
+ * This writes the sitemap files out EVEN IF sitemapPolicy is false.
+ * The thinking is that it's better/cheaper to do this b/c if sitemapPolicy is
+ * enabled later, then there is something to compare against, for more savings.
+ *
+ * @param {Object} options - Sitemap options.
+ * @param {Object} options.__sitemapIndex - Sitemap Index options.
+ * @param {String} document - xml document string.
+ * @param {Function} callback - A completion callback.
+ */
+function processSitemap (smLib, options, document, callback) {
+  smLib.parse(options, document, err => {
+    const sitemapIndexOpts = options.__sitemapIndex;
+    const outputPath = base.outputFile(sitemapIndexOpts, options.source);
+
+    if (!err && sitemapIndexOpts.sitemapOutputDir) {
+      prepareWrite(outputPath, err => {
+        if (!err) {
+          fs.writeFile(outputPath, document, callback);
+        } else {
+          callback(common.prependMsgToErr(err, outputPath, true));
+        }
+      });
+    } else {
+      callback(err);
+    }
+  });
+}
+
+/**
+ * Get and process a sitemaps collection.
+ * 
+ * @param {Object} smLib - Sitemap common module.
+ * @param {Object} options - Sitemap options.
+ * @param {Array} sitemapUrls - collection of sitemap urls to process.
+ * @param {Object} [sitemapIndexOptions] - Sitemap index options.
+ * @returns {Promise} resolves when all sitemaps are processed, reject as soon as one fails
+ */
+function processSitemaps (smLib, options, sitemapUrls, sitemapIndexOptions) {
+  // Edge case message for clarity
+  if (sitemapUrls.length === 0) {
+    console.log("[*] No sitemaps qualified for processing");
+    return Promise.resolve([]);
+  }
+
+  const smiOptions = sitemapIndexOptions || {
+    ...options,
+    ...{ outputPath: undefined }
+  };
+
+  // Get all the sitemaps, parse them, and generate input for each
+  return Promise.all(sitemapUrls.map(sitemapUrl => {
+    const sitemapOptions = Object.assign({}, options, {
+      source: sitemapUrl,
+      input: "sitemap",
+      sitemapOutputDir: false,
+      __sitemapIndex: smiOptions
+    });
+
+    console.log(`[+] Loading the sitemap: '${sitemapUrl}'`);
+
+    return smLib.getUrl(sitemapOptions, processSitemap.bind(null, smLib));
+  }));
+}
+
+module.exports = {
+  processSitemaps
+};

package/lib/common/sitemap/sitemap-index.js

@@ -0,0 +1,68 @@
+/**
+ * sitemap-index.js
+ * 
+ * Generates the urls for a sitemap-index.
+ * 
+ * Copyright (c) 2013 - 2022 Alex Grant, LocalNerve, contributors
+ * Licensed under the MIT license.
+ */
+const _ = require("lodash");
+const smc = require("./sitemap-collection");
+
+/**
+ * Create a collection of sitemap urls to process by policy, form sitemapIndexOptions.
+ *
+ * @param {Object} smLib - Sitemap module
+ * @param {Object} options - sitemap index options
+ * @param {Boolean} options.sitemapPolicy - True if sitemap policy should be used.
+ * @param {Object} parseResult - xml2js sax parse result object
+ * @returns {Object} The sitemapUrls collection and sitemapIndexOptions
+ */
+function sitemapIndexUrls (smLib, options, parseResult) {
+  const sitemapUrls = [];
+  const sitemapIndexOptions = {
+    ...options,
+    ...{ outputPath: undefined }
+  };
+
+  // Check if we should process each sitemap in the index.
+  _.forEach(
+    // if the sitemap index is malformed, just blow up
+    parseResult.sitemapindex.sitemap,
+    sitemapNode => {
+      // optionally ignore current sitemaps by sitemap policy
+      const shouldProcess = !options.sitemapPolicy ||
+        !smLib.stillCurrent(sitemapNode, sitemapIndexOptions);
+
+      if (shouldProcess) {
+        // if sitemap index is malformed, just blow up
+        sitemapUrls.push(sitemapNode.loc[0]);
+      }
+    }
+  );
+
+  return {
+    sitemapUrls,
+    sitemapIndexOptions
+  };
+}
+
+/**
+ * Collect sitemap urls from a sitemap index file for processing.
+ *
+ * @param {Object} smLib - sitemap module
+ * @param {Object} options - sitemapIndex options
+ * @param {Object} parseResult - xml2js sax parse result object
+ * @returns {Promise} resolves when all sitemaps are processed, reject as soon as one fails
+ */
+function sitemapIndex (smLib, options, parseResult) {
+  const {
+    sitemapUrls,
+    sitemapIndexOptions
+  } = sitemapIndexUrls(smLib, options, parseResult);
+  return smc.processSitemaps(smLib, options, sitemapUrls, sitemapIndexOptions);
+}
+
+module.exports = {
+  sitemapIndex
+};

package/lib/common/{sitemap.js → sitemap/sitemap.js}

@@ -9,14 +9,15 @@
  */
 
 const fs = require("fs");
+const util = require("util");
 const path = require("path");
 const urlm = require("url");
 const zlib = require("zlib");
 const _ = require("lodash");
 const xml2js = require("xml2js");
-const request = require("request");
-const common = require("./index");
-const base = require("../input-generators/_base");
+const smi = require("./sitemap-index");
+const common = require("../index");
+const base = require("../../input-generators/_base");
 
 // the out of date fallback
 const unixStart = "1970-01-01T00:00:00.000Z";
@@ -44,6 +45,7 @@ const changeFreq = {
  */
 function stillCurrent (urlNode, options) {
   let lesser, greater, oPath;
+  const statOpts = {throwIfNoEntry: false};
   const now = Date.now();
   const lMod = _.first(urlNode.lastmod);
   const cFreq = _.first(urlNode.changefreq) ? _.first(urlNode.changefreq).toLowerCase() : null;
@@ -53,7 +55,7 @@ function stillCurrent (urlNode, options) {
     // if sitemap is malformed, just blow up
     oPath = base.outputFile(options, urlNode.loc[0]);
 
-    lesser = now - ( (fs.existsSync(oPath) && fs.statSync(oPath).mtime.getTime()) || unixStart );
+    lesser = now - (fs.statSync(oPath, statOpts)?.mtime?.getTime() ?? unixStart);
     greater = now - Date.parse(lMod);
   }
   // only changefreq specified
@@ -61,7 +63,7 @@ function stillCurrent (urlNode, options) {
     // if sitemap is malformed, just blow up
     oPath = base.outputFile(options, urlNode.loc[0]);
 
-    lesser = now - ( (fs.existsSync(oPath) && fs.statSync(oPath).mtime.getTime()) || unixStart );
+    lesser = now - (fs.statSync(oPath, statOpts)?.mtime?.getTime() ?? unixStart);
     greater = changeFreq[cFreq] || changeFreq.always;
   }
   // both or neither were specified
@@ -91,43 +93,49 @@ function parse (options, document, callback) {
     normalize: true
   }, (err, result) => {
     let source = options.source;
-
     if (!err) {
-
-      // Process the url input, but break if base.input returns false.
-      //   In other words, _.find is looking for a non-falsy err.
-      // For now, this can only happen if no outputDir is defined,
-      //   which is a fatal bad option problem and will happen immediately.
-      _.find(
-        // if the sitemap is malformed, just blow up
-        result.urlset.url,
-        urlNode => {
-          // optionally ignore current urls by sitemap policy
-          let url;
-          const process = !options.sitemapPolicy ||
-                !stillCurrent(urlNode, options);
-
-          if (process) {
-            // if sitemap is malformed, just blow up
-            url = urlm.parse(urlNode.loc[0]);
-            if (!base.input(_.extend({}, options, {
-                protocol: url.protocol,
-                auth: url.auth,
-                hostname: url.hostname,
-                port: url.port
-              }),
-              urlNode.loc[0])
-            ) {
-              source = urlNode.loc[0];
-              err = base.generatorError();
+      if (result.sitemapindex) {
+        // actually a sitemap index
+        return smi.sitemapIndex(module.exports, options, result)
+          .then(() => callback())
+          .catch(callback);
+      } else {
+        // Process the url input, but break if base.input returns false.
+        //   In other words, _.find is looking for a non-falsy err.
+        // For now, this can only happen if no outputDir is defined,
+        //   which is a fatal bad option problem and will happen immediately.
+        _.find(
+          // if the sitemap is malformed, just blow up
+          result.urlset.url,
+          urlNode => {
+            // optionally ignore current urls by sitemap policy
+            let url;
+            const process = !options.sitemapPolicy ||
+                  !stillCurrent(urlNode, options);
+
+            if (process) {
+              // if sitemap is malformed, just blow up
+              url = urlm.parse(urlNode.loc[0]);
+              if (!base.input(_.extend({}, options, {
+                  protocol: url.protocol,
+                  auth: url.auth,
+                  hostname: url.hostname,
+                  port: url.port
+                }),
+                urlNode.loc[0])
+              ) {
+                source = urlNode.loc[0];
+                err = base.generatorError();
+              }
             }
-          }
 
-          return err;
-        }
-      );
+            return err;
+          }
+        );
+      }
     }
 
+    // NOTE: if !err, then prependMsgToErr returns `undefined`
     callback(common.prependMsgToErr(err, source, true));
   });
 }
@@ -165,21 +173,25 @@ function convert (options, buffer, next, callback) {
  * @param {String} options.source - Sitemap URL.
  * @param {Function} options.timeout - User supplied run timeout.
  * @param {Function} parseFn - A data parser function.
- * @param {Function} callback - A completion callback function.
+ * @return {Promise} resolves to data on completion.
  */
-function getUrl (options, parseFn, callback) {
-  request({
+async function getUrl (options, parseFn) {
+  const { default:got } = await import("got");
+  return got({
     url: options.source,
-    encoding: null,
-    timeout: options.timeout() // get the default timeout
-  }, (err, res, body) => {
-    var error = err || common.checkResponse(res, ["text/xml", "application/xml"]);
-
+    responseType: "buffer",
+    timeout: {
+      request: options.timeout() // get the default timeout
+    }
+  }).then(res => {
+    let error = common.checkResponse(res, ["text/xml", "application/xml"]);
     if (error) {
-      callback(common.prependMsgToErr(error, options.source, true));
-    } else {
-      convert(options, body, parseFn, callback);
+      throw new Error(error);
     }
+    const conv = util.promisify(convert);
+    return conv(options, res.body, parseFn);
+  }).catch(err => {
+    throw new Error(common.prependMsgToErr(err, options.source, true));
   });
 }
 
@@ -189,16 +201,15 @@ function getUrl (options, parseFn, callback) {
  * @param {Object} options - Sitemap options.
  * @param {String} options.source - Sitemap file path.
  * @param {Function} parseFn - A data parser function.
- * @param {Function} callback - A completion callback.
+ * @return {Promise} resolves to data on completion.
  */
-function getFile (options, parseFn, callback) {
-  fs.readFile(options.source, (err, data) => {
-    if (err) {
-      callback(common.prependMsgToErr(err, options.source, true));
-    } else {
-      convert(options, data, parseFn, callback);
-    }
-  });
+function getFile (options, parseFn) {
+  const readFile = util.promisify(fs.readFile);
+  const conv = util.promisify(convert);
+  return readFile(options.source).then(data => conv(options, data, parseFn))
+    .catch(err => {
+      throw new Error(common.prependMsgToErr(err, options.source, true));
+    });
 }
 
 module.exports = {

package/lib/html-snapshots.js

@@ -9,7 +9,6 @@
  * Copyright (c) 2013 - 2022 Alex Grant, LocalNerve, contributors
  * Licensed under the MIT license.
  */
-/* global Promise */
 
 const spawn = require("child_process").spawn;
 const path = require("path");

package/lib/input-generators/array.js

@@ -7,7 +7,6 @@
  * Copyright (c) 2013 - 2022 Alex Grant, LocalNerve, contributors
  * Licensed under the MIT license.
  */
-/* global Promise */
 
 const urlm = require("url");
 const common = require("../common");

package/lib/input-generators/robots.js

@@ -1,19 +1,16 @@
-/*
+/**
  * robots.js
  *
  * An input generator for html-snapshots that uses a simple robots.txt file.
- * Creates the snapshot arguments driven from robots.txt "Allow: ".
- * Does not support wildcards.
- * If you need wildcards, use the sitemap input generator.
+ * Creates the snapshot arguments driven from robots.txt Sitemap or Allow directives.
  *
  * Copyright (c) 2013 - 2022 Alex Grant, LocalNerve, contributors
  * Licensed under the MIT license.
  */
 
-const fs = require("fs");
-const request = require("request");
+const fs = require("fs").promises;
 const common = require("../common");
-const nodeCall = require("../common/node");
+const { sitemapCollection: smc } = require("../common/sitemap");
 const base = require("./_base");
 
 // The default options
@@ -39,19 +36,81 @@ const robots = {
 };
 
 /**
- * Generate input for one line of a simple robots.txt file
- * Does not support wildcards.
+ * Get all unique `Allow:` url paths from robots.txt.
+ * Reads the allow values up to whitespace (end-of-line), or wildcard.
+ * 
+ * @param {String} body - The robots.txt file
+ * @returns {Array} collection of allow url paths
  */
-function oneline (line, options) {
-  const key = "Allow: ",
-      index = line.indexOf(key);
+function getAllowUrls (body) {
+  const allowUrls = new Set();
+  const re = /\ballow\b\s*:\s*(?<allowUrl>[^*\s]+)/ig;
+  const matches = body.matchAll(re);
+  let allowUrl;
 
-  if (index !== -1) {
-    const page = line.substr(index + key.length).replace(/^\s+|\s+$/g, "");
-    return page.indexOf("*") === -1 && base.input(options, page);
+  for (const match of matches) {
+    allowUrl = match?.groups?.allowUrl;
+    if (allowUrl) {
+      allowUrls.add(allowUrl);
+    }
   }
 
-  return true;
+  return allowUrls.size > 0 ? Array.from(allowUrls) : null;
+}
+
+/**
+ * Get all the unique sitemap urls from robots.txt.
+ * 
+ * @param {String} body - The robots.txt file
+ * @returns {Array} collection of unqiue sitemap urls
+ */
+function getSitemapUrls (body) {
+  const sitemapUrls = new Set();
+  const re = /\bsitemap\b\s*:\s*(?<sitemapUrl>[^\s]+)/ig;
+  const matches = body.matchAll(re);
+  let sitemapUrl;
+  
+  for (const match of matches) {
+    sitemapUrl = match?.groups?.sitemapUrl;
+    if (sitemapUrl) {
+      sitemapUrls.add(sitemapUrl);
+    }
+  }
+  
+  return sitemapUrls.size > 0 ? Array.from(sitemapUrls) : null;
+}
+
+/**
+ * Parse urls from robots.txt and generate input.
+ * Look for sitemaps first, then look for allow urls.
+ * 
+ * @param {Object} options - robots.txt options
+ * @param {String} body - robots.txt content
+ * @returns {Promise} resolves on success
+ */
+function processRobotsTxt (options, body) {
+  let allowUrls = false;
+  const urls = getSitemapUrls(body) || (allowUrls = getAllowUrls(body));
+
+  if (allowUrls) {
+    let result;
+    for (const url of urls) {
+      result = base.input(options, url);
+      // if base.input returns false, then no outputDir was defined,
+      //   which is a fatal bad option problem and will happen immediately.
+      if (!result) {
+        throw new Error(
+          common.prependMsgToErr(base.generatorError(), url, true)
+        );
+      }
+    }
+    return Promise.resolve();
+  } else {
+    if (urls && urls.length > 0) {
+      return smc.process(options, urls);
+    }
+    throw new Error(`No urls found in ${options.source}`);
+  }
 }
 
 /**
@@ -60,54 +119,37 @@ function oneline (line, options) {
  * @param {Object} options - Robots.txt options
  * @param {String} options.source - The URL to a robots.txt
  * @param {Function} options.timeout - Returns the robots.txt request timeout.
- * @param {Function} callback - A completion callback.
+ * @returns {Promise} resolves to undefined.
  */
-function getRobotsUrl (options, callback) {
-  request({
+async function getRobotsUrl (options) {
+  const { default:got } = await import("got");
+  return got({
     url: options.source,
-    timeout: options.timeout()
-  }, (err, res, body) => {
-    let error = err || common.checkResponse(res, "text/plain");
-
+    timeout: {
+      request: options.timeout()
+    }
+  }).then(res => {
+    const error = common.checkResponse(res, "text/plain");
     if (error) {
-      callback(common.prependMsgToErr(error, options.source, true));
-    } else {
-      body.toString().split('\n').every(line => {
-        // Process the line input, but break if base.input returns false.
-        // For now, this can only happen if no outputDir is defined,
-        //   which is a fatal bad option problem and will happen immediately.
-        if (!oneline(line, options)) {
-          error = common.prependMsgToErr(base.generatorError(), line, true);
-          return false;
-        }
-        return true;
-      });
-
-      callback(error);
+      throw new Error(error);
     }
+    return processRobotsTxt(options, res.body.toString());
+  }).catch(err => {
+    throw new Error(common.prependMsgToErr(err, options.source, true));
   });
 }
 
 /**
  * Reads the robots.txt file and parses it.
+ * 
+ * @param {Object} options - Robots.txt options
  */
-function getRobotsFile (options, callback) {
-  fs.readFile(options.source, (err, data) => {
-    if (!err) {
-      data.toString().split('\n').every(line => {
-        // Process the line input, but break if base.input returns false.
-        // For now, this can only happen if no outputDir is defined,
-        //   which is a fatal bad option problem and will happen immediately.
-        if (!oneline(line, options)) {
-          err = common.prependMsgToErr(base.generatorError(), line, true);
-          return false;
-        }
-        return true;
-      });
-    }
-
-    callback(err);
-  });
+function getRobotsFile (options) {
+  return fs.readFile(options.source)
+    .then(data => processRobotsTxt(options, data.toString()))
+    .catch(err => {
+      throw new Error(common.prependMsgToErr(err, options.source, true));
+    });
 }
 
 /**
@@ -122,10 +164,8 @@ function getRobotsFile (options, callback) {
  * @returns {Promise} Resolves to undefined on completion.
  */
 function generateInput (options) {
-  return nodeCall(
-    common.isUrl(options.source) ? getRobotsUrl : getRobotsFile,
-    options
-  )
+  const retrieve = common.isUrl(options.source) ? getRobotsUrl : getRobotsFile;
+  return retrieve(options)
     .catch(err => {
       options._abort(err);
     })