html-snapshots 0.18.2 → 0.19.0

package/lib/input-generators/sitemap-index.js

@@ -7,16 +7,8 @@
  * Copyright (c) 2013 - 2022 Alex Grant, LocalNerve, contributors
  * Licensed under the MIT license.
  */
-/* global Promise */
-
-const fs = require("fs");
-const pathLib = require("path");
-const _ = require("lodash");
-const mkdirp = require("mkdirp");
-const xml2js = require("xml2js");
 const common = require("../common");
-const nodeCall = require("../common/node");
-const smLib = require("../common/sitemap");
+const { sitemap: sm } = require("../common/sitemap");
 const base = require("./_base");
 
 // The default options
@@ -44,116 +36,6 @@ const sitemapIndex = {
   }
 };
 
-/**
- * Make sure path exists so a sitemap write can succeed.
- *
- * @param {String} outputPath - File path and name.
- * @param {Function} callback - Completion callback.
- */
-function prepareWrite (outputPath, callback) {
-  const path = pathLib.parse(outputPath);
-  const dir = pathLib.join(path.root, path.dir);
-  mkdirp(dir).then(() => callback()).catch(callback);
-}
-
-/**
- * Parse a sitemap, generate input, and optionally update the local file
- * for sitemapPolicy purposes.
- * This writes the sitemap files out EVEN IF sitemapPolicy is false.
- * The thinking is that it's better/cheaper to do this b/c if sitemapPolicy is
- * enabled later, then there is something to compare against, for more savings.
- *
- * @param {Object} options - Sitemap options.
- * @param {Object} options.__sitemapIndex - Sitemap Index options.
- * @param {String} document - xml document string.
- * @param {Function} callback - A completion callback.
- */
-function processSitemap (options, document, callback) {
-  smLib.parse(options, document, err => {
-    const sitemapIndexOpts = options.__sitemapIndex;
-    const outputPath = base.outputFile(sitemapIndexOpts, options.source);
-
-    if (!err && sitemapIndexOpts.sitemapOutputDir) {
-      prepareWrite(outputPath, err => {
-        if (!err) {
-          fs.writeFile(outputPath, document, callback);
-        } else {
-          callback(common.prependMsgToErr(err, outputPath, true));
-        }
-      });
-    } else {
-      callback(err);
-    }
-  });
-}
-
-/**
- * Parse a sitemap index document
- * For each qualifying sitemap element, download, parse the sitemap.
- * Stops processing if an error occurs.
- *
- * @param {Object} options - Sitemap options.
- * @param {Boolean} options.sitemapPolicy - True if sitemap policy should be used.
- * @param {String} document - xml document string.
- * @param {Function} callback - A completion callback.
- */
-function parse (options, document, callback) {
-  xml2js.parseString(document, {
-    trim: true,
-    normalizeTags: true,
-    normalize: true
-  }, (err, result) => {
-    const sitemapUrls = [];
-    const sitemapIndexOptions = Object.assign({}, options, {
-      outputPath: undefined
-    });
-
-    if (!err) {
-      // Check if we should process each sitemap in the index.
-      _.forEach(
-        // if the sitemap index is malformed, just blow up
-        result.sitemapindex.sitemap,
-        sitemapNode => {
-          // optionally ignore current sitemaps by sitemap policy
-          const shouldProcess = !options.sitemapPolicy ||
-            !smLib.stillCurrent(sitemapNode, sitemapIndexOptions);
-
-          if (shouldProcess) {
-            // if sitemap index is malformed, just blow up
-            sitemapUrls.push(sitemapNode.loc[0]);
-          }
-        }
-      );
-
-      // Edge case message for clarity
-      if (sitemapUrls.length === 0) {
-        console.log("[*] No sitemaps qualified for processing");
-      }
-
-      // Get all the sitemaps, parse them, and generate input for each
-      Promise.all(sitemapUrls.map(sitemapUrl => {
-        const sitemapOptions = Object.assign({}, options, {
-          source: sitemapUrl,
-          input: "sitemap",
-          sitemapOutputDir: false,
-          __sitemapIndex: sitemapIndexOptions
-        });
-
-        console.log(`[+] Loading the sitemap: '${sitemapUrl}'`);
-
-        return nodeCall(smLib.getUrl, sitemapOptions, processSitemap);
-      }))
-        .then(() => {
-          // Ignore the array of results to this function
-          callback();
-        })
-        .catch(callback);
-    } else {
-      callback(err);
-    }
-  });
-}
-
 /**
  * generateInput
  *
@@ -165,11 +47,8 @@ function parse (options, document, callback) {
  * @param {String} options.source - Sitemap URL or file path.
  */
 function generateInput (options) {
-  return nodeCall(
-    common.isUrl(options.source) ? smLib.getUrl : smLib.getFile,
-    options,
-    parse
-  )
+  const retrieve = common.isUrl(options.source) ? sm.getUrl : sm.getFile;
+  return retrieve(options, sm.parse)
     .catch(err => {
       options._abort(err);
     })

package/lib/input-generators/sitemap.js

@@ -9,8 +9,7 @@
  */
 
 const common = require("../common");
-const nodeCall = require("../common/node");
-const smLib = require("../common/sitemap");
+const { sitemap: sm } = require("../common/sitemap");
 const base = require("./_base");
 
 // The default options
@@ -48,11 +47,8 @@ const sitemap = {
  * @returns {Promise} Resolves to undefined on completion.
  */
 function generateInput (options) {
-  return nodeCall(
-    common.isUrl(options.source) ? smLib.getUrl : smLib.getFile,
-    options,
-    smLib.parse
-  )
+  const retrieve = common.isUrl(options.source) ? sm.getUrl : sm.getFile;
+  return retrieve(options, sm.parse)
     .catch(err => {
       options._abort(err);
     })

package/lib/input-generators/textfile.js

@@ -8,10 +8,9 @@
  * Licensed under the MIT license.
  */
 
-const fs = require("fs");
+const fs = require("fs").promises;
 const base = require("./_base");
 const common = require("../common");
-const nodeCall = require("../common/node");
 
 // The default options
 const defaults = Object.freeze({
@@ -44,10 +43,7 @@ const textfile = {
  * @returns {Promise} resolves when complete.
  */
 function generateInput (options) {
-  return nodeCall(
-    fs.readFile,
-    options.source
-  )
+  return fs.readFile(options.source)
     .catch(err => {
       options._abort(err);
     })

package/package.json

@@ -1,10 +1,10 @@
 {
   "name": "html-snapshots",
-  "version": "0.18.2",
+  "version": "0.19.0",
   "author": {
     "name": "Alex Grant",
     "email": "alex@localnerve.com",
-    "url": "http://localnerve.com"
+    "url": "https://www.localnerve.com"
   },
   "homepage": "https://github.com/localnerve/html-snapshots",
   "description": "A selector-based html snapshot tool using PhantomJS that sources sitemap.xml, robots.txt, or arbitrary input",
@@ -24,6 +24,7 @@
     "test:common": "mocha test/mocha/common/*.js --exit --reporter spec",
     "test:input-generators": "mocha test/mocha/input-generators/*.js --exit --reporter spec",
     "test:html-snapshots": "mocha test/mocha/html-snapshots/*.js --exit --reporter spec",
+    "test:html-snapshots:debug": "mocha --inspect-brk test/mocha/html-snapshots/*.js --exit --reporter spec",
     "test:cover": "c8 -- npm test",
     "validate": "npm ls"
   },
@@ -52,16 +53,15 @@
     "lodash": "4.17.21",
     "mkdirp": "1.0.4",
     "phantomjs-prebuilt": "2.1.16",
-    "request": "2.88.2",
+    "got": "12.5.0",
     "rimraf": "3.0.2",
     "xml2js": "0.4.23"
   },
   "devDependencies": {
-    "coveralls": "3.1.1",
-    "eslint": "8.14.0",
+    "eslint": "8.23.1",
     "express": "4.18.1",
     "mocha": "10.0.0",
-    "c8": "7.11.2",
+    "c8": "7.12.0",
     "precommit-hook": "3.0.0",
     "server-destroy": "1.0.1",
     "sitemap-xml": "0.1.0",

package/test/helpers/options.js

@@ -9,7 +9,7 @@
  * If a global phantomjs is defined, decorates html-snapshots options to specify that global
  * In some test environments (travis), local phantomjs will not install if a global is found.
  */
-var assert = require("assert");
+const assert = require("assert");
 
 // @0.12.0, on Travis-ci phantomjs2 WILL install local WITH the older global.
 // So reverting back to decorateLocal only.
@@ -19,7 +19,9 @@ var assert = require("assert");
 function fillAbort (options) {
   if (!options._abort) {
     options._abort = function (err) {
-      assert.fail("test", "failed", err.toString(), "");
+      const msg = `[options._abort] called: ${err.toString()}`;
+      console.log(`@@@ ${msg}`);
+      assert.fail(msg);
     }
   }
 }

package/test/mocha/html-snapshots/process-limit.js

@@ -4,41 +4,41 @@
  * Copyright (c) 2013 - 2022, Alex Grant, LocalNerve, contributors
  */
 /* global it */
-var assert = require("assert");
-var rimraf = require("rimraf").sync;
-var utils = require("./utils");
-var optHelp = require("../../helpers/options");
-var ss = require("../../../lib/html-snapshots");
-var robotsTests = require("./robots");
-
-// missing destructuring, will write postcard...
-var timeout = utils.timeout;
-var outputDir = utils.outputDir;
-var cleanup = utils.cleanup;
-var killSpawnedProcesses = utils.killSpawnedProcesses;
-var countSpawnedProcesses = utils.countSpawnedProcesses;
-var unexpectedError = utils.unexpectedError;
-var multiError = utils.multiError;
-var checkActualFiles = utils.checkActualFiles;
-
-var urls = robotsTests.urlCount;
-var inputFile = robotsTests.inputFile;
+const assert = require("assert");
+const rimraf = require("rimraf").sync;
+const utils = require("./utils");
+const optHelp = require("../../helpers/options");
+const ss = require("../../../lib/html-snapshots");
+const robotsTests = require("./robots");
+
+const {
+  timeout,
+  outputDir,
+  cleanup,
+  killSpawnedProcesses,
+  countSpawnedProcesses,
+  unexpectedError,
+  multiError,
+  checkActualFiles  
+} = utils;
+
+const urls = robotsTests.urlCount;
 
 function processLimitTests (options) {
-  var port = options.port;
-  var pollInterval = 50;
-  var phantomCount = 0;
-  var timer;
+  const { port, localRobotsFile: inputFile } = options;
+  const pollInterval = 50;
+  let phantomCount = 0;
+  let timer;
 
   function completeTest (done, processLimit, e, files) {
-    var countError = phantomCount ?
-      new Error(phantomCount + " exceeded processLimit " + processLimit) :
+    const countError = phantomCount ?
+      new Error(`${phantomCount} exceeded processLimit ${processLimit}`) :
       undefined;
 
     clearInterval(timer);
 
     if (files) {
-      checkActualFiles(files).then(function () {
+      checkActualFiles(files).then(() => {
         cleanup(done, multiError(e, countError));
       });
     } else {
@@ -48,8 +48,8 @@ function processLimitTests (options) {
 
   return function () {
     it("should limit as expected", function (done) {
-      var processLimit = urls - 1;
-      var complete = completeTest.bind(null, done, processLimit);
+      const processLimit = urls - 1;
+      const complete = completeTest.bind(null, done, processLimit);
 
       if (process.platform === "win32") {
         assert.ok(true, "Skipping posix compliant tests for processLimit");
@@ -58,15 +58,15 @@ function processLimitTests (options) {
         rimraf(outputDir);
 
         killSpawnedProcesses(function (err) {
-          var options = {
+          const options = {
             source: inputFile,
             hostname: "localhost",
-            port: port,
             selector: "#dynamic-content",
-            outputDir: outputDir,
             outputDirClean: true,
-            timeout: timeout,
-            processLimit: processLimit
+            outputDir,
+            timeout,
+            processLimit,
+            port
           };
 
           if (err) {
@@ -74,14 +74,14 @@ function processLimitTests (options) {
           }
 
           ss.run(optHelp.decorate(options))
-            .then(function () {
+            .then(() => {
               complete();
             })
-            .catch(function (e) {
+            .catch(e => {
               complete(e || unexpectedError, e.notCompleted);
             });
 
-          timer = setInterval(function () {
+          timer = setInterval(() => {
             countSpawnedProcesses(function (count) {
               // console.log("@@@ DEBUG @@@ phantom count: "+count);
               if (count > processLimit) {
@@ -95,8 +95,8 @@ function processLimitTests (options) {
     });
 
     it("should limit to just one process", function (done) {
-      var processLimit = 1;
-      var complete = completeTest.bind(null, done, processLimit);
+      const processLimit = 1;
+      const complete = completeTest.bind(null, done, processLimit);
 
       if (process.platform === "win32") {
         assert.ok(true, "Skipping posix compliant tests for processLimit");
@@ -105,15 +105,15 @@ function processLimitTests (options) {
         rimraf(outputDir);
 
         killSpawnedProcesses(function (err) {
-          var options = {
+          const options = {
             source: inputFile,
             hostname: "localhost",
-            port: port,
             selector: "#dynamic-content",
-            outputDir: outputDir,
             outputDirClean: true,
-            timeout: timeout,
-            processLimit: processLimit
+            outputDir,
+            timeout,
+            processLimit,
+            port
           };
 
           if (err) {
@@ -121,10 +121,10 @@ function processLimitTests (options) {
           }
 
           ss.run(optHelp.decorate(options))
-            .then(function () {
+            .then(() => {
               complete();
             })
-            .catch(function (e) {
+            .catch(e => {
               complete(e || unexpectedError, e.notCompleted);
             });
 
@@ -144,7 +144,5 @@ function processLimitTests (options) {
 }
 
 module.exports = {
-  testSuite: processLimitTests,
-  inputFile: inputFile,
-  urlCount: urls
+  testSuite: processLimitTests
 };