html-snapshots 0.18.3 → 1.0.0

package/lib/input-generators/robots.js

@@ -1,19 +1,16 @@
-/*
+/**
  * robots.js
  *
  * An input generator for html-snapshots that uses a simple robots.txt file.
- * Creates the snapshot arguments driven from robots.txt "Allow: ".
- * Does not support wildcards.
- * If you need wildcards, use the sitemap input generator.
+ * Creates the snapshot arguments driven from robots.txt Sitemap or Allow directives.
  *
  * Copyright (c) 2013 - 2022 Alex Grant, LocalNerve, contributors
  * Licensed under the MIT license.
  */
 
-const fs = require("fs");
-const request = require("request");
+const fs = require("fs").promises;
 const common = require("../common");
-const nodeCall = require("../common/node");
+const { sitemapCollection: smc } = require("../common/sitemap");
 const base = require("./_base");
 
 // The default options
@@ -39,19 +36,81 @@ const robots = {
 };
 
 /**
- * Generate input for one line of a simple robots.txt file
- * Does not support wildcards.
+ * Get all unique `Allow:` url paths from robots.txt.
+ * Reads the allow values up to whitespace (end-of-line), or wildcard.
+ * 
+ * @param {String} body - The robots.txt file
+ * @returns {Array} collection of allow url paths
  */
-function oneline (line, options) {
-  const key = "Allow: ",
-      index = line.indexOf(key);
+function getAllowUrls (body) {
+  const allowUrls = new Set();
+  const re = /\ballow\b\s*:\s*(?<allowUrl>[^*\s]+)/ig;
+  const matches = body.matchAll(re);
+  let allowUrl;
 
-  if (index !== -1) {
-    const page = line.substr(index + key.length).replace(/^\s+|\s+$/g, "");
-    return page.indexOf("*") === -1 && base.input(options, page);
+  for (const match of matches) {
+    allowUrl = match?.groups?.allowUrl;
+    if (allowUrl) {
+      allowUrls.add(allowUrl);
+    }
   }
 
-  return true;
+  return allowUrls.size > 0 ? Array.from(allowUrls) : null;
+}
+
+/**
+ * Get all the unique sitemap urls from robots.txt.
+ * 
+ * @param {String} body - The robots.txt file
+ * @returns {Array} collection of unqiue sitemap urls
+ */
+function getSitemapUrls (body) {
+  const sitemapUrls = new Set();
+  const re = /\bsitemap\b\s*:\s*(?<sitemapUrl>[^\s]+)/ig;
+  const matches = body.matchAll(re);
+  let sitemapUrl;
+  
+  for (const match of matches) {
+    sitemapUrl = match?.groups?.sitemapUrl;
+    if (sitemapUrl) {
+      sitemapUrls.add(sitemapUrl);
+    }
+  }
+  
+  return sitemapUrls.size > 0 ? Array.from(sitemapUrls) : null;
+}
+
+/**
+ * Parse urls from robots.txt and generate input.
+ * Look for sitemaps first, then look for allow urls.
+ * 
+ * @param {Object} options - robots.txt options
+ * @param {String} body - robots.txt content
+ * @returns {Promise} resolves on success
+ */
+function processRobotsTxt (options, body) {
+  let allowUrls = false;
+  const urls = getSitemapUrls(body) || (allowUrls = getAllowUrls(body));
+
+  if (allowUrls) {
+    let result;
+    for (const url of urls) {
+      result = base.input(options, url);
+      // if base.input returns false, then no outputDir was defined,
+      //   which is a fatal bad option problem and will happen immediately.
+      if (!result) {
+        throw new Error(
+          common.prependMsgToErr(base.generatorError(), url, true)
+        );
+      }
+    }
+    return Promise.resolve();
+  } else {
+    if (urls && urls.length > 0) {
+      return smc.process(options, urls);
+    }
+    throw new Error(`No urls found in ${options.source}`);
+  }
 }
 
 /**
@@ -60,54 +119,37 @@ function oneline (line, options) {
  * @param {Object} options - Robots.txt options
  * @param {String} options.source - The URL to a robots.txt
  * @param {Function} options.timeout - Returns the robots.txt request timeout.
- * @param {Function} callback - A completion callback.
+ * @returns {Promise} resolves to undefined.
  */
-function getRobotsUrl (options, callback) {
-  request({
+async function getRobotsUrl (options) {
+  const { default:got } = await import("got");
+  return got({
     url: options.source,
-    timeout: options.timeout()
-  }, (err, res, body) => {
-    let error = err || common.checkResponse(res, "text/plain");
-
+    timeout: {
+      request: options.timeout()
+    }
+  }).then(res => {
+    const error = common.checkResponse(res, "text/plain");
     if (error) {
-      callback(common.prependMsgToErr(error, options.source, true));
-    } else {
-      body.toString().split('\n').every(line => {
-        // Process the line input, but break if base.input returns false.
-        // For now, this can only happen if no outputDir is defined,
-        //   which is a fatal bad option problem and will happen immediately.
-        if (!oneline(line, options)) {
-          error = common.prependMsgToErr(base.generatorError(), line, true);
-          return false;
-        }
-        return true;
-      });
-
-      callback(error);
+      throw new Error(error);
     }
+    return processRobotsTxt(options, res.body.toString());
+  }).catch(err => {
+    throw new Error(common.prependMsgToErr(err, options.source, true));
   });
 }
 
 /**
  * Reads the robots.txt file and parses it.
+ * 
+ * @param {Object} options - Robots.txt options
  */
-function getRobotsFile (options, callback) {
-  fs.readFile(options.source, (err, data) => {
-    if (!err) {
-      data.toString().split('\n').every(line => {
-        // Process the line input, but break if base.input returns false.
-        // For now, this can only happen if no outputDir is defined,
-        //   which is a fatal bad option problem and will happen immediately.
-        if (!oneline(line, options)) {
-          err = common.prependMsgToErr(base.generatorError(), line, true);
-          return false;
-        }
-        return true;
-      });
-    }
-
-    callback(err);
-  });
+function getRobotsFile (options) {
+  return fs.readFile(options.source)
+    .then(data => processRobotsTxt(options, data.toString()))
+    .catch(err => {
+      throw new Error(common.prependMsgToErr(err, options.source, true));
+    });
 }
 
 /**
@@ -122,10 +164,8 @@ function getRobotsFile (options, callback) {
  * @returns {Promise} Resolves to undefined on completion.
  */
 function generateInput (options) {
-  return nodeCall(
-    common.isUrl(options.source) ? getRobotsUrl : getRobotsFile,
-    options
-  )
+  const retrieve = common.isUrl(options.source) ? getRobotsUrl : getRobotsFile;
+  return retrieve(options)
     .catch(err => {
       options._abort(err);
     })

package/lib/input-generators/sitemap-index.js

@@ -7,16 +7,8 @@
  * Copyright (c) 2013 - 2022 Alex Grant, LocalNerve, contributors
  * Licensed under the MIT license.
  */
-/* global Promise */
-
-const fs = require("fs");
-const pathLib = require("path");
-const _ = require("lodash");
-const mkdirp = require("mkdirp");
-const xml2js = require("xml2js");
 const common = require("../common");
-const nodeCall = require("../common/node");
-const smLib = require("../common/sitemap");
+const { sitemap: sm } = require("../common/sitemap");
 const base = require("./_base");
 
 // The default options
@@ -44,116 +36,6 @@ const sitemapIndex = {
   }
 };
 
-/**
- * Make sure path exists so a sitemap write can succeed.
- *
- * @param {String} outputPath - File path and name.
- * @param {Function} callback - Completion callback.
- */
-function prepareWrite (outputPath, callback) {
-  const path = pathLib.parse(outputPath);
-  const dir = pathLib.join(path.root, path.dir);
-  mkdirp(dir).then(() => callback()).catch(callback);
-}
-
-/**
- * Parse a sitemap, generate input, and optionally update the local file
- * for sitemapPolicy purposes.
- * This writes the sitemap files out EVEN IF sitemapPolicy is false.
- * The thinking is that it's better/cheaper to do this b/c if sitemapPolicy is
- * enabled later, then there is something to compare against, for more savings.
- *
- * @param {Object} options - Sitemap options.
- * @param {Object} options.__sitemapIndex - Sitemap Index options.
- * @param {String} document - xml document string.
- * @param {Function} callback - A completion callback.
- */
-function processSitemap (options, document, callback) {
-  smLib.parse(options, document, err => {
-    const sitemapIndexOpts = options.__sitemapIndex;
-    const outputPath = base.outputFile(sitemapIndexOpts, options.source);
-
-    if (!err && sitemapIndexOpts.sitemapOutputDir) {
-      prepareWrite(outputPath, err => {
-        if (!err) {
-          fs.writeFile(outputPath, document, callback);
-        } else {
-          callback(common.prependMsgToErr(err, outputPath, true));
-        }
-      });
-    } else {
-      callback(err);
-    }
-  });
-}
-
-/**
- * Parse a sitemap index document
- * For each qualifying sitemap element, download, parse the sitemap.
- * Stops processing if an error occurs.
- *
- * @param {Object} options - Sitemap options.
- * @param {Boolean} options.sitemapPolicy - True if sitemap policy should be used.
- * @param {String} document - xml document string.
- * @param {Function} callback - A completion callback.
- */
-function parse (options, document, callback) {
-  xml2js.parseString(document, {
-    trim: true,
-    normalizeTags: true,
-    normalize: true
-  }, (err, result) => {
-    const sitemapUrls = [];
-    const sitemapIndexOptions = Object.assign({}, options, {
-      outputPath: undefined
-    });
-
-    if (!err) {
-      // Check if we should process each sitemap in the index.
-      _.forEach(
-        // if the sitemap index is malformed, just blow up
-        result.sitemapindex.sitemap,
-        sitemapNode => {
-          // optionally ignore current sitemaps by sitemap policy
-          const shouldProcess = !options.sitemapPolicy ||
-            !smLib.stillCurrent(sitemapNode, sitemapIndexOptions);
-
-          if (shouldProcess) {
-            // if sitemap index is malformed, just blow up
-            sitemapUrls.push(sitemapNode.loc[0]);
-          }
-        }
-      );
-
-      // Edge case message for clarity
-      if (sitemapUrls.length === 0) {
-        console.log("[*] No sitemaps qualified for processing");
-      }
-
-      // Get all the sitemaps, parse them, and generate input for each
-      Promise.all(sitemapUrls.map(sitemapUrl => {
-        const sitemapOptions = Object.assign({}, options, {
-          source: sitemapUrl,
-          input: "sitemap",
-          sitemapOutputDir: false,
-          __sitemapIndex: sitemapIndexOptions
-        });
-
-        console.log(`[+] Loading the sitemap: '${sitemapUrl}'`);
-
-        return nodeCall(smLib.getUrl, sitemapOptions, processSitemap);
-      }))
-        .then(() => {
-          // Ignore the array of results to this function
-          callback();
-        })
-        .catch(callback);
-    } else {
-      callback(err);
-    }
-  });
-}
-
 /**
  * generateInput
  *
@@ -165,11 +47,8 @@ function parse (options, document, callback) {
  * @param {String} options.source - Sitemap URL or file path.
  */
 function generateInput (options) {
-  return nodeCall(
-    common.isUrl(options.source) ? smLib.getUrl : smLib.getFile,
-    options,
-    parse
-  )
+  const retrieve = common.isUrl(options.source) ? sm.getUrl : sm.getFile;
+  return retrieve(options, sm.parse)
     .catch(err => {
       options._abort(err);
     })

package/lib/input-generators/sitemap.js

@@ -9,8 +9,7 @@
  */
 
 const common = require("../common");
-const nodeCall = require("../common/node");
-const smLib = require("../common/sitemap");
+const { sitemap: sm } = require("../common/sitemap");
 const base = require("./_base");
 
 // The default options
@@ -48,11 +47,8 @@ const sitemap = {
  * @returns {Promise} Resolves to undefined on completion.
  */
 function generateInput (options) {
-  return nodeCall(
-    common.isUrl(options.source) ? smLib.getUrl : smLib.getFile,
-    options,
-    smLib.parse
-  )
+  const retrieve = common.isUrl(options.source) ? sm.getUrl : sm.getFile;
+  return retrieve(options, sm.parse)
     .catch(err => {
       options._abort(err);
     })

package/lib/input-generators/textfile.js

@@ -8,10 +8,9 @@
  * Licensed under the MIT license.
  */
 
-const fs = require("fs");
+const fs = require("fs").promises;
 const base = require("./_base");
 const common = require("../common");
-const nodeCall = require("../common/node");
 
 // The default options
 const defaults = Object.freeze({
@@ -44,10 +43,7 @@ const textfile = {
  * @returns {Promise} resolves when complete.
  */
 function generateInput (options) {
-  return nodeCall(
-    fs.readFile,
-    options.source
-  )
+  return fs.readFile(options.source)
     .catch(err => {
       options._abort(err);
     })

package/lib/puppeteer/index.js

@@ -0,0 +1,98 @@
+#!/usr/bin/env -S node --unhandled-rejections=strict
+/**
+ * puppeteer process entry point
+ * 
+ * Command line arguments:
+ * 0   - /path/to/node
+ * 1   - *this script*
+ * 2   - outputFile
+ * 3   - url
+ * 4   - selector
+ * [5] - timeout
+ * [6] - filter
+ * [7] - debug
+ * [8] - slowMo
+ */
+const puppeteer = require('puppeteer');
+const path = require('path');
+const { promises: fs } = require('fs');
+
+if (process.argv.length < 5) {
+  throw Error(`puppeteer script '${
+    process.argv[1]
+  }' expected these arguments:\n\
+  OUTPUTFILE URL SELECTOR [TIMEOUT FILTER DEBUG SLOWMO]`);
+}
+
+const options = {
+  outputFile: process.argv[2],
+  url: process.argv[3],
+  selector: process.argv[4],
+  timeout: process.argv[5] ? parseInt(process.argv[5], 10) : 10000,
+  filter: process.argv[6] && process.argv[6] !== 'false' ? process.argv[6] : null,
+  debug: process.argv[7] && process.argv[7].toLowerCase() === 'true' ? true : false,
+  slowMo: process.argv[8] ? parseInt(process.argv[8], 10) : 500
+};
+const launchOptions = options.debug ? {
+  headless: false,
+  slowMo: options.slowMo,
+  devtools: true
+} : {};
+
+/**
+ * Run the snapshot script.
+ * 
+ * @returns {Promise} on completion
+ */
+async function run () {
+  let timeLeft, browser;
+  const start = Date.now();
+
+  timeLeft = options.timeout - 200; // leave some room for reporting
+  if (timeLeft < 0) timeLeft = 0;
+
+  try {
+    const filter =
+      options.filter ? require(options.filter) : passthru => passthru;
+
+    browser = await puppeteer.launch(launchOptions);
+    const page = await browser.newPage();
+    await page.goto(options.url, {
+      timeout: timeLeft
+    });
+
+    timeLeft -= (Date.now() - start);
+    if (timeLeft < 0) timeLeft = 0;
+
+    await fs.mkdir(path.dirname(options.outputFile), {
+      recursive: true
+    });
+
+    await page.waitForSelector(options.selector, {
+      Visible: true,
+      timeout: timeLeft
+    });
+
+    const content = await page.content();
+    await fs.writeFile(options.outputFile, filter(content));
+    const elapsed = Date.now() - start;
+
+    await browser.close();
+    
+    return `snapshot for ${options.url} finished in ${elapsed} ms\n\
+  written to ${options.outputFile}`;
+    } catch (e) {
+      const err = new Error(`failed to snapshot ${options.url}`, {
+        cause: e
+      });
+      if (browser) {
+        browser.close().finally(() => {
+          throw err;
+        });
+      }
+      throw err;
+    }
+}
+run().then(console.log).catch(e => {
+  throw e;
+});

package/lib/puppeteer/removeScripts.js

@@ -0,0 +1,8 @@
+/**
+ * Remove all script tags from content
+ */
+function removeScriptTags (content) {
+  return content.replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, '');
+}
+
+module.exports = removeScriptTags;

package/package.json

@@ -1,10 +1,10 @@
 {
   "name": "html-snapshots",
-  "version": "0.18.3",
+  "version": "1.0.0",
   "author": {
     "name": "Alex Grant",
     "email": "alex@localnerve.com",
-    "url": "http://localnerve.com"
+    "url": "https://www.localnerve.com"
   },
   "homepage": "https://github.com/localnerve/html-snapshots",
   "description": "A selector-based html snapshot tool using PhantomJS that sources sitemap.xml, robots.txt, or arbitrary input",
@@ -22,8 +22,12 @@
     "test": "mocha test/mocha/**/*.js --exit --recursive --reporter spec",
     "test:async": "mocha test/mocha/async/test.js --exit --reporter spec",
     "test:common": "mocha test/mocha/common/*.js --exit --reporter spec",
+    "test:browsers": "mocha test/mocha/browsers/test.js --exit --reporter spec",
+    "test:browsers:cover": "c8 -- npm run test:browsers",
     "test:input-generators": "mocha test/mocha/input-generators/*.js --exit --reporter spec",
+    "test:input-generators:debug": "mocha --inspect-brk test/mocha/input-generators/*.js --exit --reporter spec",
     "test:html-snapshots": "mocha test/mocha/html-snapshots/*.js --exit --reporter spec",
+    "test:html-snapshots:debug": "mocha --inspect-brk test/mocha/html-snapshots/*.js --exit --reporter spec",
     "test:cover": "c8 -- npm test",
     "validate": "npm ls"
   },
@@ -49,15 +53,16 @@
     "async": "3.2.4",
     "async-lock": "1.3.2",
     "combine-errors": "3.0.3",
+    "got": "12.5.0",
     "lodash": "4.17.21",
     "mkdirp": "1.0.4",
     "phantomjs-prebuilt": "2.1.16",
-    "request": "2.88.2",
+    "puppeteer": "^18.0.5",
     "rimraf": "3.0.2",
     "xml2js": "0.4.23"
   },
   "devDependencies": {
-    "eslint": "8.21.0",
+    "eslint": "8.23.1",
     "express": "4.18.1",
     "mocha": "10.0.0",
     "c8": "7.12.0",

package/test/helpers/options.js

@@ -9,7 +9,7 @@
  * If a global phantomjs is defined, decorates html-snapshots options to specify that global
  * In some test environments (travis), local phantomjs will not install if a global is found.
  */
-var assert = require("assert");
+const assert = require("assert");
 
 // @0.12.0, on Travis-ci phantomjs2 WILL install local WITH the older global.
 // So reverting back to decorateLocal only.
@@ -19,7 +19,9 @@ var assert = require("assert");
 function fillAbort (options) {
   if (!options._abort) {
     options._abort = function (err) {
-      assert.fail("test", "failed", err.toString(), "");
+      const msg = `[options._abort] called: ${err.toString()}`;
+      console.log(`@@@ ${msg}`);
+      assert.fail(msg);
     }
   }
 }

package/test/mocha/browsers/puppeteer.js

@@ -0,0 +1,106 @@
+/**
+ * Tests that target the puppeteer script specifically.
+ * 
+ * Copyright (c) 2013 - 2022, Alex Grant, LocalNerve, contributors
+ */
+/* global before, it */
+const assert = require("assert");
+const path = require("path");
+const rimraf = require("rimraf").sync;
+const spawn = require("child_process").spawn;
+const server = require("../../server");
+
+const outputDir = path.join(__dirname, "tmp");
+const port = 9340;
+
+function puppeteerTests () {
+  const snapshotScript = path.resolve(__dirname, '../../../lib/puppeteer/index.js');
+  const outputFile = path.join(outputDir, "snapshot.html");
+
+  /**
+   * Launch the puppeteer script as a process.
+   *
+   * @param {Array} args - array of puppeteer args
+   */
+  function spawnPuppeteer (args, done) {
+    const cp = spawn(
+      snapshotScript,
+      args
+      , { cwd: process.cwd(), stdio: "inherit", detached: true }
+    );
+    
+    cp.on("exit", code => {
+      if (code !== 0) {
+        return done(new Error(`error exit code: ${code}`));
+      }
+      done();
+    });
+  }
+
+  return function () {
+    before(function() {
+      server.start(path.join(__dirname, "./server"), port);
+    });
+    
+    it("should succeed with minimal args", function (done) {
+      this.timeout(5000);
+
+      rimraf(outputDir);
+
+      spawnPuppeteer([
+        outputFile,
+        `http://localhost:${port}/`,
+        "body"
+      ], done);
+    });
+
+    it("should fail if insufficient input args", function (done) {
+      rimraf(outputDir);
+      
+      spawnPuppeteer([
+        outputFile,
+        "http://someurl.domain/path/resource"
+      ], e => {
+        const msg = "expected process error";
+        assert.ok(e, msg);
+        done(e ? undefined : new Error(msg));
+      });
+    });
+
+    it("should fail with bad url", function (done) {
+      rimraf(outputDir);
+
+      spawnPuppeteer([
+        outputFile,
+        "http://nonexistent.domain/path/resource",
+        "body",
+        1000
+      ], e => {
+        const msg = "expected process error";
+        assert.ok(e, msg);
+        done(e ? undefined : new Error(msg));
+      });
+    });
+
+    it("should launch debugging", function (done) {
+      const timeout = 5000;
+      this.timeout(timeout);
+
+      rimraf(outputDir);
+
+      spawnPuppeteer([
+        outputFile,
+        `http://localhost:${port}/`,
+        "body",
+        timeout - 1000,
+        "false",
+        "true",
+        10
+      ], done);
+    });
+  };
+}
+
+module.exports = {
+  testSuite: puppeteerTests
+};

package/test/mocha/browsers/server/index.html

@@ -0,0 +1,9 @@
+<!DOCTYPE html>
+<html lang="en-US">
+  <body>
+    <h1>Hello World</h1>
+    <p>
+      This is a simple test file, how do you do?
+    </p>
+  </body>
+</html>