doc-detective 1.0.4 → 1.0.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "doc-detective",
-  "version": "1.0.4",
+  "version": "1.0.6",
   "description": "Unit test documentation (and record videos of those tests).",
   "main": "src/index.js",
   "scripts": {

package/sample/config.json

@@ -14,14 +14,16 @@
     ".json"
   ],
   "mediaDirectory": "sample",
+  "saveFailedTestRecordings": true,
+  "failedTestDirectory": "sample/failedTests",
   "fileTypes": [
     {
       "extensions": [
         ".md",
         ".mdx"
       ],
-      "openTestStatement": "[comment]: # (test",
-      "closeTestStatement": ")"
+      "openActionStatement": "[comment]: # (action",
+      "closeActionStatement": ")"
     },
     {
       "extensions": [
@@ -29,8 +31,8 @@
         ".htm",
         ".xml"
       ],
-      "openTestStatement": "<!-- test",
-      "closeTestStatement": "-->"
+      "openActionStatement": "<!-- action",
+      "closeActionStatement": "-->"
     }
   ],
   "browserOptions": {

package/sample/doc-content.md

@@ -4,19 +4,19 @@ To use Google Search to find information on kittens,
 
 1.  Open [Google Search](https://www.google.com).
 
-    [comment]: # (test {"testId":"process-search-kittens", "action":"startRecording", "overwrite":false, "filename":"results.gif", "fps":15})
-    [comment]: # (test {"testId":"process-search-kittens", "action":"goTo", "uri":"www.google.com"})
+    [comment]: # (action {"testId":"process-search-kittens", "action":"startRecording", "overwrite":false, "filename":"results.gif", "fps":15})
+    [comment]: # (action {"testId":"process-search-kittens", "action":"goTo", "uri":"www.google.com"})
 
 2.  In the search bar, enter "kittens", then press Enter.
 
-    [comment]: # (test {"testId":"process-search-kittens", "action":"moveMouse", "css":"#gbqfbb", "alignH": "center", "alignV": "center"})
-    [comment]: # (test {"testId":"process-search-kittens", "action":"wait", "duration":"5000"})
-    [comment]: # (test {"testId":"process-search-kittens", "action":"moveMouse", "css":"[title=Search]", "alignV": "center"})
-    [comment]: # (test {"testId":"process-search-kittens", "action":"type", "css":"[title=Search]", "keys":"kittens", "trailingSpecialKey":"Enter"})
-    [comment]: # (test {"testId":"process-search-kittens", "action":"wait", "duration":"5000"})
-    [comment]: # (test {"testId":"process-search-kittens", "action":"scroll", "y": 300})
-    [comment]: # (test {"testId":"process-search-kittens", "action":"stopRecording"})
-    [comment]: # (test {"testId":"process-search-kittens", "action":"screenshot", "filename":"results.png", "matchPrevious": true, "matchThreshold": 0.1})
+    [comment]: # (action {"testId":"process-search-kittens", "action":"moveMouse", "css":"#gbqfbb", "alignH": "center", "alignV": "center"})
+    [comment]: # (action {"testId":"process-search-kittens", "action":"wait", "duration":"5000"})
+    [comment]: # (action {"testId":"process-search-kittens", "action":"moveMouse", "css":"[title=Search]", "alignV": "center"})
+    [comment]: # (action {"testId":"process-search-kittens", "action":"type", "css":"[title=Search]", "keys":"kittens", "trailingSpecialKey":"Enter"})
+    [comment]: # (action {"testId":"process-search-kittens", "action":"wait", "duration":"5000"})
+    [comment]: # (action {"testId":"process-search-kittens", "action":"scroll", "y": 300})
+    [comment]: # (action {"testId":"process-search-kittens", "action":"stopRecording"})
+    [comment]: # (action {"testId":"process-search-kittens", "action":"screenshot", "filename":"results.png", "matchPrevious": true, "matchThreshold": 0.1})
 
 Search results appear on the page.
 
@@ -26,17 +26,17 @@ Search results appear on the page.
 
 To go directly to a recommended result for your search, use the **I'm Feeling Lucky** button. If you're searching for american shorthair information,
 
-[comment]: # (test {"testId":"text-match-lucky", "action":"goTo", "uri":"www.google.com"})
-[comment]: # (test {"testId":"text-match-lucky", "action":"matchText", "css":"#gbqfbb", "text":"I'm Feeling Lucky"})
+[comment]: # (action {"testId":"text-match-lucky", "action":"goTo", "uri":"www.google.com"})
+[comment]: # (action {"testId":"text-match-lucky", "action":"matchText", "css":"#gbqfbb", "text":"I'm Feeling Lucky"})
 
 1.  Open [Google Search](https://www.google.com).
 
-    [comment]: # (test {"testId":"process-lucky-shorthair", "action":"goTo", "uri":"www.google.com"})
+    [comment]: # (action {"testId":"process-lucky-shorthair", "action":"goTo", "uri":"www.google.com"})
 
 2.  In the search bar, enter "american shorthair cats".
 
-    [comment]: # (test {"testId":"process-lucky-shorthair", "action":"type", "css":"[title=Search]", "keys":"american shorthair cats"})
+    [comment]: # (action {"testId":"process-lucky-shorthair", "action":"type", "css":"[title=Search]", "keys":"american shorthair cats"})
 
 3.  Click **I'm Feeling Lucky**.
 
-    [comment]: # (test {"testId":"process-lucky-shorthair", "action":"click", "css":"#gbqfbb"})
+    [comment]: # (action {"testId":"process-lucky-shorthair", "action":"click", "css":"#gbqfbb"})

package/sample/tests.json

@@ -2,6 +2,8 @@
   "tests": [
     {
       "id": "process-search-kittens",
+      "saveFailedTestRecordings": true,
+      "failedTestDirectory": "sample",
       "actions": [
         {
           "action": "goTo",

package/src/config.json

@@ -14,14 +14,16 @@
     ".json"
   ],
   "mediaDirectory": ".",
+  "saveFailedTestRecordings": true,
+  "failedTestDirectory": ".",
   "fileTypes": [
     {
       "extensions": [
         ".md",
         ".mdx"
       ],
-      "openTestStatement": "[comment]: # (test",
-      "closeTestStatement": ")"
+      "openActionStatement": "[comment]: # (action",
+      "closeActionStatement": ")"
     },
     {
       "extensions": [
@@ -29,8 +31,8 @@
         ".htm",
         ".xml"
       ],
-      "openTestStatement": "<!-- test",
-      "closeTestStatement": "-->"
+      "openActionStatement": "<!-- action",
+      "closeActionStatement": "-->"
     }
   ],
   "browserOptions": {

package/src/lib/tests/httpRequest.js

@@ -1,6 +1,4 @@
 const axios = require("axios");
-const { async } = require("q");
-const { exit } = require("yargs");
 const { setEnvs, loadEnvsForObject } = require("../utils");
 
 exports.httpRequest = httpRequest;

package/src/lib/tests/record.js

@@ -87,12 +87,17 @@ async function startRecording(action, page, config) {
   }
 
   // Set FPS
-  fps = action.fps || action.gifFps || defaultPayload.fps;
+  targetFps = action.fps || action.gifFps || defaultPayload.fps;
   try {
-    fps = Number(fps);
+    targetFps = Number(targetFps);
+    if (targetFps >= 30) {
+      fps = targetFps;
+    } else {
+      fps = 30;
+    }
   } catch {
-    fps = defaultPayload;
-    log(config, "warning", `Invalid FPS. Reverting to default: ${fps}`);
+    targetFps = defaultPayload.fps;
+    log(config, "warning", `Invalid FPS. Reverting to default: ${targetFps}`);
   }
 
   // Set height
@@ -126,6 +131,7 @@ async function startRecording(action, page, config) {
       filepath,
       tempFilepath,
       fps,
+      targetFps,
       height,
       width,
     };
@@ -143,12 +149,30 @@ async function stopRecording(videoDetails, config) {
   let status;
   let description;
   let result;
+
+  if (typeof videoDetails.recorder === "undefined") {
+    status = "PASS";
+    description = `Skipping action. No action-defined recording in progress.`;
+    result = { status, description };
+    return { result };
+  }
+
+  recorder = videoDetails.recorder;
+  targetExtension = videoDetails.targetExtension;
+  height = videoDetails.height;
+  width = videoDetails.width;
+  filepath = videoDetails.filepath;
+  tempFilepath = videoDetails.tempFilepath;
+  fps = videoDetails.fps;
+  targetFps = videoDetails.targetFps;
+
   try {
-    await videoDetails.recorder.stop();
+    await recorder.stop();
     if (
-      videoDetails.targetExtension === ".gif" ||
-      videoDetails.height != config.browserOptions.height ||
-      videoDetails.width != config.browserOptions.width
+      targetExtension === ".gif" ||
+      height != config.browserOptions.height ||
+      width != config.browserOptions.width ||
+      targetFps != fps
     ) {
       let output = await convertVideo(config, videoDetails);
       filepath = output;

package/src/lib/tests/screenshot.js

@@ -87,36 +87,59 @@ async function screenshot(action, page, config) {
   if (action.matchPrevious) {
     const expected = PNG.sync.read(fs.readFileSync(previousFilePath));
     const actual = PNG.sync.read(fs.readFileSync(filePath));
-    const numDiffPixels = pixelmatch(
-      expected.data,
-      actual.data,
-      null,
-      expected.width,
-      expected.height,
-      {
-        threshold: action.matchThreshold,
-      }
-    );
-    fs.unlink(filePath, function (err) {
-      if (err) {
-        log(config, "warning", `Couldn't delete intermediate file: ${filePath}`);
+    try {
+      const numDiffPixels = pixelmatch(
+        expected.data,
+        actual.data,
+        null,
+        expected.width,
+        expected.height,
+        {
+          threshold: action.matchThreshold,
+        }
+      );
+      fs.unlink(filePath, function (err) {
+        if (err) {
+          log(
+            config,
+            "warning",
+            `Couldn't delete intermediate file: ${filePath}`
+          );
+        } else {
+          log(config, "debug", `Deleted intermediate file: ${filePath}`);
+        }
+      });
+      if (numDiffPixels) {
+        // FAIL: Couldn't capture screenshot
+        const diffPercentage =
+          numDiffPixels / (expected.width * expected.height);
+        status = "FAIL";
+        description = `Screenshot comparison had larger diff (${diffPercentage}) than threshold (${action.matchThreshold}).`;
+        result = { status, description };
+        return { result };
       } else {
-        log(config, "debug", `Deleted intermediate file: ${filePath}`);
+        // PASS
+        status = "PASS";
+        description = `Screenshot matches previously captured image.`;
+        result = { status, description, image: previousFilePath };
+        return { result };
       }
-    });
-    if (numDiffPixels) {
-      // FAIL: Couldn't capture screenshot
-      const diffPercentage = numDiffPixels / (expected.width * expected.height);
+    } catch {
+      fs.unlink(filePath, function (err) {
+        if (err) {
+          log(
+            config,
+            "warning",
+            `Couldn't delete intermediate file: ${filePath}`
+          );
+        } else {
+          log(config, "debug", `Deleted intermediate file: ${filePath}`);
+        }
+      });
       status = "FAIL";
-      description = `Screenshot comparison had larger diff (${diffPercentage}) than threshold (${action.matchThreshold}).`;
+      description = `Image sizes don't match.`;
       result = { status, description };
       return { result };
-    } else {
-      // PASS
-      status = "PASS";
-      description = `Screenshot matches previously captured image.`;
-      result = { status, description, image: previousFilePath };
-      return { result };
     }
   }
 }

package/src/lib/tests.js

@@ -2,7 +2,7 @@ const puppeteer = require("puppeteer");
 const fs = require("fs");
 const { exit, stdout, exitCode } = require("process");
 const { installMouseHelper } = require("./install-mouse-helper");
-const { setEnvs, log } = require("./utils");
+const { setEnvs, log, timestamp } = require("./utils");
 const util = require("util");
 const exec = util.promisify(require("child_process").exec);
 const axios = require("axios");
@@ -88,33 +88,54 @@ async function runTests(config, tests) {
     let pass = 0;
     let warning = 0;
     let fail = 0;
-    let videoDetails;
+    config.videoDetails = {};
+    config.debugRecording = {};
     // Instantiate page
     const page = await browser.newPage();
+    if (
+      test.saveFailedTestRecordings ||
+      (config.saveFailedTestRecordings &&
+        test.saveFailedTestRecordings != false)
+    ) {
+      failedTestDirectory =
+        test.failedTestDirectory || config.failedTestDirectory;
+      debugRecordingOptions = {
+        action: "startRecording",
+        mediaDirectory: failedTestDirectory,
+        filename: `${test.id}-${timestamp()}.mp4`,
+        overwrite: true,
+      };
+      config.debugRecording = await startRecording(
+        debugRecordingOptions,
+        page,
+        config
+      );
+    }
     // Instantiate mouse cursor
     await installMouseHelper(page);
     // Iterate through actions
     for (const action of test.actions) {
       log(config, "debug", `ACTION: ${JSON.stringify(action)}`);
-      action.result = await runAction(config, action, page, videoDetails);
+      action.result = await runAction(
+        config,
+        action,
+        page,
+        config.videoDetails
+      );
       if (action.result.videoDetails) {
-        videoDetails = action.result.videoDetails;
+        config.videoDetails = action.result.videoDetails;
       }
       action.result = action.result.result;
       if (action.result.status === "FAIL") fail++;
       if (action.result.status === "WARNING") warning++;
       if (action.result.status === "PASS") pass++;
-      log(config, "debug", `RESULT: ${action.result.status}. ${action.result.description}`);
+      log(
+        config,
+        "debug",
+        `RESULT: ${action.result.status}. ${action.result.description}`
+      );
     }
 
-    // Close open recorders/pages
-    if (videoDetails) {
-      await runAction("", { action: "stopRecording" }, "", videoDetails);
-    }
-    try {
-      await page.close();
-    } catch {}
-
     // Calc overall test result
     if (fail) {
       test.status = "FAIL";
@@ -126,6 +147,33 @@ async function runTests(config, tests) {
       console.log("Error: Couldn't read test action results.");
       exit(1);
     }
+
+    // Close open recorders/pages
+    if (config.debugRecording.videoDetails) {
+      await stopRecording(config.debugRecording.videoDetails, config);
+      if (!fail) {
+        fs.unlink(config.debugRecording.videoDetails.filepath, function (err) {
+          if (err) {
+            log(
+              config,
+              "warning",
+              `Couldn't delete debug recording: ${config.debugRecording.videoDetails.filepath}`
+            );
+          } else {
+            log(
+              config,
+              "debug",
+              `Deleted debug recording: ${config.debugRecording.videoDetails.filepath}`
+            );
+          }
+        });
+      }
+    }
+
+    // Close page
+    try {
+      await page.close();
+    } catch {}
   }
   await browser.close();
   return tests;

package/src/lib/utils.js

@@ -16,6 +16,7 @@ exports.outputResults = outputResults;
 exports.setEnvs = setEnvs;
 exports.loadEnvsForObject = loadEnvsForObject;
 exports.log = log;
+exports.timestamp = timestamp;
 
 const analyticsRequest =
   "Thanks for using Doc Detective! If you want to contribute to the project, consider sending analytics to help us understand usage patterns and functional gaps. To turn on analytics, set 'analytics.send = true' in your config, or use the '-a true' argument. See https://github.com/hawkeyexl/doc-detective#analytics";
@@ -284,6 +285,67 @@ function setMediaDirectory(config, argv) {
   return config;
 }
 
+function setFailedTestRecording(config, argv) {
+  config.saveFailedTestRecordings =
+    argv.saveFailedTestRecordings ||
+    process.env.DOC_SAVE_FAILED_RECORDINGS ||
+    config.saveFailedTestRecordings;
+  switch (config.saveFailedTestRecordings) {
+    case true:
+    case "true":
+      config.saveFailedTestRecordings = true;
+      log(
+        config,
+        "debug",
+        `Save failed test recordings set: ${config.saveFailedTestRecordings}`
+      );
+      break;
+    case false:
+    case "false":
+      config.saveFailedTestRecordings = false;
+      log(
+        config,
+        "debug",
+        `Save failed test recordings set: ${config.saveFailedTestRecordings}`
+      );
+      log(config, "info", analyticsRequest);
+      break;
+    default:
+      config.saveFailedTestRecordings = defaultConfig.saveFailedTestRecordings;
+      log(
+        config,
+        "warning",
+        `Invalid save failed test recordings value. Reverted to default: ${config.saveFailedTestRecordings}`
+      );
+  }
+  return config;
+}
+
+function setFailedTestDirectory(config, argv) {
+  config.failedTestDirectory =
+    argv.failedTestDirectory ||
+    process.env.DOC_FAILED_TEST_DIRECTORY_PATH ||
+    config.failedTestDirectory;
+  config.failedTestDirectory = path.resolve(config.failedTestDirectory);
+  if (fs.existsSync(config.failedTestDirectory)) {
+    log(
+      config,
+      "debug",
+      `Failed test directory set: ${config.failedTestDirectory}`
+    );
+  } else {
+    config.failedTestDirectory = path.resolve(
+      defaultConfig.failedTestDirectory
+    );
+    log(
+      config,
+      "warning",
+      `Invalid failed test directory. Reverted to default: ${config.failedTestDirectory}`
+    );
+  }
+  return config;
+}
+
 function setRecursion(config, argv) {
   config.recursive =
     argv.recursive || process.env.DOC_RECURSIVE || config.recursive;
@@ -537,6 +599,10 @@ function setConfig(config, argv) {
 
   config = setMediaDirectory(config, argv);
 
+  config = setFailedTestRecording(config, argv);
+
+  config = setFailedTestDirectory(config, argv);
+
   config = setRecursion(config, argv);
 
   config = setTestFileExtensions(config, argv);
@@ -654,16 +720,16 @@ function parseFiles(config, files) {
         let lineJson = "";
         let subStart = "";
         let subEnd = "";
-        if (line.includes(fileType.openTestStatement)) {
+        if (line.includes(fileType.openActionStatement)) {
           const lineAscii = line.toString("ascii");
-          if (fileType.closeTestStatement) {
-            subEnd = lineAscii.lastIndexOf(fileType.closeTestStatement);
+          if (fileType.closeActionStatement) {
+            subEnd = lineAscii.lastIndexOf(fileType.closeActionStatement);
           } else {
             subEnd = lineAscii.length;
           }
           subStart =
-            lineAscii.indexOf(fileType.openTestStatement) +
-            fileType.openTestStatement.length;
+            lineAscii.indexOf(fileType.openActionStatement) +
+            fileType.openActionStatement.length;
           lineJson = JSON.parse(lineAscii.substring(subStart, subEnd));
           if (!lineJson.testId) {
             lineJson.testId = id;
@@ -692,6 +758,7 @@ async function outputResults(config, results) {
   log(config, "info", "RESULTS:");
   log(config, "info", results);
   log(config, "info", `See detailed results at ${config.output}`);
+  log(config, "info", "Cleaning up and finishing post-processing.");
 }
 
 async function setEnvs(envsFile) {
@@ -759,3 +826,14 @@ function loadEnvsForObject(object) {
   });
   return object;
 }
+
+function timestamp() {
+  let timestamp = new Date();
+  return `${timestamp.getFullYear()}${("0" + (timestamp.getMonth() + 1)).slice(
+    -2
+  )}${("0" + timestamp.getDate()).slice(-2)}-${(
+    "0" + timestamp.getHours()
+  ).slice(-2)}${("0" + timestamp.getMinutes()).slice(-2)}${(
+    "0" + timestamp.getSeconds()
+  ).slice(-2)}`;
+}