doc-detective 1.0.5 → 1.0.7

package/README.md

@@ -530,9 +530,7 @@ If you opt into sending analytics, you can add additional servers that Doc Detec
 
 ```json
 {
-  ...
   "analytics": {
-    ...
     "customServers": [
       {
         "name": "My Analytics Server",
@@ -547,7 +545,6 @@ If you opt into sending analytics, you can add additional servers that Doc Detec
       }
     ]
   }
-  ...
 }
 ```
 
@@ -569,15 +566,12 @@ Analytics reporting is off by default. If you want to make extra sure that Doc D
 - New/upgraded test actions:
   - New: Test if a referenced image (such as an icon) is present in the captured screenshot.
   - Upgrade: Additional `httpRequest` input sanitization.
-  - Upgrade: `screenshot` and `startRecording` boolean for whether to perform the action or not if the expected output file already exists.
   - Upgrade: `startRecording` and `stopRecording` to support start, stop, and intermediate test action state image matching to track differences between video captures from different runs.
-  - Upgrade: `startRecording` to store the output file in a different location if a recorded action fails. This could help with debugging.
-- In-content test framing to identify when content is covered by a test defined in another file. This could enable content coverage analysis.
+- Content coverage analysis based on in-content test statements and markup declarations.
 - Suggest tests by parsing document text.
   - Automatically insert suggested tests based on document text.
   - Detailed field descriptions per action.
 - Refactor tests into individual files.
-- Rewrite cross-action recording status tracking.
 
 ## License
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "doc-detective",
-  "version": "1.0.5",
+  "version": "1.0.7",
   "description": "Unit test documentation (and record videos of those tests).",
   "main": "src/index.js",
   "scripts": {

package/sample/config.json

@@ -14,14 +14,19 @@
     ".json"
   ],
   "mediaDirectory": "sample",
+  "saveFailedTestRecordings": true,
+  "failedTestDirectory": "sample/failedTests",
   "fileTypes": [
     {
       "extensions": [
         ".md",
         ".mdx"
       ],
-      "openTestStatement": "[comment]: # (test",
-      "closeTestStatement": ")"
+      "testStartStatementOpen": "[comment]: # (test start",
+      "testStartStatementClose": ")",
+      "testEndStatement": "[comment]: # (test end)",
+      "actionStatementOpen": "[comment]: # (action",
+      "actionStatementClose": ")"
     },
     {
       "extensions": [
@@ -29,8 +34,11 @@
         ".htm",
         ".xml"
       ],
-      "openTestStatement": "<!-- test",
-      "closeTestStatement": "-->"
+      "testStartStatementOpen": "<!-- test start",
+      "testStartStatementClose": "-->",
+      "testEndStatement": "<!-- test end -->",
+      "actionStatementOpen": "<!-- action",
+      "actionStatementClose": "-->"
     }
   ],
   "browserOptions": {

package/sample/doc-content.md

@@ -2,21 +2,25 @@
 
 To use Google Search to find information on kittens,
 
+[comment]: # (test start {"id":"process-search-kittens" })
+
 1.  Open [Google Search](https://www.google.com).
 
-    [comment]: # (test {"testId":"process-search-kittens", "action":"startRecording", "overwrite":false, "filename":"results.gif", "fps":15})
-    [comment]: # (test {"testId":"process-search-kittens", "action":"goTo", "uri":"www.google.com"})
+    [comment]: # (action {"action":"startRecording", "overwrite":false, "filename":"results.gif", "fps":15})
+    [comment]: # (action {"action":"goTo", "uri":"www.google.com"})
 
 2.  In the search bar, enter "kittens", then press Enter.
 
-    [comment]: # (test {"testId":"process-search-kittens", "action":"moveMouse", "css":"#gbqfbb", "alignH": "center", "alignV": "center"})
-    [comment]: # (test {"testId":"process-search-kittens", "action":"wait", "duration":"5000"})
-    [comment]: # (test {"testId":"process-search-kittens", "action":"moveMouse", "css":"[title=Search]", "alignV": "center"})
-    [comment]: # (test {"testId":"process-search-kittens", "action":"type", "css":"[title=Search]", "keys":"kittens", "trailingSpecialKey":"Enter"})
-    [comment]: # (test {"testId":"process-search-kittens", "action":"wait", "duration":"5000"})
-    [comment]: # (test {"testId":"process-search-kittens", "action":"scroll", "y": 300})
-    [comment]: # (test {"testId":"process-search-kittens", "action":"stopRecording"})
-    [comment]: # (test {"testId":"process-search-kittens", "action":"screenshot", "filename":"results.png", "matchPrevious": true, "matchThreshold": 0.1})
+    [comment]: # (action {"action":"moveMouse", "css":"#gbqfbb", "alignH": "center", "alignV": "center"})
+    [comment]: # (action {"action":"wait", "duration":"5000"})
+    [comment]: # (action {"action":"moveMouse", "css":"[title=Search]", "alignV": "center"})
+    [comment]: # (action {"action":"type", "css":"[title=Search]", "keys":"kittens", "trailingSpecialKey":"Enter"})
+    [comment]: # (action {"action":"wait", "duration":"5000"})
+    [comment]: # (action {"action":"scroll", "y": 300})
+    [comment]: # (action {"action":"stopRecording"})
+    [comment]: # (action {"action":"screenshot", "filename":"results.png", "matchPrevious": true, "matchThreshold": 0.1})
+
+[comment]: # (test end {"id":"process-search-kittens" })
 
 Search results appear on the page.
 
@@ -26,17 +30,17 @@ Search results appear on the page.
 
 To go directly to a recommended result for your search, use the **I'm Feeling Lucky** button. If you're searching for american shorthair information,
 
-[comment]: # (test {"testId":"text-match-lucky", "action":"goTo", "uri":"www.google.com"})
-[comment]: # (test {"testId":"text-match-lucky", "action":"matchText", "css":"#gbqfbb", "text":"I'm Feeling Lucky"})
+[comment]: # (action {"testId":"text-match-lucky", "action":"goTo", "uri":"www.google.com"})
+[comment]: # (action {"testId":"text-match-lucky", "action":"matchText", "css":"#gbqfbb", "text":"I'm Feeling Lucky"})
 
 1.  Open [Google Search](https://www.google.com).
 
-    [comment]: # (test {"testId":"process-lucky-shorthair", "action":"goTo", "uri":"www.google.com"})
+    [comment]: # (action {"testId":"process-lucky-shorthair", "action":"goTo", "uri":"www.google.com"})
 
 2.  In the search bar, enter "american shorthair cats".
 
-    [comment]: # (test {"testId":"process-lucky-shorthair", "action":"type", "css":"[title=Search]", "keys":"american shorthair cats"})
+    [comment]: # (action {"testId":"process-lucky-shorthair", "action":"type", "css":"[title=Search]", "keys":"american shorthair cats"})
 
 3.  Click **I'm Feeling Lucky**.
 
-    [comment]: # (test {"testId":"process-lucky-shorthair", "action":"click", "css":"#gbqfbb"})
+    [comment]: # (action {"testId":"process-lucky-shorthair", "action":"click", "css":"#gbqfbb"})

package/sample/results.json

@@ -2,20 +2,65 @@
   "tests": [
     {
       "id": "process-search-kittens",
+      "file": "/config/workspace/doc-detective/sample/doc-content.md",
       "actions": [
+        {
+          "action": "startRecording",
+          "overwrite": false,
+          "filename": "results.gif",
+          "fps": 15,
+          "line": 9,
+          "result": {
+            "status": "PASS",
+            "description": "Started recording: /config/workspace/doc-detective/sample/temp_results.mp4",
+            "video": "/config/workspace/doc-detective/sample/temp_results.mp4"
+          }
+        },
         {
           "action": "goTo",
           "uri": "www.google.com",
+          "line": 10,
           "result": {
             "status": "PASS",
             "description": "Opened URI."
           }
         },
+        {
+          "action": "moveMouse",
+          "css": "#gbqfbb",
+          "alignH": "center",
+          "alignV": "center",
+          "line": 14,
+          "result": {
+            "status": "PASS",
+            "description": "Moved mouse to element."
+          }
+        },
+        {
+          "action": "wait",
+          "duration": "5000",
+          "line": 15,
+          "result": {
+            "status": "PASS",
+            "description": "Wait complete."
+          }
+        },
+        {
+          "action": "moveMouse",
+          "css": "[title=Search]",
+          "alignV": "center",
+          "line": 16,
+          "result": {
+            "status": "PASS",
+            "description": "Moved mouse to element."
+          }
+        },
         {
           "action": "type",
           "css": "[title=Search]",
           "keys": "kittens",
           "trailingSpecialKey": "Enter",
+          "line": 17,
           "result": {
             "status": "PASS",
             "description": "Typed keys."
@@ -24,16 +69,35 @@
         {
           "action": "wait",
           "duration": "5000",
+          "line": 18,
           "result": {
             "status": "PASS",
             "description": "Wait complete."
           }
         },
+        {
+          "action": "scroll",
+          "y": 300,
+          "line": 19,
+          "result": {
+            "status": "PASS",
+            "description": "Scroll complete."
+          }
+        },
+        {
+          "action": "stopRecording",
+          "line": 20,
+          "result": {
+            "status": "PASS",
+            "description": "Stopped recording: /config/workspace/doc-detective/sample/results.gif"
+          }
+        },
         {
           "action": "screenshot",
           "filename": "results.png",
           "matchPrevious": false,
           "matchThreshold": 0.1,
+          "line": 21,
           "result": {
             "status": "PASS",
             "description": "Captured screenshot.",
@@ -45,10 +109,12 @@
     },
     {
       "id": "text-match-lucky",
+      "file": "/config/workspace/doc-detective/sample/doc-content.md",
       "actions": [
         {
           "action": "goTo",
           "uri": "www.google.com",
+          "line": 33,
           "result": {
             "status": "PASS",
             "description": "Opened URI."
@@ -58,16 +124,7 @@
           "action": "matchText",
           "css": "#gbqfbb",
           "text": "I'm Feeling Lucky",
-          "result": {
-            "status": "PASS",
-            "description": "Element text matched expected text."
-          }
-        },
-        {
-          "action": "matchText",
-          "css": "#gbqfbb",
-          "text": "$TEXT",
-          "env": "./sample/variables.env",
+          "line": 34,
           "result": {
             "status": "PASS",
             "description": "Element text matched expected text."
@@ -78,11 +135,12 @@
     },
     {
       "id": "process-lucky-shorthair",
+      "file": "/config/workspace/doc-detective/sample/doc-content.md",
       "actions": [
         {
           "action": "goTo",
-          "uri": "$URL",
-          "env": "./sample/variables.env",
+          "uri": "www.google.com",
+          "line": 38,
           "result": {
             "status": "PASS",
             "description": "Opened URI."
@@ -91,7 +149,8 @@
         {
           "action": "type",
           "css": "[title=Search]",
-          "keys": "$SHORTHAIR_CAT_SEARCH",
+          "keys": "american shorthair cats",
+          "line": 42,
           "result": {
             "status": "PASS",
             "description": "Typed keys."
@@ -100,6 +159,7 @@
         {
           "action": "click",
           "css": "#gbqfbb",
+          "line": 46,
           "result": {
             "status": "PASS",
             "description": "Clicked element."
@@ -107,57 +167,6 @@
         }
       ],
       "status": "PASS"
-    },
-    {
-      "id": "non-ui-tests",
-      "actions": [
-        {
-          "action": "runShell",
-          "command": "echo $USERNAME",
-          "env": "./sample/variables.env",
-          "result": {
-            "status": "PASS",
-            "description": "Executed command.",
-            "stdout": "foo",
-            "stderr": "",
-            "exitCode": 0
-          }
-        },
-        {
-          "action": "checkLink",
-          "uri": "https://www.google.com",
-          "statusCodes": [
-            200
-          ],
-          "result": {
-            "status": "PASS",
-            "description": "Returned 200"
-          }
-        },
-        {
-          "action": "checkLink",
-          "uri": "$URL",
-          "statusCodes": [
-            200
-          ],
-          "result": {
-            "status": "PASS",
-            "description": "Returned 200"
-          }
-        },
-        {
-          "action": "httpRequest",
-          "uri": "$URL",
-          "statusCodes": [
-            200
-          ],
-          "result": {
-            "status": "PASS",
-            "description": "Returned 200."
-          }
-        }
-      ],
-      "status": "PASS"
     }
   ]
 }

package/sample/tests.json

@@ -2,6 +2,8 @@
   "tests": [
     {
       "id": "process-search-kittens",
+      "saveFailedTestRecordings": true,
+      "failedTestDirectory": "sample",
       "actions": [
         {
           "action": "goTo",

package/src/config.json

@@ -14,14 +14,19 @@
     ".json"
   ],
   "mediaDirectory": ".",
+  "saveFailedTestRecordings": true,
+  "failedTestDirectory": ".",
   "fileTypes": [
     {
       "extensions": [
         ".md",
         ".mdx"
       ],
-      "openTestStatement": "[comment]: # (test",
-      "closeTestStatement": ")"
+      "testStartStatementOpen": "[comment]: # (test start",
+      "testStartStatementClose": ")",
+      "testEndStatement": "[comment]: # (test end)",
+      "actionStatementOpen": "[comment]: # (action",
+      "actionStatementClose": ")"
     },
     {
       "extensions": [
@@ -29,8 +34,11 @@
         ".htm",
         ".xml"
       ],
-      "openTestStatement": "<!-- test",
-      "closeTestStatement": "-->"
+      "testStartStatementOpen": "<!-- test start",
+      "testStartStatementClose": "-->",
+      "testEndStatement": "<!-- test end -->",
+      "actionStatementOpen": "<!-- action",
+      "actionStatementClose": "-->"
     }
   ],
   "browserOptions": {

package/src/index.js

@@ -2,7 +2,7 @@ const {
   setArgs,
   setConfig,
   setFiles,
-  parseFiles,
+  parseTests,
   outputResults,
   log,
 } = require("./lib/utils");
@@ -30,7 +30,7 @@ async function main(config, argv) {
   log(config, "debug", files);
 
   // Set tests
-  const tests = parseFiles(config, files);
+  const tests = parseTests(config, files);
   if (config.logLevel === "debug") {
     console.log("(DEBUG) TESTS:");
     tests.tests.forEach((test) => {

package/src/lib/tests/record.js

@@ -87,12 +87,17 @@ async function startRecording(action, page, config) {
   }
 
   // Set FPS
-  fps = action.fps || action.gifFps || defaultPayload.fps;
+  targetFps = action.fps || action.gifFps || defaultPayload.fps;
   try {
-    fps = Number(fps);
+    targetFps = Number(targetFps);
+    if (targetFps >= 30) {
+      fps = targetFps;
+    } else {
+      fps = 30;
+    }
   } catch {
-    fps = defaultPayload;
-    log(config, "warning", `Invalid FPS. Reverting to default: ${fps}`);
+    targetFps = defaultPayload.fps;
+    log(config, "warning", `Invalid FPS. Reverting to default: ${targetFps}`);
   }
 
   // Set height
@@ -126,6 +131,7 @@ async function startRecording(action, page, config) {
       filepath,
       tempFilepath,
       fps,
+      targetFps,
       height,
       width,
     };
@@ -143,12 +149,30 @@ async function stopRecording(videoDetails, config) {
   let status;
   let description;
   let result;
+
+  if (typeof videoDetails.recorder === "undefined") {
+    status = "PASS";
+    description = `Skipping action. No action-defined recording in progress.`;
+    result = { status, description };
+    return { result };
+  }
+
+  recorder = videoDetails.recorder;
+  targetExtension = videoDetails.targetExtension;
+  height = videoDetails.height;
+  width = videoDetails.width;
+  filepath = videoDetails.filepath;
+  tempFilepath = videoDetails.tempFilepath;
+  fps = videoDetails.fps;
+  targetFps = videoDetails.targetFps;
+
   try {
-    await videoDetails.recorder.stop();
+    await recorder.stop();
     if (
-      videoDetails.targetExtension === ".gif" ||
-      videoDetails.height != config.browserOptions.height ||
-      videoDetails.width != config.browserOptions.width
+      targetExtension === ".gif" ||
+      height != config.browserOptions.height ||
+      width != config.browserOptions.width ||
+      targetFps != fps
     ) {
       let output = await convertVideo(config, videoDetails);
       filepath = output;

package/src/lib/tests/screenshot.js

@@ -87,36 +87,59 @@ async function screenshot(action, page, config) {
   if (action.matchPrevious) {
     const expected = PNG.sync.read(fs.readFileSync(previousFilePath));
     const actual = PNG.sync.read(fs.readFileSync(filePath));
-    const numDiffPixels = pixelmatch(
-      expected.data,
-      actual.data,
-      null,
-      expected.width,
-      expected.height,
-      {
-        threshold: action.matchThreshold,
-      }
-    );
-    fs.unlink(filePath, function (err) {
-      if (err) {
-        log(config, "warning", `Couldn't delete intermediate file: ${filePath}`);
+    try {
+      const numDiffPixels = pixelmatch(
+        expected.data,
+        actual.data,
+        null,
+        expected.width,
+        expected.height,
+        {
+          threshold: action.matchThreshold,
+        }
+      );
+      fs.unlink(filePath, function (err) {
+        if (err) {
+          log(
+            config,
+            "warning",
+            `Couldn't delete intermediate file: ${filePath}`
+          );
+        } else {
+          log(config, "debug", `Deleted intermediate file: ${filePath}`);
+        }
+      });
+      if (numDiffPixels) {
+        // FAIL: Couldn't capture screenshot
+        const diffPercentage =
+          numDiffPixels / (expected.width * expected.height);
+        status = "FAIL";
+        description = `Screenshot comparison had larger diff (${diffPercentage}) than threshold (${action.matchThreshold}).`;
+        result = { status, description };
+        return { result };
       } else {
-        log(config, "debug", `Deleted intermediate file: ${filePath}`);
+        // PASS
+        status = "PASS";
+        description = `Screenshot matches previously captured image.`;
+        result = { status, description, image: previousFilePath };
+        return { result };
       }
-    });
-    if (numDiffPixels) {
-      // FAIL: Couldn't capture screenshot
-      const diffPercentage = numDiffPixels / (expected.width * expected.height);
+    } catch {
+      fs.unlink(filePath, function (err) {
+        if (err) {
+          log(
+            config,
+            "warning",
+            `Couldn't delete intermediate file: ${filePath}`
+          );
+        } else {
+          log(config, "debug", `Deleted intermediate file: ${filePath}`);
+        }
+      });
       status = "FAIL";
-      description = `Screenshot comparison had larger diff (${diffPercentage}) than threshold (${action.matchThreshold}).`;
+      description = `Image sizes don't match.`;
       result = { status, description };
       return { result };
-    } else {
-      // PASS
-      status = "PASS";
-      description = `Screenshot matches previously captured image.`;
-      result = { status, description, image: previousFilePath };
-      return { result };
     }
   }
 }

package/src/lib/tests.js

@@ -2,7 +2,7 @@ const puppeteer = require("puppeteer");
 const fs = require("fs");
 const { exit, stdout, exitCode } = require("process");
 const { installMouseHelper } = require("./install-mouse-helper");
-const { setEnvs, log } = require("./utils");
+const { setEnvs, log, timestamp } = require("./utils");
 const util = require("util");
 const exec = util.promisify(require("child_process").exec);
 const axios = require("axios");
@@ -88,33 +88,54 @@ async function runTests(config, tests) {
     let pass = 0;
     let warning = 0;
     let fail = 0;
-    let videoDetails;
+    config.videoDetails = {};
+    config.debugRecording = {};
     // Instantiate page
     const page = await browser.newPage();
+    if (
+      test.saveFailedTestRecordings ||
+      (config.saveFailedTestRecordings &&
+        test.saveFailedTestRecordings != false)
+    ) {
+      failedTestDirectory =
+        test.failedTestDirectory || config.failedTestDirectory;
+      debugRecordingOptions = {
+        action: "startRecording",
+        mediaDirectory: failedTestDirectory,
+        filename: `${test.id}-${timestamp()}.mp4`,
+        overwrite: true,
+      };
+      config.debugRecording = await startRecording(
+        debugRecordingOptions,
+        page,
+        config
+      );
+    }
     // Instantiate mouse cursor
     await installMouseHelper(page);
     // Iterate through actions
     for (const action of test.actions) {
       log(config, "debug", `ACTION: ${JSON.stringify(action)}`);
-      action.result = await runAction(config, action, page, videoDetails);
+      action.result = await runAction(
+        config,
+        action,
+        page,
+        config.videoDetails
+      );
       if (action.result.videoDetails) {
-        videoDetails = action.result.videoDetails;
+        config.videoDetails = action.result.videoDetails;
       }
       action.result = action.result.result;
       if (action.result.status === "FAIL") fail++;
       if (action.result.status === "WARNING") warning++;
       if (action.result.status === "PASS") pass++;
-      log(config, "debug", `RESULT: ${action.result.status}. ${action.result.description}`);
+      log(
+        config,
+        "debug",
+        `RESULT: ${action.result.status}. ${action.result.description}`
+      );
     }
 
-    // Close open recorders/pages
-    if (videoDetails) {
-      await runAction("", { action: "stopRecording" }, "", videoDetails);
-    }
-    try {
-      await page.close();
-    } catch {}
-
     // Calc overall test result
     if (fail) {
       test.status = "FAIL";
@@ -126,6 +147,33 @@ async function runTests(config, tests) {
       console.log("Error: Couldn't read test action results.");
       exit(1);
     }
+
+    // Close open recorders/pages
+    if (config.debugRecording.videoDetails) {
+      await stopRecording(config.debugRecording.videoDetails, config);
+      if (!fail) {
+        fs.unlink(config.debugRecording.videoDetails.filepath, function (err) {
+          if (err) {
+            log(
+              config,
+              "warning",
+              `Couldn't delete debug recording: ${config.debugRecording.videoDetails.filepath}`
+            );
+          } else {
+            log(
+              config,
+              "debug",
+              `Deleted debug recording: ${config.debugRecording.videoDetails.filepath}`
+            );
+          }
+        });
+      }
+    }
+
+    // Close page
+    try {
+      await page.close();
+    } catch {}
   }
   await browser.close();
   return tests;

package/src/lib/utils.js

@@ -11,11 +11,12 @@ const defaultConfig = require("../config.json");
 exports.setArgs = setArgs;
 exports.setConfig = setConfig;
 exports.setFiles = setFiles;
-exports.parseFiles = parseFiles;
+exports.parseTests = parseTests;
 exports.outputResults = outputResults;
 exports.setEnvs = setEnvs;
 exports.loadEnvsForObject = loadEnvsForObject;
 exports.log = log;
+exports.timestamp = timestamp;
 
 const analyticsRequest =
   "Thanks for using Doc Detective! If you want to contribute to the project, consider sending analytics to help us understand usage patterns and functional gaps. To turn on analytics, set 'analytics.send = true' in your config, or use the '-a true' argument. See https://github.com/hawkeyexl/doc-detective#analytics";
@@ -182,7 +183,8 @@ function selectConfig(config, argv) {
 }
 
 function setEnv(config, argv) {
-  config.env = argv.env || process.env.DOC_ENV_PATH || config.env;
+  config.env =
+    argv.env || process.env.DOC_ENV_PATH || config.env || defaultConfig.env;
   if (config.env) {
     config.env = path.resolve(config.env);
     if (fs.existsSync(config.env)) {
@@ -201,7 +203,11 @@ function setEnv(config, argv) {
 }
 
 function setInput(config, argv) {
-  config.input = argv.input || process.env.DOC_INPUT_PATH || config.input;
+  config.input =
+    argv.input ||
+    process.env.DOC_INPUT_PATH ||
+    config.input ||
+    defaultConfig.input;
   if (config.input) {
     config.input = path.resolve(config.input);
     if (fs.existsSync(config.input)) {
@@ -225,14 +231,19 @@ function setInput(config, argv) {
 }
 
 function setOutput(config, argv) {
-  config.output = argv.output || process.env.DOC_OUTPUT_PATH || config.output;
+  config.output =
+    argv.output ||
+    process.env.DOC_OUTPUT_PATH ||
+    config.output ||
+    defaultConfig.output;
   config.output = path.resolve(config.output);
   log(config, "debug", `Output path set: ${config.output}`);
   return config;
 }
 
 function setSetup(config, argv) {
-  config.setup = argv.setup || process.env.DOC_SETUP || config.setup;
+  config.setup =
+    argv.setup || process.env.DOC_SETUP || config.setup || defaultConfig.setup;
   if (config.setup === "") {
     log(config, "debug", `No setup tests.`);
     return config;
@@ -249,7 +260,11 @@ function setSetup(config, argv) {
 }
 
 function setCleanup(config, argv) {
-  config.cleanup = argv.cleanup || process.env.DOC_CLEANUP || config.cleanup;
+  config.cleanup =
+    argv.cleanup ||
+    process.env.DOC_CLEANUP ||
+    config.cleanup ||
+    defaultConfig.cleanup;
   if (config.cleanup === "") {
     log(config, "debug", `No cleanup tests.`);
     return config;
@@ -269,7 +284,8 @@ function setMediaDirectory(config, argv) {
   config.mediaDirectory =
     argv.mediaDir ||
     process.env.DOC_MEDIA_DIRECTORY_PATH ||
-    config.mediaDirectory;
+    config.mediaDirectory ||
+    defaultConfig.mediaDirectory;
   config.mediaDirectory = path.resolve(config.mediaDirectory);
   if (fs.existsSync(config.mediaDirectory)) {
     log(config, "debug", `Media directory set: ${config.mediaDirectory}`);
@@ -284,9 +300,75 @@ function setMediaDirectory(config, argv) {
   return config;
 }
 
+function setFailedTestRecording(config, argv) {
+  config.saveFailedTestRecordings =
+    argv.saveFailedTestRecordings ||
+    process.env.DOC_SAVE_FAILED_RECORDINGS ||
+    config.saveFailedTestRecordings ||
+    defaultConfig.saveFailedTestRecordings;
+  switch (config.saveFailedTestRecordings) {
+    case true:
+    case "true":
+      config.saveFailedTestRecordings = true;
+      log(
+        config,
+        "debug",
+        `Save failed test recordings set: ${config.saveFailedTestRecordings}`
+      );
+      break;
+    case false:
+    case "false":
+      config.saveFailedTestRecordings = false;
+      log(
+        config,
+        "debug",
+        `Save failed test recordings set: ${config.saveFailedTestRecordings}`
+      );
+      log(config, "info", analyticsRequest);
+      break;
+    default:
+      config.saveFailedTestRecordings = defaultConfig.saveFailedTestRecordings;
+      log(
+        config,
+        "warning",
+        `Invalid save failed test recordings value. Reverted to default: ${config.saveFailedTestRecordings}`
+      );
+  }
+  return config;
+}
+
+function setFailedTestDirectory(config, argv) {
+  config.failedTestDirectory =
+    argv.failedTestDirectory ||
+    process.env.DOC_FAILED_TEST_DIRECTORY_PATH ||
+    config.failedTestDirectory ||
+    defaultConfig.failedTestDirectory;
+  config.failedTestDirectory = path.resolve(config.failedTestDirectory);
+  if (fs.existsSync(config.failedTestDirectory)) {
+    log(
+      config,
+      "debug",
+      `Failed test directory set: ${config.failedTestDirectory}`
+    );
+  } else {
+    config.failedTestDirectory = path.resolve(
+      defaultConfig.failedTestDirectory
+    );
+    log(
+      config,
+      "warning",
+      `Invalid failed test directory. Reverted to default: ${config.failedTestDirectory}`
+    );
+  }
+  return config;
+}
+
 function setRecursion(config, argv) {
   config.recursive =
-    argv.recursive || process.env.DOC_RECURSIVE || config.recursive;
+    argv.recursive ||
+    process.env.DOC_RECURSIVE ||
+    config.recursive ||
+    defaultConfig.recursive;
   switch (config.recursive) {
     case true:
     case "true":
@@ -309,9 +391,33 @@ function setRecursion(config, argv) {
   return config;
 }
 
+function setFileTypes(config, argv) {
+  config.fileTypes =
+    argv.fileTypes ||
+    process.env.DOC_FILE_TYPES ||
+    config.fileTypes ||
+    defaultConfig.fileTypes;
+  if (config.fileTypes.length > 0) {
+    log(config, "debug", `File types set: ${JSON.stringify(config.fileTypes)}`);
+  } else {
+    config.fileTypes = defaultConfig.fileTypes;
+    log(
+      config,
+      "debug",
+      `Invalid file type value(s). Reverted to default: ${JSON.stringify(
+        config.fileTypes
+      )}`
+    );
+  }
+  return config;
+}
+
 function setTestFileExtensions(config, argv) {
   config.testExtensions =
-    argv.ext || process.env.DOC_TEST_EXTENSTIONS || config.testExtensions;
+    argv.ext ||
+    process.env.DOC_TEST_EXTENSTIONS ||
+    config.testExtensions ||
+    defaultConfig.testExtensions;
   if (typeof config.testExtensions === "string")
     config.testExtensions = config.testExtensions
       .replace(/\s+/g, "")
@@ -339,7 +445,8 @@ function setBrowserHeadless(config, argv) {
   config.browserOptions.headless =
     argv.browserHeadless ||
     process.env.DOC_BROWSER_HEADLESS ||
-    config.browserOptions.headless;
+    config.browserOptions.headless ||
+    defaultConfig.browserOptions.headless;
   switch (config.browserOptions.headless) {
     case true:
     case "true":
@@ -374,7 +481,8 @@ function setBrowserPath(config, argv) {
   config.browserOptions.path =
     argv.browserPath ||
     process.env.DOC_BROWSER_PATH ||
-    config.browserOptions.path;
+    config.browserOptions.path ||
+    defaultConfig.browserOptions.path;
   if (config.browserOptions.path === "") {
     log(config, "debug", `Browser set to default Chromium install.`);
     return config;
@@ -398,7 +506,8 @@ function setBrowserHeight(config, argv) {
   config.browserOptions.height =
     argv.browserHeight ||
     process.env.DOC_BROWSER_HEIGHT ||
-    config.browserOptions.height;
+    config.browserOptions.height ||
+    defaultConfig.browserOptions.height;
   if (typeof config.browserOptions.height === "string") {
     try {
       config.browserOptions.height = Number(config.browserOptions.height);
@@ -428,7 +537,8 @@ function setBrowserWidth(config, argv) {
   config.browserOptions.width =
     argv.browserWidth ||
     process.env.DOC_BROWSER_WIDTH ||
-    config.browserOptions.width;
+    config.browserOptions.width ||
+    defaultConfig.browserOptions.width;
   if (typeof config.browserOptions.width === "string") {
     try {
       config.browserOptions.width = Number(config.browserOptions.width);
@@ -456,7 +566,10 @@ function setBrowserWidth(config, argv) {
 
 function setAnalytics(config, argv) {
   config.analytics.send =
-    argv.analytics || process.env.DOC_ANALYTICS || config.analytics.send;
+    argv.analytics ||
+    process.env.DOC_ANALYTICS ||
+    config.analytics.send ||
+    defaultConfig.analytics.send;
   switch (config.analytics.send) {
     case true:
     case "true":
@@ -484,7 +597,8 @@ function setAnalyticsUserId(config, argv) {
   config.analytics.userId =
     argv.analyticsUserId ||
     process.env.DOC_ANALYTICS_USER_ID ||
-    config.analytics.userId;
+    config.analytics.userId ||
+    defaultConfig.analytics.userId;
   log(config, "debug", `Analytics user ID set: ${config.analytics.userId}`);
   return config;
 }
@@ -494,7 +608,8 @@ function setAnalyticsDetailLevel(config, argv) {
   detailLevel =
     argv.analyticsDetailLevel ||
     process.env.DOC_ANALYTCS_DETAIL_LEVEL ||
-    config.analytics.detailLevel;
+    config.analytics.detailLevel ||
+    defaultConfig.analytics.detailLevel;
   detailLevel = String(detailLevel).toLowerCase();
   if (enums.indexOf(detailLevel) >= 0) {
     config.analytics.detailLevel = detailLevel;
@@ -537,8 +652,14 @@ function setConfig(config, argv) {
 
   config = setMediaDirectory(config, argv);
 
+  config = setFailedTestRecording(config, argv);
+
+  config = setFailedTestDirectory(config, argv);
+
   config = setRecursion(config, argv);
 
+  config = setFileTypes(config, argv);
+
   config = setTestFileExtensions(config, argv);
 
   config = setBrowserHeadless(config, argv);
@@ -620,13 +741,14 @@ function setFiles(config) {
 }
 
 // Parse files for tests
-function parseFiles(config, files) {
+function parseTests(config, files) {
   let json = { tests: [] };
 
   // Loop through test files
   files.forEach((file) => {
     log(config, "debug", `file: ${file}`);
-    let id = uuid.v4();
+    let fileId = `${uuid.v4()}`;
+    let id = fileId;
     let line;
     let lineNumber = 1;
     let inputFile = new nReadlines(file);
@@ -634,36 +756,135 @@ function parseFiles(config, files) {
     let fileType = config.fileTypes.find((fileType) =>
       fileType.extensions.includes(extension)
     );
+    let testStartStatementOpen;
+    let testStartStatementClose;
+    let testEndStatement;
+    let actionStatementOpen;
+    let actionStatementClose;
+
+    if (typeof fileType != "undefined") {
+      testStartStatementOpen = fileType.testStartStatementOpen;
+      if (!testStartStatementOpen) {
+        log(
+          config,
+          "warning",
+          `Skipping tests in ${file}. No 'testStartStatementOpen' value specified.`
+        );
+        return;
+      }
+      testStartStatementClose = fileType.testStartStatementClose;
+      if (!testStartStatementClose) {
+        log(
+          config,
+          "warning",
+          `Skipping tests in ${file}. No 'testStartStatementClose' value specified.`
+        );
+        return;
+      }
+      testEndStatement = fileType.testEndStatement;
+      if (!testEndStatement) {
+        log(
+          config,
+          "warning",
+          `Skipping tests in ${file}. No 'testEndStatement' value specified.`
+        );
+        return;
+      }
+      actionStatementOpen =
+        fileType.actionStatementOpen ||
+        fileType.openActionStatement ||
+        fileType.openTestStatement;
+      if (!actionStatementOpen) {
+        log(
+          config,
+          "warning",
+          `Skipping tests in ${file}. No 'actionStatementOpen' value specified.`
+        );
+        return;
+      }
+      actionStatementClose =
+        fileType.actionStatementClose ||
+        fileType.closeActionStatement ||
+        fileType.closeTestStatement;
+      if (!actionStatementClose) {
+        log(
+          config,
+          "warning",
+          `Skipping tests in ${file}. No 'actionStatementClose' value specified.`
+        );
+        return;
+      }
+    }
+
     if (!fileType && extension !== ".json") {
       // Missing filetype options
-      console.log(
-        `Error: Specify options for the ${extension} extension in your config file.`
+      log(
+        config,
+        "warning",
+        `Skipping file with ${extension} extension. Specify options for the ${extension} extension in your config file.`
       );
-      exit(1);
+      return;
     }
 
     // If file is JSON, add tests straight to array
     if (path.extname(file) === ".json") {
       content = require(file);
-      content.tests.forEach((test) => {
-        json.tests.push(test);
-      });
+      if (typeof content.tests === "object" && content.tests.length > 0) {
+        content.tests.forEach((test) => {
+          json.tests.push(test);
+        });
+      } else {
+        log(
+          config,
+          "warning",
+          `Skipping ${file} because of unexpected object structure.`
+        );
+        return;
+      }
     } else {
       // Loop through lines
       while ((line = inputFile.next())) {
-        let lineJson = "";
-        let subStart = "";
-        let subEnd = "";
-        if (line.includes(fileType.openTestStatement)) {
-          const lineAscii = line.toString("ascii");
-          if (fileType.closeTestStatement) {
-            subEnd = lineAscii.lastIndexOf(fileType.closeTestStatement);
+        let lineJson;
+        let subStart;
+        let subEnd;
+        const lineAscii = line.toString("ascii");
+
+        if (line.includes(testStartStatementOpen)) {
+          // Test start
+          if (testStartStatementClose) {
+            subEnd = lineAscii.lastIndexOf(testStartStatementClose);
           } else {
             subEnd = lineAscii.length;
           }
           subStart =
-            lineAscii.indexOf(fileType.openTestStatement) +
-            fileType.openTestStatement.length;
+            lineAscii.indexOf(testStartStatementOpen) +
+            testStartStatementOpen.length;
+          lineJson = JSON.parse(lineAscii.substring(subStart, subEnd));
+          // If test is defined in this file instead of referencing a test defined in another file
+          if (!lineJson.file) {
+            test = { id, file, actions: [] };
+            if (lineJson.id) {
+              test.id = lineJson.id;
+              // Set ID for following actions
+              id = lineJson.id;
+            }
+            if (lineJson.saveFailedTestRecordings)
+              test.saveFailedTestRecordings = lineJson.saveFailedTestRecordings;
+            if (lineJson.failedTestDirectory)
+              test.failedTestDirectory = lineJson.failedTestDirectory;
+            json.tests.push(test);
+          }
+        } else if (line.includes(testEndStatement)) {
+          // Revert back to file-based ID
+          id = fileId;
+        } else if (line.includes(actionStatementOpen)) {
+          if (actionStatementClose) {
+            subEnd = lineAscii.lastIndexOf(actionStatementClose);
+          } else {
+            subEnd = lineAscii.length;
+          }
+          subStart =
+            lineAscii.indexOf(actionStatementOpen) + actionStatementOpen.length;
           lineJson = JSON.parse(lineAscii.substring(subStart, subEnd));
           if (!lineJson.testId) {
             lineJson.testId = id;
@@ -692,6 +913,7 @@ async function outputResults(config, results) {
   log(config, "info", "RESULTS:");
   log(config, "info", results);
   log(config, "info", `See detailed results at ${config.output}`);
+  log(config, "info", "Cleaning up and finishing post-processing.");
 }
 
 async function setEnvs(envsFile) {
@@ -759,3 +981,14 @@ function loadEnvsForObject(object) {
   });
   return object;
 }
+
+function timestamp() {
+  let timestamp = new Date();
+  return `${timestamp.getFullYear()}${("0" + (timestamp.getMonth() + 1)).slice(
+    -2
+  )}${("0" + timestamp.getDate()).slice(-2)}-${(
+    "0" + timestamp.getHours()
+  ).slice(-2)}${("0" + timestamp.getMinutes()).slice(-2)}${(
+    "0" + timestamp.getSeconds()
+  ).slice(-2)}`;
+}