doc-detective 1.1.0 → 1.2.0

package/cli/suggest.js

@@ -0,0 +1,4 @@
+const { suggest } = require("../src/index.js");
+const { argv } = require("node:process");
+
+suggest({}, argv);

package/package.json

@@ -1,11 +1,12 @@
 {
   "name": "doc-detective",
-  "version": "1.1.0",
+  "version": "1.2.0",
   "description": "Unit test documentation (and record videos of those tests).",
   "main": "src/index.js",
   "scripts": {
     "test": "node ./cli/index.js",
-    "coverage": "node ./cli/coverage.js"
+    "coverage": "node ./cli/coverage.js",
+    "suggest": "node ./cli/suggest.js"
   },
   "repository": {
     "type": "git",
@@ -33,6 +34,7 @@
     "n-readlines": "^1.0.1",
     "pixelmatch": "^5.3.0",
     "pngjs": "^6.0.0",
+    "prompt-sync": "^4.2.0",
     "puppeteer": "^13.7.0",
     "puppeteer-screen-recorder": "^2.0.2",
     "uuid": "^8.3.2",
@@ -41,4 +43,4 @@
   "devDependencies": {
     "make": "^0.8.1"
   }
-}
+}

package/sample/coverage.json

@@ -1,8 +1,8 @@
 {
   "name": "Doc Detective Content Coverage Report",
-  "timestamp": "20221017-083214",
+  "timestamp": "20221021-132211",
   "summary": {
-    "covered": 12,
+    "covered": 11,
     "uncovered": 0,
     "onscreenText": {
       "covered": 2,
@@ -37,14 +37,14 @@
       "uncovered": 0
     },
     "interaction": {
-      "covered": 2,
+      "covered": 1,
       "uncovered": 0
     }
   },
   "files": [
     {
       "file": "/config/workspace/doc-detective/sample/doc-content.md",
-      "covered": 12,
+      "covered": 11,
       "uncovered": 0,
       "onscreenText": {
         "covered": 2,
@@ -87,7 +87,7 @@
         "uncoveredMatches": []
       },
       "interaction": {
-        "covered": 2,
+        "covered": 1,
         "uncovered": 0,
         "uncoveredMatches": []
       }

package/src/config.json

@@ -6,6 +6,9 @@
   "cleanup": "",
   "recursive": true,
   "coverageOutput": "./sample/coverage.json",
+  "testSuggestions": {
+    "reportOutput": "./sample/suggestions.json"
+  },
   "testExtensions": [
     ".md",
     ".mdx",
@@ -31,42 +34,62 @@
       "actionStatementOpen": "[comment]: # (action",
       "actionStatementClose": ")",
       "markup": {
-        "onscreenText": [
-          "\\*\\*.+?\\*\\*"
-        ],
-        "emphasis": [
-          "(?<!\\*)\\*(?!\\*).+?(?<!\\*)\\*(?!\\*)"
-        ],
-        "image": [
-          "!\\[.+?\\]\\(.+?\\)"
-        ],
-        "hyperlink": [
-          "(?<!!)\\[.+?\\]\\(.+?\\)"
-        ],
-        "orderedList": [
-          "(?<=\n) *?[0-9][0-9]?[0-9]?.\\s*.*"
-        ],
-        "unorderedList": [
-          "(?<=\n) *?\\*.\\s*.*",
-          "(?<=\n) *?-.\\s*.*"
-        ],
-        "codeInline": [
-          "(?<!`)`(?!`).+?(?<!`)`(?!`)"
-        ],
-        "codeBlock": [
-          "(?=(```))(\\w|\\W)*(?<=```)"
-        ],
-        "interaction": [
-          "[cC]lick",
-          "[tT]ap",
-          "[tT]ouch",
-          "[sS]elect",
-          "[cC]hoose",
-          "[tT]oggle",
-          "[eE]nable",
-          "[dD]isable",
-          "[tT]urn [oO][ff|n]"
-        ]
+        "onscreenText": {
+          "regex": [
+            "\\*\\*.+?\\*\\*"
+          ]
+        },
+        "emphasis": {
+          "regex": [
+            "(?<!\\*)\\*(?!\\*).+?(?<!\\*)\\*(?!\\*)"
+          ]
+        },
+        "image": {
+          "regex": [
+            "!\\[.+?\\]\\(.+?\\)"
+          ]
+        },
+        "hyperlink": {
+          "regex": [
+            "(?<!!)\\[.+?\\]\\(.+?\\)"
+          ]
+        },
+        "orderedList": {
+          "regex": [
+            "(?<=\n) *?[0-9][0-9]?[0-9]?.\\s*.*"
+          ]
+        },
+        "unorderedList": {
+          "regex": [
+            "(?<=\n) *?\\*.\\s*.*",
+            "(?<=\n) *?-.\\s*.*"
+          ]
+        },
+        "codeInline": {
+          "regex": [
+            "(?<!`)`(?!`).+?(?<!`)`(?!`)"
+          ]
+        },
+        "codeBlock": {
+          "regex": [
+            "(?=(```))(\\w|\\W)*(?<=```)"
+          ]
+        },
+        "interaction": {
+          "regex": [
+            "[cC]lick",
+            "[tT]ap",
+            "[tT]ouch",
+            "[sS]elect",
+            "[cC]hoose",
+            "[tT]oggle",
+            "[eE]nable",
+            "[dD]isable",
+            "[tT]urn [oO][ff|n]",
+            "[tT]ype",
+            "[eE]nter"
+          ]
+        }
       }
     },
     {
@@ -82,41 +105,61 @@
       "actionStatementOpen": "<!-- action",
       "actionStatementClose": "-->",
       "markup": {
-        "onscreenText": [
-          "(?=(<b))(\\w|\\W)*(?<=<\/b>)"
-        ],
-        "emphasis": [
-          "(?=(<i))(\\w|\\W)*(?<=<\/i>)"
-        ],
-        "image": [
-          "(?=(<img))(\\w|\\W)*(?<=<\/img>|>)"
-        ],
-        "hyperlink": [
-          "(?=(<a))(\\w|\\W)*(?<=<\/a>)"
-        ],
-        "orderedList": [
-          "(?=(<ol))(\\w|\\W)*(?<=<\/ol>)"
-        ],
-        "unorderedList": [
-          "(?=(<ul))(\\w|\\W)*(?<=<\/ul>)"
-        ],
-        "codeInline": [
-          "(?=(<code))(\\w|\\W)*(?<=<\/code>)"
-        ],
-        "codeBlock": [
-          "(?=(<pre))(\\w|\\W)*(?<=<\/pre>)"
-        ],
-        "interaction": [
-          "[cC]lick",
-          "[tT]ap",
-          "[tT]ouch",
-          "[sS]elect",
-          "[cC]hoose",
-          "[tT]oggle",
-          "[eE]nable",
-          "[dD]isable",
-          "[tT]urn [oO][ff|n]"
-        ]
+        "onscreenText": {
+          "regex": [
+            "(?=(<b))(\\w|\\W)*(?<=<\/b>)"
+          ]
+        },
+        "emphasis": {
+          "regex": [
+            "(?=(<i))(\\w|\\W)*(?<=<\/i>)"
+          ]
+        },
+        "image": {
+          "regex": [
+            "(?=(<img))(\\w|\\W)*(?<=<\/img>|>)"
+          ]
+        },
+        "hyperlink": {
+          "regex": [
+            "(?=(<a))(\\w|\\W)*(?<=<\/a>)"
+          ]
+        },
+        "orderedList": {
+          "regex": [
+            "(?=(<ol))(\\w|\\W)*(?<=<\/ol>)"
+          ]
+        },
+        "unorderedList": {
+          "regex": [
+            "(?=(<ul))(\\w|\\W)*(?<=<\/ul>)"
+          ]
+        },
+        "codeInline": {
+          "regex": [
+            "(?=(<code))(\\w|\\W)*(?<=<\/code>)"
+          ]
+        },
+        "codeBlock": {
+          "regex": [
+            "(?=(<pre))(\\w|\\W)*(?<=<\/pre>)"
+          ]
+        },
+        "interaction": {
+          "regex": [
+            "[cC]lick",
+            "[tT]ap",
+            "[tT]ouch",
+            "[sS]elect",
+            "[cC]hoose",
+            "[tT]oggle",
+            "[eE]nable",
+            "[dD]isable",
+            "[tT]urn [oO][ff|n]",
+            "[tT]ype",
+            "[eE]nter"
+          ]
+        }
       }
     }
   ],

package/src/index.js

@@ -8,10 +8,14 @@ const {
 } = require("./lib/utils");
 const { sendAnalytics } = require("./lib/analytics.js");
 const { runTests } = require("./lib/tests");
-const { checkTestCoverage, checkMarkupCoverage } = require("./lib/coverage");
+const { checkTestCoverage, checkMarkupCoverage } = require("./lib/analysis");
+const { reportCoverage } = require("./lib/coverage");
+const { suggestTests, runSuggestions } = require("./lib/suggest");
+const { exit } = require("process");
 
 exports.run = main;
 exports.coverage = coverage;
+exports.suggest = suggest;
 
 async function main(config, argv) {
   // Set args
@@ -44,7 +48,7 @@ async function main(config, argv) {
   // Output
   outputResults(config.output, results, config);
   if (config.analytics.send) {
-    sendAnalytics(config, results);
+    // sendAnalytics(config, results);
   }
 }
 
@@ -65,13 +69,55 @@ async function coverage(config, argv) {
   log(config, "debug", files);
 
   const testCoverage = checkTestCoverage(config, files);
-  log(config, "debug", "(DEBUG) TEST COVERAGE:");
+  log(config, "debug", "TEST COVERAGE:");
   log(config, "debug", testCoverage);
 
   const markupCoverage = checkMarkupCoverage(config, testCoverage);
-  log(config, "debug", "(DEBUG) MARKUP COVERAGE:");
+  log(config, "debug", "MARKUP COVERAGE:");
   log(config, "debug", markupCoverage);
 
+  const coverageReport = reportCoverage(config, markupCoverage);
+  log(config, "debug", "COVERAGE REPORT:");
+  log(config, "debug", coverageReport);
+
+  // Output
+  outputResults(config.coverageOutput, coverageReport, config);
+}
+
+async function suggest(config, argv) {
+  // Set args
+  argv = setArgs(argv);
+  log(config, "debug", `ARGV:`);
+  log(config, "debug", argv);
+
+  // Set config
+  config = setConfig(config, argv);
+  log(config, "debug", `CONFIG:`);
+  log(config, "debug", config);
+
+  // Set files
+  const files = setFiles(config);
+  log(config, "debug", `FILES:`);
+  log(config, "debug", files);
+
+  const testCoverage = checkTestCoverage(config, files);
+  log(config, "debug", "TEST COVERAGE:");
+  log(config, "debug", testCoverage);
+
+  const markupCoverage = checkMarkupCoverage(config, testCoverage);
+  log(config, "debug", "MARKUP COVERAGE:");
+  log(config, "debug", markupCoverage);
+
+  const suggestionReport = suggestTests(config, markupCoverage);
+  log(config, "debug", "TEST SUGGESTIONS:");
+  log(config, "debug", suggestionReport);
+
+  await runSuggestions(config, suggestionReport);
+
   // Output
-  outputResults(config.coverageOutput, markupCoverage, config);
+  outputResults(
+    config.testSuggestions.reportOutput,
+    suggestionReport,
+    config
+  );
 }

package/src/lib/analysis.js

@@ -0,0 +1,276 @@
+const { log } = require("./utils");
+const uuid = require("uuid");
+const nReadlines = require("n-readlines");
+const path = require("path");
+const { exit } = require("process");
+const fs = require("fs");
+
+exports.checkTestCoverage = checkTestCoverage;
+exports.checkMarkupCoverage = checkMarkupCoverage;
+
+function checkTestCoverage(config, files) {
+  let testCoverage = {
+    files: [],
+    errors: [],
+  };
+
+  // Loop through test files
+  files.forEach((file) => {
+    log(config, "debug", `file: ${file}`);
+    fileJson = {
+      file,
+      coveredLines: [],
+      uncoveredLines: [],
+    };
+    let inTest = false;
+    let line;
+    let lineNumber = 1;
+    let inputFile = new nReadlines(file);
+    let extension = path.extname(file);
+    let fileType = config.fileTypes.find((fileType) =>
+      fileType.extensions.includes(extension)
+    );
+    fileJson.fileType = fileType;
+
+    if (typeof fileType === "undefined") {
+      // Missing filetype options
+      log(
+        config,
+        "debug",
+        `Skipping ${file}. Specify options for the ${extension} extension in your config file.`
+      );
+      return;
+    }
+
+    let testStartStatementOpen = fileType.testStartStatementOpen;
+    if (!testStartStatementOpen) {
+      log(
+        config,
+        "warning",
+        `Skipping ${file}. No 'testStartStatementOpen' value specified.`
+      );
+      return;
+    }
+    let testStartStatementClose = fileType.testStartStatementClose;
+    if (!testStartStatementClose) {
+      log(
+        config,
+        "warning",
+        `Skipping ${file}. No 'testStartStatementClose' value specified.`
+      );
+      return;
+    }
+    let testIgnoreStatement = fileType.testIgnoreStatement;
+    if (!testIgnoreStatement) {
+      log(
+        config,
+        "warning",
+        `Skipping ${file}. No 'testIgnoreStatement' value specified.`
+      );
+      return;
+    }
+    let testEndStatement = fileType.testEndStatement;
+    if (!testEndStatement) {
+      log(
+        config,
+        "warning",
+        `Skipping ${file}. No 'testEndStatement' value specified.`
+      );
+      return;
+    }
+    let actionStatementOpen =
+      fileType.actionStatementOpen ||
+      fileType.openActionStatement ||
+      fileType.openTestStatement;
+    if (!actionStatementOpen) {
+      log(
+        config,
+        "warning",
+        `Skipping ${file}. No 'actionStatementOpen' value specified.`
+      );
+      return;
+    }
+    let actionStatementClose =
+      fileType.actionStatementClose ||
+      fileType.closeActionStatement ||
+      fileType.closeTestStatement;
+    if (!actionStatementClose) {
+      log(
+        config,
+        "warning",
+        `Skipping ${file}. No 'actionStatementClose' value specified.`
+      );
+      return;
+    }
+
+    // Loop through lines
+    while ((line = inputFile.next())) {
+      let lineJson;
+      let subStart;
+      let subEnd;
+      let ignore = false;
+      const lineAscii = line.toString("ascii");
+
+      if (line.includes(testStartStatementOpen)) {
+        // Test start
+        if (testStartStatementClose) {
+          subEnd = lineAscii.lastIndexOf(testStartStatementClose);
+        } else {
+          subEnd = lineAscii.length;
+        }
+        subStart =
+          lineAscii.indexOf(testStartStatementOpen) +
+          testStartStatementOpen.length;
+        lineJson = JSON.parse(lineAscii.substring(subStart, subEnd));
+        // Set inTest to true
+        inTest = true;
+        ignore = true;
+        // Check if test is defined externally
+        if (lineJson.file) {
+          referencePath = path.resolve(path.dirname(file), lineJson.file);
+          // Check to make sure file exists
+          if (fs.existsSync(referencePath)) {
+            if (lineJson.id) {
+              remoteJson = require(referencePath);
+              // Make sure test of matching ID exists in file
+              idMatch = remoteJson.tests.find(
+                (test) => test.id === lineJson.id
+              );
+              if (!idMatch) {
+                // log error
+                testCoverage.errors.push({
+                  file,
+                  lineNumber,
+                  description: `Test with ID ${lineJson.id} missing from ${referencePath}.`,
+                });
+              }
+            }
+          } else {
+            // log error
+            testCoverage.errors.push({
+              file,
+              lineNumber,
+              description: `Referenced file missing: ${referencePath}.`,
+            });
+          }
+        }
+      } else if (line.includes(testIgnoreStatement)) {
+        inTest = true;
+        ignore = true;
+      } else if (line.includes(testEndStatement)) {
+        inTest = false;
+        ignore = true;
+      } else if (line.includes(actionStatementOpen) && line.includes(actionStatementClose)) {
+        ignore = true;
+      }
+
+      if (inTest && !ignore) {
+        fileJson.coveredLines.push(lineNumber);
+      } else if (!inTest && !ignore) {
+        fileJson.uncoveredLines.push(lineNumber);
+      }
+
+      lineNumber++;
+    }
+    testCoverage.files.push(fileJson);
+  });
+  return testCoverage;
+}
+
+function checkMarkupCoverage(config, testCoverage) {
+  let markupCoverage = testCoverage;
+
+  markupCoverage.files.forEach((file) => {
+    file.markup = {};
+    let extension = path.extname(file.file);
+    let markup = file.fileType.markup;
+
+    Object.keys(markup).forEach((mark) => {
+      if (markup[mark].regex.length === 1 && markup[mark].regex[0] === "") {
+        log(
+          config,
+          "warning",
+          `No regex for '${mark}'. Set 'fileType.markup.${mark}' for the '${extension}' extension in your config.`
+        );
+        delete markup[mark];
+      }
+    });
+
+    const fileBody = fs.readFileSync(file.file, {
+      encoding: "utf8",
+      flag: "r",
+    });
+
+    // Only keep marks that have a truthy (>0) length
+    Object.keys(markup).forEach((mark) => {
+      markCoverage = {
+        coveredLines: [],
+        coveredMatches: [],
+        uncoveredLines: [],
+        uncoveredMatches: [],
+      }
+
+      markup[mark].regex.forEach((matcher) => {
+        // Run a match
+        regex = new RegExp(matcher, "g");
+        matches = fileBody.match(regex);
+        if (matches != null) {
+          matches.forEach((match) => {
+            // Check for duplicates and handle lines separately
+            matchEscaped = match.replace(/[-[\]{}()*+?.,\\^$|#\s]/g, "\\$&");
+            start = 0;
+            occuranceRegex = new RegExp(matchEscaped, "g");
+            occurances = fileBody.match(occuranceRegex).length;
+            for (i = 0; i < occurances; i++) {
+              index = fileBody.slice(start).match(matchEscaped).index;
+              line = fileBody
+                .slice(0, start + index)
+                .split(/\r\n|\r|\n/).length;
+              start = start + index + 1;
+              matchObject = {
+                line,
+                indexInFile: start + index,
+                text: match,
+              };
+              isCovered = file.coveredLines.includes(line);
+              isUncovered = file.uncoveredLines.includes(line);
+              inCoveredMatches = markCoverage.coveredMatches.some(
+                (object) =>
+                  object.line === matchObject.line &&
+                  object.text === matchObject.text &&
+                  object.indexInFile === matchObject.indexInFile
+              );
+              inUncoveredMatches = markCoverage.uncoveredMatches.some(
+                (object) =>
+                  object.line === matchObject.line &&
+                  object.text === matchObject.text &&
+                  object.indexInFile === matchObject.indexInFile
+              );
+              inCoveredLines = markCoverage.coveredLines.includes(line);
+              inUncoveredLines = markCoverage.uncoveredLines.includes(line);
+              // console.log({
+              //   mark,
+              //   matchObject,
+              //   isCovered,
+              //   isUncovered,
+              //   inCoveredLines,
+              //   inCoveredMatches,
+              //   inUncoveredLines,
+              //   inUncoveredMatches,
+              // });
+              if (isCovered) {
+                if (!inCoveredLines) markCoverage.coveredLines.push(line);
+                if (!inCoveredMatches) markCoverage.coveredMatches.push(matchObject);
+              } else if (isUncovered) {
+                if (!inUncoveredLines) markCoverage.uncoveredLines.push(line);
+                if (!inUncoveredMatches) markCoverage.uncoveredMatches.push(matchObject);
+              }
+            }
+          });
+        }
+      });
+      file.markup[mark] = markCoverage;
+    });
+  });
+  return markupCoverage;
+}