doc-detective 1.0.2 → 1.0.4

package/README.md

@@ -1,4 +1,4 @@
-# Doc Detective: A Documentation Unit Testing Framework
+# Doc Detective: The Documentation Testing Framework
 
 Unit test documentation to validate UX flows, in-GUI text, and images. Primarily useful for process docs, Doc Detective supports test definitions single-sourced in documentation or defined in separate test files to suit your infrastructure needs.
 
@@ -6,7 +6,7 @@ Doc Detective ingests text files, parses them for test actions, then executes th
 
 This project handles test parsing and web-based UI testing--it doesn't support results reporting or notifications. This framework is a part of testing infrastructures and needs to be complimented by other components.
 
-Doc Detective uses `puppeteer` to install, launch, and drive Chromium to perform tests. `puppeteer` removes the requirement to manually configure a local web browser and enables easy screenshotting and video recording.
+Doc Detective uses `puppeteer` to install, launch, and drive Chromium to perform tests. `puppeteer` removes the requirement to manually configure a local web browser and enables easy screenshotting and video recording. In the event `puppeteer` fails to launch Chromium, Doc Detective tries to fall back to local installs of Chromium, Chrome, and Firefox.
 
 **Note:** By default, `puppeteer`'s Chromium doesn't run in Docker containers, which means that `puppeteer` doesn't work either. Don't run Doc Detective in a Docker container unless you first confirm that you have a custom implementation of headless Chrome/Chromium functional in the container. The approved answer to [this question](https://askubuntu.com/questions/79280/how-to-install-chrome-browser-properly-via-command-line) works for me, but it may not work in all environments.
 
@@ -99,6 +99,7 @@ Simple format:
   "css": "[title=Search]"
 }
 ```
+
 Advanced format:
 
 ```json
@@ -106,7 +107,7 @@ Advanced format:
   "action": "find",
   "css": "[title=Search]",
   "wait": {
-    "duration": 10000,
+    "duration": 10000
   },
   "matchText": {
     "text": "Search"
@@ -121,7 +122,7 @@ Advanced format:
   "type": {
     "keys": "$SHORTHAIR_CAT_SEARCH",
     "trailingSpecialKey": "Enter",
-    "envs": "./sample/variables.env"
+    "env": "./sample/variables.env"
   }
 }
 ```
@@ -155,7 +156,6 @@ Format:
   "alignV": "center",
   "offsetX": 10,
   "offsetY": 10
-
 }
 ```
 
@@ -186,7 +186,7 @@ Advanced format with an environment variable:
   "css": "input#password",
   "keys": "$PASSWORD",
   "trailingSpecialKey": "Enter",
-  "envs": "./sample/variables.env"
+  "env": "./sample/variables.env"
 }
 ```
 
@@ -255,9 +255,52 @@ Format:
 }
 ```
 
+### HTTP request
+
+Perform a generic HTTP request, for example to a REST API. Checks if the server returns an acceptable status code. If `uri` doesn't include a protocol, the protocol defaults to HTTPS. If `statusCodes` isn't specified, defaults to `[200]`.
+
+Format:
+
+```json
+{
+  "action": "httpRequest",
+  "env": "path/to/variables.env",
+  "uri": "https://www.api-server.com",
+  "method": "post",
+  "requestHeaders": {
+    "header": "value"
+  },
+  "requestParams": {
+    "param": "value"
+  },
+  "requestData": {
+    "field": "value"
+  },
+  "responseHeaders": {
+    "header": "value"
+  },
+  "responseData": {
+    "field": "value"
+  },
+  "statusCodes": [200]
+}
+```
+
+| Field             | Description                                                                                                                                                                                                                                                                                                                                                                                             | Example                                                                                                                                                                                    |
+| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| `action`          | Required. The action to perform.                                                                                                                                                                                                                                                                                                                                                                        | `"httpRequest"`                                                                                                                                                                            |
+| `env`             | Optional. File path for a .env file of variables to load.                                                                                                                                                                                                                                                                                                                                               | `"./path/to/variables.env"`                                                                                                                                                                |
+| `uri`             | Required. The URI to send the request to.<br>Supports setting values from environment variables.                                                                                                                                                                                                                                                                                                        | <ul><li>`"https://www.api-server.com"`</li><li>`"$URI"`</li></ul>                                                                                                                          |
+| `method`          | Optional. The HTTP method for the request. Similar to curl's `-X` option.<br>Defaults to "get". Accepted values: ["get", "post", "put", "patch", "delete"]                                                                                                                                                                                                                                              | `"get"`                                                                                                                                                                                    |
+| `requestHeaders`  | Optional. Headers to include in the request formatted as a JSON object. Similar to curl's `-H` option. <br> Supports setting the whole headers object and individual header values from environment variables.                                                                                                                                                                                          | <ul><li>`{ "Content-Type": "application/json" }`</li><li>`"$REQUEST_HEADERS"`</li><li>`{ "Content-Type": "$CONTENT_TYPE" }`</li></ul>                                                      |
+| `requestData`     | Optional. Data to send with the request. Similar to curl's `-d` option. <br> Supports setting the whole data object and individual data values from environment variables.                                                                                                                                                                                                                              | <ul><li>`{ "id": 1, "first_name": "George", "last_name": "Washington" }`</li><li>`"$REQUEST_DATA"`</li><li>`{ "id": 1, "first_name": "$FIRST_NAME", "last_name": "$LAST_NAME" }`</li></ul> |
+| `responseHeaders` | Optional. A non-exhaustive object of headers that should be included in the response. If a specified header is missing, or if a specified header value is incorrect, the action fails. If the response headers include headers that aren't specified by this option, the test can still pass. <br> Supports setting the whole headers object and individual header values from environment variables.   | <ul><li>`{ "Content-Type": "application/json" }`</li><li>`$RESPONSE_HEADERS`</li><li>`{ "Content-Type": "$CONTENT_TYPE" }`</li></ul>                                                       |
+| `responseData`    | Optional. A non-exhaustive data object that should be included in the response. If a specified data field is missing, or if a specified value is incorrect, the action fails. If the response data payload includes additional fields that aren't specified by this option, the test can still pass. <br> Supports setting the whole data object and individual data values from environment variables. | <ul><li>`{ "id": 1, "last_name": "Washington" }`</li><li>`"$REQUEST_DATA"`</li><li>`{ "id": 1, "last_name": "$LAST_NAME" }`</li></ul>                                                      |
+| `statusCodes`     | Optional. An array of accepted HTTP status response codes. Defaults to `[200]`. If the response's status code isn't included in this array, the action fails.                                                                                                                                                                                                                                           | [ 200, 204 ]                                                                                                                                                                               |
+
 ### Check a link
 
-Check if a link returns an acceptable status code from a GET request. If `uri` doesn't include a protocol, the protocol defaults to HTTPS. If `statuscodes` isn't specified, defaults to `[200]`.
+Check if a link returns an acceptable status code from a GET request. If `uri` doesn't include a protocol, the protocol defaults to HTTPS. If `statusCodes` isn't specified, defaults to `[200]`.
 
 Format:
 
@@ -265,25 +308,27 @@ Format:
 {
   "action": "checkLink",
   "uri": "https://www.google.com",
-  "statusCodes": [ 200 ]
+  "statusCodes": [200]
 }
 ```
 
 ### Start recording
 
-Start recording the current browser viewport. Must be followed by a `stopRecording` action. Supported extensions: .mp4, .gif
+Start recording the current browser viewport. Must be followed by a `stopRecording` action. Supported extensions: .mp4, .webm, .gif
 
-**Note:** `.gif` format is **not** recommended. Because of file format and encoding differences, `.gif` files tend to be ~6.5 times larger than `.mp4` files, and with lower visual fidelity. But if `.gif` is a hard requirement for you, it's here. Creating `.gif` files also creates `.mp4` files of the recordings.
+**Note:** `.gif` format is **not** recommended. Because of file format and encoding differences, `.gif` files tend to be ~6.5 times larger than `.mp4` files, and with lower visual fidelity. But if `.gif` is a hard requirement for you, it's here.
 
 Format:
 
 ```json
 {
   "action": "startRecording",
-  "mediaDirectory": "samples",
+  "overwrite": false,
+  "mediaDirectory": "./samples",
   "filename": "results.mp4",
-  "gifFps": 15,
-  "gifWidth": 400
+  "fps": 30,
+  "width": 1200,
+  "height": 800
 }
 ```
 
@@ -518,20 +563,21 @@ Analytics reporting is off by default. If you want to make extra sure that Doc D
 
 **Note:** Updating Doc Detective may revert any modified code, so be ready to make code edits repeatedly.
 
-## Potential future features
+## Potential future updates
 
-- Browser auto-detection and fallback.
 - Docker image with bundled Chromium/Chrome/Firefox.
 - New/upgraded test actions:
-  - New: Curl commands. (Support substitution/setting env vars. Only check for `200 OK`.)
   - New: Test if a referenced image (such as an icon) is present in the captured screenshot.
+  - Upgrade: Additional `httpRequest` input sanitization.
   - Upgrade: `screenshot` and `startRecording` boolean for whether to perform the action or not if the expected output file already exists.
-  - Upgrade: `startRecording` to remove MP4 when the output is a GIF.
   - Upgrade: `startRecording` and `stopRecording` to support start, stop, and intermediate test action state image matching to track differences between video captures from different runs.
   - Upgrade: `startRecording` to store the output file in a different location if a recorded action fails. This could help with debugging.
 - In-content test framing to identify when content is covered by a test defined in another file. This could enable content coverage analysis.
 - Suggest tests by parsing document text.
   - Automatically insert suggested tests based on document text.
+  - Detailed field descriptions per action.
+- Refactor tests into individual files.
+- Rewrite cross-action recording status tracking.
 
 ## License
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "doc-detective",
-  "version": "1.0.2",
+  "version": "1.0.4",
   "description": "Unit test documentation (and record videos of those tests).",
   "main": "src/index.js",
   "scripts": {

package/sample/config.json

@@ -36,8 +36,8 @@
   "browserOptions": {
     "headless": "true",
     "path": "",
-    "height": 600,
-    "width": 800
+    "height": 800,
+    "width": 1200
   },
   "analytics": {
     "send": false,

package/sample/doc-content.md

@@ -4,7 +4,7 @@ To use Google Search to find information on kittens,
 
 1.  Open [Google Search](https://www.google.com).
 
-    [comment]: # (test {"testId":"process-search-kittens", "action":"startRecording", "filename":"results.gif", "gifFps":15, "gifWidth":300})
+    [comment]: # (test {"testId":"process-search-kittens", "action":"startRecording", "overwrite":false, "filename":"results.gif", "fps":15})
     [comment]: # (test {"testId":"process-search-kittens", "action":"goTo", "uri":"www.google.com"})
 
 2.  In the search bar, enter "kittens", then press Enter.

package/sample/results.json

@@ -1,7 +1,7 @@
 {
   "tests": [
     {
-      "id": "process-lucky-shorthair",
+      "id": "process-search-kittens",
       "actions": [
         {
           "action": "goTo",
@@ -14,7 +14,84 @@
         {
           "action": "type",
           "css": "[title=Search]",
-          "keys": "american shorthair cats",
+          "keys": "kittens",
+          "trailingSpecialKey": "Enter",
+          "result": {
+            "status": "PASS",
+            "description": "Typed keys."
+          }
+        },
+        {
+          "action": "wait",
+          "duration": "5000",
+          "result": {
+            "status": "PASS",
+            "description": "Wait complete."
+          }
+        },
+        {
+          "action": "screenshot",
+          "filename": "results.png",
+          "matchPrevious": false,
+          "matchThreshold": 0.1,
+          "result": {
+            "status": "PASS",
+            "description": "Captured screenshot.",
+            "image": "/config/workspace/doc-detective/sample/results.png"
+          }
+        }
+      ],
+      "status": "PASS"
+    },
+    {
+      "id": "text-match-lucky",
+      "actions": [
+        {
+          "action": "goTo",
+          "uri": "www.google.com",
+          "result": {
+            "status": "PASS",
+            "description": "Opened URI."
+          }
+        },
+        {
+          "action": "matchText",
+          "css": "#gbqfbb",
+          "text": "I'm Feeling Lucky",
+          "result": {
+            "status": "PASS",
+            "description": "Element text matched expected text."
+          }
+        },
+        {
+          "action": "matchText",
+          "css": "#gbqfbb",
+          "text": "$TEXT",
+          "env": "./sample/variables.env",
+          "result": {
+            "status": "PASS",
+            "description": "Element text matched expected text."
+          }
+        }
+      ],
+      "status": "PASS"
+    },
+    {
+      "id": "process-lucky-shorthair",
+      "actions": [
+        {
+          "action": "goTo",
+          "uri": "$URL",
+          "env": "./sample/variables.env",
+          "result": {
+            "status": "PASS",
+            "description": "Opened URI."
+          }
+        },
+        {
+          "action": "type",
+          "css": "[title=Search]",
+          "keys": "$SHORTHAIR_CAT_SEARCH",
           "result": {
             "status": "PASS",
             "description": "Typed keys."
@@ -22,7 +99,6 @@
         },
         {
           "action": "click",
-          "moveMouse": true,
           "css": "#gbqfbb",
           "result": {
             "status": "PASS",
@@ -31,6 +107,57 @@
         }
       ],
       "status": "PASS"
+    },
+    {
+      "id": "non-ui-tests",
+      "actions": [
+        {
+          "action": "runShell",
+          "command": "echo $USERNAME",
+          "env": "./sample/variables.env",
+          "result": {
+            "status": "PASS",
+            "description": "Executed command.",
+            "stdout": "foo",
+            "stderr": "",
+            "exitCode": 0
+          }
+        },
+        {
+          "action": "checkLink",
+          "uri": "https://www.google.com",
+          "statusCodes": [
+            200
+          ],
+          "result": {
+            "status": "PASS",
+            "description": "Returned 200"
+          }
+        },
+        {
+          "action": "checkLink",
+          "uri": "$URL",
+          "statusCodes": [
+            200
+          ],
+          "result": {
+            "status": "PASS",
+            "description": "Returned 200"
+          }
+        },
+        {
+          "action": "httpRequest",
+          "uri": "$URL",
+          "statusCodes": [
+            200
+          ],
+          "result": {
+            "status": "PASS",
+            "description": "Returned 200."
+          }
+        }
+      ],
+      "status": "PASS"
     }
   ]
 }

package/sample/tests.json

@@ -85,6 +85,13 @@
           "statusCodes": [
             200
           ]
+        },
+        {
+          "action": "httpRequest",
+          "uri": "$URL",
+          "statusCodes": [
+            200
+          ]
         }
       ]
     }

package/src/index.js

@@ -40,8 +40,6 @@ async function main(config, argv) {
 
   // Run tests
   const results = await runTests(config, tests);
-  log(config, "debug", `RESULTS:`);
-  log(config, "debug", results);
 
   // Output
   outputResults(config, results);