doc-detective 3.4.0-dev.1 → 3.4.0-dev.10

package/dev/dev.config.json

@@ -0,0 +1,9 @@
+{
+  "input": "dev.spec.json",
+  "logLevel": "debug",
+  "integrations": {
+    "docDetectiveApi": {
+      "apiKey": "cd6f10bf0a47db1436cb3294b54e4f3e1599ca64be0313039fb0fc474ec8d116"
+    }
+  }
+}

package/dev/dev.spec.json

@@ -1,45 +1,19 @@
 {
-  "id": "Do all the things! - Spec",
   "tests": [
     {
-      "id": "Do all the things! - Test",
-      "description": "This test includes nearly every property across all actions.",
       "steps": [
-        {
-          "action": "setVariables",
-          "path": ".env"
-        },
         {
           "action": "runShell",
-          "command": "echo",
-          "args": ["$USER"]
+          "command": "echo Hello, World!"
         },
         {
           "action": "checkLink",
           "url": "https://www.duckduckgo.com"
         },
-        {
-          "action": "httpRequest",
-          "url": "https://reqres.in/api/users",
-          "method": "post",
-          "requestData": {
-            "name": "morpheus",
-            "job": "leader"
-          },
-          "responseData": {
-            "name": "morpheus",
-            "job": "leader"
-          },
-          "statusCodes": [200, 201]
-        },
         {
           "action": "goTo",
           "url": "https://www.google.com"
         },
-        {
-          "action": "startRecording",
-          "path": "test.mp4"
-        },
         {
           "action": "find",
           "selector": "[title=Search]",
@@ -49,12 +23,6 @@
           "typeKeys": {
             "keys": ["shorthair cat", "$ENTER$"]
           }
-        },
-        {
-          "action": "wait"
-        },
-        {
-          "action": "stopRecording"
         }
       ]
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "doc-detective",
-  "version": "3.4.0-dev.1",
+  "version": "3.4.0-dev.10",
   "description": "Treat doc content as testable assertions to validate doc accuracy and product UX.",
   "bin": {
     "doc-detective": "src/index.js"
@@ -33,8 +33,9 @@
   "homepage": "https://github.com/doc-detective/doc-detective#readme",
   "dependencies": {
     "@ffmpeg-installer/ffmpeg": "^1.1.0",
-    "doc-detective-common": "^3.4.0-dita.0-dev.1",
-    "doc-detective-core": "^3.4.0-dita.0-dev.1",
+    "axios": "^1.12.2",
+    "doc-detective-common": "3.4.0-dev.2",
+    "doc-detective-core": "3.4.0-dev.6",
     "yargs": "^17.7.2"
   },
   "devDependencies": {

package/scripts/bump-sync-version-core.js

@@ -20,8 +20,8 @@ function execCommand(command, options = {}) {
 
 function main() {
   // Clean git state
-    execCommand("git checkout -- .");
-    execCommand("git clean -fd");
+  execCommand("git checkout -- .");
+  execCommand("git clean -fd");
 
   // Get current project version
   const packageJsonPath = path.join(process.cwd(), "package.json");
@@ -54,42 +54,40 @@ function main() {
   // Extract major and minor versions using semver
   const projMajor = semver.major(projVersion);
   const projMinor = semver.minor(projVersion);
+  const projPatch = semver.patch(projVersion);
   const coreMajor = semver.major(coreVersion);
   const coreMinor = semver.minor(coreVersion);
+  const corePatch = semver.patch(coreVersion);
 
-  console.log(`Project version: ${projMajor}.${projMinor}.x`);
-  console.log(`core version: ${coreMajor}.${coreMinor}.x`);
+  console.log(`Project version: ${projMajor}.${projMinor}.${projPatch}`);
+  console.log(`core version: ${coreMajor}.${coreMinor}.${corePatch}`);
 
   let newVersion;
 
   if (projMajor !== coreMajor || projMinor !== coreMinor) {
     // Major or minor mismatch: set version to match doc-detective-core major.minor.0
     newVersion = `${coreMajor}.${coreMinor}.0`;
-
-    // Validate the new version before setting it
-    if (!semver.valid(newVersion)) {
-      console.error(`Error: Generated invalid version: ${newVersion}`);
-      process.exit(1);
-    }
-
     console.log(`Version mismatch detected. Setting version to: ${newVersion}`);
-    execCommand(`npm version --no-git-tag-version ${newVersion}`);
   } else {
     // Project version is already equal or greater than core version, just bump patch
-    console.log("Project version is current or ahead. Bumping patch version.");
-    execCommand("npm version patch --no-git-tag-version");
-    // Get the new version after bumping
-    const updatedPackageJson = JSON.parse(
-      fs.readFileSync(packageJsonPath, "utf8")
+    console.log(
+      "Project version is current or ahead. Bumping patch version to:",
+      newVersion
     );
-    newVersion = updatedPackageJson.version;
+    newVersion = `${projMajor}.${projMinor}.${projPatch + 1}`;
+  }
+  
+  // Validate the new version before setting it
+  if (!semver.valid(newVersion)) {
+    console.error(`Error: Generated invalid version: ${newVersion}`);
+    process.exit(1);
   }
 
+  execCommand(`npm version --no-git-tag-version ${newVersion}`);
+
   // Commit changes
   execCommand("git add package.json package-lock.json");
-  execCommand(
-    'git commit -m "update doc-detective-core [skip ci]"'
-  );
+  execCommand('git commit -m "update doc-detective-core [skip ci]"');
 
   // Create tag
   execCommand(`git tag "v${newVersion}"`);

package/src/index.js

@@ -1,7 +1,16 @@
 #!/usr/bin/env node
 
-const { runTests, runCoverage } = require("doc-detective-core");
-const { setArgs, setConfig, outputResults, setMeta, getVersionData, log } = require("./utils");
+const { runTests } = require("doc-detective-core");
+const {
+  setArgs,
+  setConfig,
+  outputResults,
+  setMeta,
+  getVersionData,
+  log,
+  getResolvedTestsFromEnv,
+  reportResults,
+} = require("./utils");
 const { argv } = require("node:process");
 const path = require("path");
 const fs = require("fs");
@@ -36,16 +45,28 @@ async function main(argv) {
   // Set config
   const config = await setConfig({ configPath: configPath, args: argv });
 
-  if (config.logLevel === "debug") {
-    console.log(`CLI:VERSION INFO:\n${JSON.stringify(getVersionData(), null, 2)}`);
-    console.log(`CLI:CONFIG:\n${JSON.stringify(config, null, 2)}`);
-  }
+  log(
+    `CLI:VERSION INFO:\n${JSON.stringify(getVersionData(), null, 2)}`,
+    "debug",
+    config
+  );
+  log(`CLI:CONFIG:\n${JSON.stringify(config, null, 2)}`, "debug", config);
+
+  // Check for DOC_DETECTIVE_API environment variable
+  let api = await getResolvedTestsFromEnv(config);
+  let resolvedTests = api?.resolvedTests || null;
+  let apiConfig = api?.apiConfig || null;
 
   // Run tests
   const output = config.output;
-  const results = await runTests(config);
+  const results = resolvedTests
+    ? await runTests(config, { resolvedTests })
+    : await runTests(config);
 
-  // Output results
-  await outputResults(config, output, results, { command: "runTests" });
-
-}
+  if (apiConfig) {
+    await reportResults({ apiConfig, results });
+  } else {
+    // Output results
+    await outputResults(config, output, results, { command: "runTests" });
+  }
+}

package/src/utils.js

@@ -5,6 +5,7 @@ const path = require("path");
 const fs = require("fs");
 const { spawn } = require("child_process");
 const os = require("os");
+const axios = require("axios");
 
 exports.setArgs = setArgs;
 exports.setConfig = setConfig;
@@ -12,6 +13,26 @@ exports.outputResults = outputResults;
 exports.spawnCommand = spawnCommand;
 exports.setMeta = setMeta;
 exports.getVersionData = getVersionData;
+exports.log = log;
+exports.getResolvedTestsFromEnv = getResolvedTestsFromEnv;
+exports.reportResults = reportResults;
+
+// Log function that respects logLevel
+function log(message, level = "info", config = {}) {
+  const logLevels = ["silent", "error", "warning", "info", "debug"];
+  const currentLevel = config.logLevel || "info";
+  const currentLevelIndex = logLevels.indexOf(currentLevel);
+  const messageLevelIndex = logLevels.indexOf(level);
+
+  // Only log if the message level is at or above the current log level
+  if (currentLevelIndex >= messageLevelIndex && messageLevelIndex > 0) {
+    if (level === "error") {
+      console.error(message);
+    } else {
+      console.log(message);
+    }
+  }
+}
 
 // Define args
 function setArgs(args) {
@@ -50,6 +71,116 @@ function setArgs(args) {
   return argv;
 }
 
+// Get resolved tests from environment variable, if set
+async function getResolvedTestsFromEnv(config = {}) {
+  if (!process.env.DOC_DETECTIVE_API) {
+    return null;
+  }
+
+  let resolvedTests = null;
+  let apiConfig = null;
+  try {
+    // Parse the environment variable as JSON
+    apiConfig = JSON.parse(process.env.DOC_DETECTIVE_API);
+
+    // Validate the structure: { accountId, url, token, contextIds }
+    if (!apiConfig.accountId || !apiConfig.url || !apiConfig.token || !apiConfig.contextIds) {
+      log(
+        "Invalid DOC_DETECTIVE_API: must contain 'accountId', 'url', 'token', and 'contextIds' properties",
+        "error",
+        config
+      );
+      process.exit(1);
+    }
+
+    log(`CLI:Fetching resolved tests from ${apiConfig.url}/resolved-tests`, "debug", config);
+
+    // Make GET request to the specified URL with token in header
+    const response = await axios.get(`${apiConfig.url}/resolved-tests`, {
+      headers: {
+        "x-runner-token": apiConfig.token,
+      },
+    });
+
+    // The response is the resolvedTests
+    resolvedTests = response.data;
+
+    // Validate against resolvedTests_v3 schema
+    const validation = validate({
+      schemaKey: "resolvedTests_v3",
+      object: resolvedTests,
+    });
+
+    if (!validation.valid) {
+      log(
+        "Invalid resolvedTests from API response. " + validation.errors,
+        "error",
+        config
+      );
+      process.exit(1);
+    }
+
+    // Get config from environment variable for merging
+    const envConfig = await getConfigFromEnv();
+    if (envConfig) {
+      // Apply config overrides to resolvedTests.config
+      if (resolvedTests.config) {
+        resolvedTests.config = { ...resolvedTests.config, ...envConfig };
+      } else {
+        resolvedTests.config = envConfig;
+      }
+    }
+
+    log(
+      `CLI:RESOLVED_TESTS:\n${JSON.stringify(resolvedTests, null, 2)}`,
+      "debug",
+      config
+    );
+  } catch (error) {
+    log(
+      `Error fetching resolved tests from DOC_DETECTIVE_API: ${error.message}`,
+      "error",
+      config
+    );
+    process.exit(1);
+  }
+  return { apiConfig, resolvedTests };
+}
+
+async function getConfigFromEnv() {
+  if (!process.env.DOC_DETECTIVE_CONFIG) {
+    return null;
+  }
+
+  let envConfig = null;
+  try {
+    // Parse the environment variable as JSON
+    envConfig = JSON.parse(process.env.DOC_DETECTIVE_CONFIG);
+
+    // Validate the environment variable config
+    const envValidation = validate({
+      schemaKey: "config_v3",
+      object: envConfig,
+    });
+
+    if (!envValidation.valid) {
+      console.error(
+        "Invalid config from DOC_DETECTIVE_CONFIG environment variable.",
+        envValidation.errors
+      );
+      process.exit(1);
+    }
+
+    log(`CLI:ENV_CONFIG:\n${JSON.stringify(envConfig, null, 2)}`, "debug", envConfig);
+  } catch (error) {
+    console.error(
+      `Error parsing DOC_DETECTIVE_CONFIG environment variable: ${error.message}`
+    );
+    process.exit(1);
+  }
+  return envConfig;
+}
+
 // Override config values based on args and validate the config
 async function setConfig({ configPath, args }) {
   if (args.config && !configPath) {
@@ -67,6 +198,13 @@ async function setConfig({ configPath, args }) {
     }
   }
 
+  // Check for DOC_DETECTIVE_CONFIG environment variable
+  const envConfig = await getConfigFromEnv();
+  if (envConfig) {
+    // Merge with file config, preferring environment variable config (use raw envConfig, not validated with defaults)
+    config = { ...config, ...envConfig };
+  }
+
   // Validate config
   const validation = validate({
     schemaKey: "config_v3",
@@ -588,6 +726,70 @@ function registerReporter(name, reporterFunction) {
 // Export the registerReporter function
 exports.registerReporter = registerReporter;
 
+async function reportResults({ apiConfig, results }) {
+  // Transform results into the required format for the API
+  // Extract contexts from the nested structure and format them
+  const contexts = [];
+
+  if (results.specs) {
+    results.specs.forEach((spec) => {
+      if (spec.tests) {
+        spec.tests.forEach((test) => {
+          if (test.contexts) {
+            test.contexts.forEach((context) => {
+              // Extract or generate contextId
+              const contextId =
+                context.contextId;
+
+              // Convert result status to lowercase (PASS -> passed, FAIL -> failed, etc.)
+              let status;
+              if (context.result === "PASS") {
+                status = "passed";
+              } else if (context.result === "FAIL") {
+                status = "failed";
+              } else if (context.result === "WARNING") {
+                status = "warning";
+              } else if (context.result === "SKIPPED") {
+                status = "skipped";
+              }
+              if (!status) {
+                log(config, "error", `Unknown context result status for context ID ${contextId}`);
+                return; 
+              }
+
+              // Build the context payload with the entire context object embedded
+              contexts.push({
+                contextId: contextId,
+                status: status,
+                result: context,
+              });
+            });
+          }
+        });
+      }
+    });
+  }
+
+  // POST to the /contexts endpoint
+  try {
+    const url = `${apiConfig.url}/contexts`;
+    const payload = { contexts };
+
+    console.log(payload);
+
+    const response = await axios.post(url, payload, {
+      headers: {
+        "x-runner-token": apiConfig.token,
+      },
+    });
+    console.log("Results reported successfully:", response.data);
+  } catch (error) {
+    console.error(
+      `Error reporting results to ${apiConfig.url}/contexts: ${error.message}`
+    );
+  }
+}
+
 async function outputResults(config = {}, outputPath, results, options = {}) {
   // Default to using both built-in reporters if none specified
   const defaultReporters = ["terminal", "json"];
@@ -653,7 +855,9 @@ async function spawnCommand(cmd, args) {
     }
   }
 
-  const runCommand = spawn(cmd, args);
+  const runCommand = spawn(cmd, args, {
+    env: process.env, // Explicitly pass environment variables
+  });
 
   // Capture stdout
   let stdout = "";

package/test/resolvedTests.test.js

@@ -0,0 +1,193 @@
+const { createServer } = require("./server");
+const path = require("path");
+const { spawnCommand } = require("../src/utils");
+const assert = require("assert").strict;
+const fs = require("fs");
+const artifactPath = path.resolve(__dirname, "./artifacts");
+const outputFile = path.resolve(`${artifactPath}/resolvedTestsResults.json`);
+
+// Create a server with custom options
+const server = createServer({
+  port: 8093,
+  staticDir: "./test/server/public",
+});
+
+// Start the server before tests
+before(async () => {
+  try {
+    await server.start();
+  } catch (error) {
+    console.error(`Failed to start test server: ${error.message}`);
+    throw error;
+  }
+});
+
+// Stop the server after tests
+after(async () => {
+  try {
+    await server.stop();
+  } catch (error) {
+    console.error(`Failed to stop test server: ${error.message}`);
+  }
+});
+
+describe("DOC_DETECTIVE_API environment variable", function () {
+  // Set indefinite timeout
+  this.timeout(0);
+
+  it("Should fetch and run resolved tests from API", async () => {
+    const apiConfig = {
+      accountId: "test-account",
+      url: "http://localhost:8093/api/resolved-tests",
+      token: "test-token-123",
+      contextIds: "test-context",
+    };
+
+    // Set environment variable
+    const originalEnv = process.env.DOC_DETECTIVE_API;
+    process.env.DOC_DETECTIVE_API = JSON.stringify(apiConfig);
+
+    try {
+      const result = await spawnCommand(
+        `node ./src/index.js -o ${outputFile}`
+      );
+
+      // Wait until the file is written
+      let waitCount = 0;
+      while (!fs.existsSync(outputFile) && waitCount < 50) {
+        await new Promise((resolve) => setTimeout(resolve, 100));
+        waitCount++;
+      }
+
+      if (fs.existsSync(outputFile)) {
+        const testResult = require(outputFile);
+        console.log(
+          "API Result summary:",
+          JSON.stringify(testResult.summary, null, 2)
+        );
+        // Clean up the require cache
+        delete require.cache[require.resolve(outputFile)];
+        fs.unlinkSync(outputFile);
+
+        // Check that tests were run
+        assert.ok(testResult.summary);
+        assert.ok(testResult.specs);
+      }
+    } finally {
+      // Restore original env
+      if (originalEnv !== undefined) {
+        process.env.DOC_DETECTIVE_API = originalEnv;
+      } else {
+        delete process.env.DOC_DETECTIVE_API;
+      }
+    }
+  });
+
+  it("Should reject API config without required fields", async () => {
+    const invalidApiConfig = {
+      accountId: "test-account",
+      // Missing url and token
+    };
+
+    const originalEnv = process.env.DOC_DETECTIVE_API;
+    process.env.DOC_DETECTIVE_API = JSON.stringify(invalidApiConfig);
+
+    try {
+      const result = await spawnCommand(
+        `node ./src/index.js -o ${outputFile}`
+      );
+
+      // Should exit with non-zero code
+      assert.notEqual(result.exitCode, 0);
+    } finally {
+      // Restore original env
+      if (originalEnv !== undefined) {
+        process.env.DOC_DETECTIVE_API = originalEnv;
+      } else {
+        delete process.env.DOC_DETECTIVE_API;
+      }
+    }
+  });
+
+  it("Should reject unauthorized API requests", async () => {
+    const apiConfigBadToken = {
+      accountId: "test-account",
+      url: "http://localhost:8093/api/resolved-tests",
+      token: "wrong-token",
+      contextIds: "test-context",
+    };
+
+    const originalEnv = process.env.DOC_DETECTIVE_API;
+    process.env.DOC_DETECTIVE_API = JSON.stringify(apiConfigBadToken);
+
+    try {
+      const result = await spawnCommand(
+        `node ./src/index.js -o ${outputFile}`
+      );
+
+      // Should exit with non-zero code due to 401 response
+      assert.notEqual(result.exitCode, 0);
+    } finally {
+      // Restore original env
+      if (originalEnv !== undefined) {
+        process.env.DOC_DETECTIVE_API = originalEnv;
+      } else {
+        delete process.env.DOC_DETECTIVE_API;
+      }
+    }
+  });
+
+  it("Should apply config overrides from DOC_DETECTIVE_CONFIG to API-fetched tests", async () => {
+    const apiConfig = {
+      accountId: "test-account",
+      url: "http://localhost:8093/api/resolved-tests",
+      token: "test-token-123",
+      contextIds: "test-context",
+    };
+
+    const configOverride = {
+      logLevel: "debug",
+    };
+
+    const originalApiEnv = process.env.DOC_DETECTIVE_API;
+    const originalConfigEnv = process.env.DOC_DETECTIVE_CONFIG;
+    process.env.DOC_DETECTIVE_API = JSON.stringify(apiConfig);
+    process.env.DOC_DETECTIVE_CONFIG = JSON.stringify(configOverride);
+
+    try {
+      await spawnCommand(
+        `node ./src/index.js -o ${outputFile}`
+      );
+
+      // Wait until the file is written
+      let waitCount = 0;
+      while (!fs.existsSync(outputFile) && waitCount < 50) {
+        await new Promise((resolve) => setTimeout(resolve, 100));
+        waitCount++;
+      }
+
+      if (fs.existsSync(outputFile)) {
+        const testResult = require(outputFile);
+        // Clean up the require cache
+        delete require.cache[require.resolve(outputFile)];
+        fs.unlinkSync(outputFile);
+
+        // Check that tests were run
+        assert.ok(testResult.summary);
+        assert.ok(testResult.specs);
+      }
+    } finally {
+      // Restore original env
+      if (originalApiEnv !== undefined) {
+        process.env.DOC_DETECTIVE_API = originalApiEnv;
+      } else {
+        delete process.env.DOC_DETECTIVE_API;
+      }
+      if (originalConfigEnv !== undefined) {
+        process.env.DOC_DETECTIVE_CONFIG = originalConfigEnv;
+      } else {
+        delete process.env.DOC_DETECTIVE_CONFIG;
+      }
+    }
+  });
+});

package/test/server/index.js

@@ -54,6 +54,52 @@ function createServer(options = {}) {
     }
   });
 
+  // Endpoint for testing DOC_DETECTIVE_API - returns resolved tests
+  app.get("/api/resolved-tests", (req, res) => {
+    try {
+      // Check for x-runner-token header
+      const token = req.headers['x-runner-token'];
+      
+      if (!token || token !== 'test-token-123') {
+        return res.status(401).json({ error: "Unauthorized" });
+      }
+
+      // Return a valid resolvedTests object
+      const resolvedTests = {
+        "resolvedTestsId": "api-resolved-tests-id",
+        "config": {
+          "logLevel": "info"
+        },
+        "specs": [
+          {
+            "specId": "api-spec",
+            "tests": [
+              {
+                "testId": "api-test",
+                "contexts": [
+                  {
+                    "contextId": "api-context",
+                    "steps": [
+                      {
+                        "stepId": "step-1",
+                        "checkLink": `http://localhost:${port}`
+                      }
+                    ]
+                  }
+                ]
+              }
+            ]
+          }
+        ]
+      };
+
+      res.json(resolvedTests);
+    } catch (error) {
+      console.error("Error processing resolved tests request:", error);
+      res.status(500).json({ error: "Internal server error" });
+    }
+  });
+
   return {
     /**
      * Start the server

package/test/utils.test.js

@@ -204,6 +204,59 @@ describe("Util tests", function () {
     // Clean up
     fs.unlinkSync(outputResultsPath);
   });
+
+  // Test environment variable config detection
+  it("Config from DOC_DETECTIVE_CONFIG environment variable is loaded and merged", async function () {
+    this.timeout(5000);
+    
+    // Save the original environment variable value
+    const originalEnvConfig = process.env.DOC_DETECTIVE_CONFIG;
+    
+    try {
+      // Test 1: Valid environment variable config without file config
+      process.env.DOC_DETECTIVE_CONFIG = JSON.stringify({
+        logLevel: "debug"
+      });
+      
+      const config1 = await setConfig({ args: setArgs(["node", "runTests.js"]) });
+      expect(config1.logLevel).to.equal("debug");
+      
+      // Test 2: Environment variable config merged with file config (env var takes precedence)
+      process.env.DOC_DETECTIVE_CONFIG = JSON.stringify({
+        logLevel: "error"
+      });
+      
+      const config2 = await setConfig({ 
+        configPath: "./test/test-config.json",
+        args: setArgs(["node", "runTests.js", "--config", "./test/test-config.json"]) 
+      });
+      // Environment variable should override file config
+      expect(config2.logLevel).to.equal("error");
+      // Check that other values from file config are preserved
+      expect(config2.telemetry.send).to.equal(false);
+      
+      // Test 3: Environment variable config with command line args (args take precedence)
+      process.env.DOC_DETECTIVE_CONFIG = JSON.stringify({
+        logLevel: "warning",
+        input: "env-input.json"
+      });
+      
+      const config3 = await setConfig({ 
+        args: setArgs(["node", "runTests.js", "--input", "cli-input.json", "--logLevel", "debug"]) 
+      });
+      // Command line args should override environment variable
+      expect(config3.logLevel).to.equal("debug");
+      expect(config3.input).to.deep.equal([path.resolve(process.cwd(), "cli-input.json")]);
+      
+    } finally {
+      // Restore the original environment variable value
+      if (originalEnvConfig !== undefined) {
+        process.env.DOC_DETECTIVE_CONFIG = originalEnvConfig;
+      } else {
+        delete process.env.DOC_DETECTIVE_CONFIG;
+      }
+    }
+  });
 });
 
 // Deeply compares two objects