doc-detective 3.4.0-dev.2 → 3.4.0-dev.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "doc-detective",
-  "version": "3.4.0-dev.2",
+  "version": "3.4.0-dev.4",
   "description": "Treat doc content as testable assertions to validate doc accuracy and product UX.",
   "bin": {
     "doc-detective": "src/index.js"
@@ -33,8 +33,9 @@
   "homepage": "https://github.com/doc-detective/doc-detective#readme",
   "dependencies": {
     "@ffmpeg-installer/ffmpeg": "^1.1.0",
-    "doc-detective-common": "^3.4.0",
-    "doc-detective-core": "^3.4.0",
+    "axios": "^1.12.2",
+    "doc-detective-common": "3.4.0-dev.1",
+    "doc-detective-core": "3.4.0-dev.2",
     "yargs": "^17.7.2"
   },
   "devDependencies": {

package/src/index.js

@@ -1,7 +1,16 @@
 #!/usr/bin/env node
 
-const { runTests, runCoverage } = require("doc-detective-core");
-const { setArgs, setConfig, outputResults, setMeta, getVersionData, log } = require("./utils");
+const { runTests } = require("doc-detective-core");
+const {
+  setArgs,
+  setConfig,
+  outputResults,
+  setMeta,
+  getVersionData,
+  log,
+  getResolvedTestsFromEnv,
+  reportResults,
+} = require("./utils");
 const { argv } = require("node:process");
 const path = require("path");
 const fs = require("fs");
@@ -36,16 +45,28 @@ async function main(argv) {
   // Set config
   const config = await setConfig({ configPath: configPath, args: argv });
 
-  if (config.logLevel === "debug") {
-    console.log(`CLI:VERSION INFO:\n${JSON.stringify(getVersionData(), null, 2)}`);
-    console.log(`CLI:CONFIG:\n${JSON.stringify(config, null, 2)}`);
-  }
+  log(
+    `CLI:VERSION INFO:\n${JSON.stringify(getVersionData(), null, 2)}`,
+    "debug",
+    config
+  );
+  log(`CLI:CONFIG:\n${JSON.stringify(config, null, 2)}`, "debug", config);
+
+  // Check for DOC_DETECTIVE_API environment variable
+  let api = await getResolvedTestsFromEnv(config);
+  let resolvedTests = api?.resolvedTests || null;
+  let apiConfig = api?.apiConfig || null;
 
   // Run tests
   const output = config.output;
-  const results = await runTests(config);
+  const results = resolvedTests
+    ? await runTests(config, { resolvedTests })
+    : await runTests(config);
 
-  // Output results
-  await outputResults(config, output, results, { command: "runTests" });
-
-}
+  if (apiConfig) {
+    await reportResults({ apiConfig, results });
+  } else {
+    // Output results
+    await outputResults(config, output, results, { command: "runTests" });
+  }
+}

package/src/utils.js

@@ -5,6 +5,7 @@ const path = require("path");
 const fs = require("fs");
 const { spawn } = require("child_process");
 const os = require("os");
+const axios = require("axios");
 
 exports.setArgs = setArgs;
 exports.setConfig = setConfig;
@@ -12,6 +13,26 @@ exports.outputResults = outputResults;
 exports.spawnCommand = spawnCommand;
 exports.setMeta = setMeta;
 exports.getVersionData = getVersionData;
+exports.log = log;
+exports.getResolvedTestsFromEnv = getResolvedTestsFromEnv;
+exports.reportResults = reportResults;
+
+// Log function that respects logLevel
+function log(message, level = "info", config = {}) {
+  const logLevels = ["silent", "error", "warning", "info", "debug"];
+  const currentLevel = config.logLevel || "info";
+  const currentLevelIndex = logLevels.indexOf(currentLevel);
+  const messageLevelIndex = logLevels.indexOf(level);
+
+  // Only log if the message level is at or above the current log level
+  if (currentLevelIndex >= messageLevelIndex && messageLevelIndex > 0) {
+    if (level === "error") {
+      console.error(message);
+    } else {
+      console.log(message);
+    }
+  }
+}
 
 // Define args
 function setArgs(args) {
@@ -50,6 +71,116 @@ function setArgs(args) {
   return argv;
 }
 
+// Get resolved tests from environment variable, if set
+async function getResolvedTestsFromEnv(config = {}) {
+  if (!process.env.DOC_DETECTIVE_API) {
+    return null;
+  }
+
+  let resolvedTests = null;
+  let apiConfig = null;
+  try {
+    // Parse the environment variable as JSON
+    apiConfig = JSON.parse(process.env.DOC_DETECTIVE_API);
+
+    // Validate the structure: { accountId, url, token, contextIds }
+    if (!apiConfig.accountId || !apiConfig.url || !apiConfig.token || !apiConfig.contextIds) {
+      log(
+        "Invalid DOC_DETECTIVE_API: must contain 'accountId', 'url', 'token', and 'contextIds' properties",
+        "error",
+        config
+      );
+      process.exit(1);
+    }
+
+    log(`CLI:Fetching resolved tests from ${apiConfig.url}`, "debug", config);
+
+    // Make GET request to the specified URL with token in header
+    const response = await axios.get(apiConfig.url, {
+      headers: {
+        "x-runner-token": apiConfig.token,
+      },
+    });
+
+    // The response is the resolvedTests
+    resolvedTests = response.data;
+
+    // Validate against resolvedTests_v3 schema
+    const validation = validate({
+      schemaKey: "resolvedTests_v3",
+      object: resolvedTests,
+    });
+
+    if (!validation.valid) {
+      log(
+        "Invalid resolvedTests from API response. " + validation.errors,
+        "error",
+        config
+      );
+      process.exit(1);
+    }
+
+    // Get config from environment variable for merging
+    const envConfig = await getConfigFromEnv();
+    if (envConfig) {
+      // Apply config overrides to resolvedTests.config
+      if (resolvedTests.config) {
+        resolvedTests.config = { ...resolvedTests.config, ...envConfig };
+      } else {
+        resolvedTests.config = envConfig;
+      }
+    }
+
+    log(
+      `CLI:RESOLVED_TESTS:\n${JSON.stringify(resolvedTests, null, 2)}`,
+      "debug",
+      config
+    );
+  } catch (error) {
+    log(
+      `Error fetching resolved tests from DOC_DETECTIVE_API: ${error.message}`,
+      "error",
+      config
+    );
+    process.exit(1);
+  }
+  return { apiConfig, resolvedTests };
+}
+
+async function getConfigFromEnv() {
+  if (!process.env.DOC_DETECTIVE_CONFIG) {
+    return null;
+  }
+
+  let envConfig = null;
+  try {
+    // Parse the environment variable as JSON
+    envConfig = JSON.parse(process.env.DOC_DETECTIVE_CONFIG);
+
+    // Validate the environment variable config
+    const envValidation = validate({
+      schemaKey: "config_v3",
+      object: envConfig,
+    });
+
+    if (!envValidation.valid) {
+      console.error(
+        "Invalid config from DOC_DETECTIVE_CONFIG environment variable.",
+        envValidation.errors
+      );
+      process.exit(1);
+    }
+
+    log(`CLI:ENV_CONFIG:\n${JSON.stringify(envConfig, null, 2)}`, "debug", envConfig);
+  } catch (error) {
+    console.error(
+      `Error parsing DOC_DETECTIVE_CONFIG environment variable: ${error.message}`
+    );
+    process.exit(1);
+  }
+  return envConfig;
+}
+
 // Override config values based on args and validate the config
 async function setConfig({ configPath, args }) {
   if (args.config && !configPath) {
@@ -68,28 +199,10 @@ async function setConfig({ configPath, args }) {
   }
 
   // Check for DOC_DETECTIVE_CONFIG environment variable
-  if (process.env.DOC_DETECTIVE_CONFIG) {
-    try {
-      // Parse the environment variable as JSON
-      const envConfig = JSON.parse(process.env.DOC_DETECTIVE_CONFIG);
-      
-      // Validate the environment variable config
-      const envValidation = validate({
-        schemaKey: "config_v3",
-        object: envConfig,
-      });
-      
-      if (!envValidation.valid) {
-        console.error("Invalid config from DOC_DETECTIVE_CONFIG environment variable.", envValidation.errors);
-        process.exit(1);
-      }
-      
-      // Merge with file config, preferring environment variable config (use raw envConfig, not validated with defaults)
-      config = { ...config, ...envConfig };
-    } catch (error) {
-      console.error(`Error parsing DOC_DETECTIVE_CONFIG environment variable: ${error.message}`);
-      process.exit(1);
-    }
+  const envConfig = await getConfigFromEnv();
+  if (envConfig) {
+    // Merge with file config, preferring environment variable config (use raw envConfig, not validated with defaults)
+    config = { ...config, ...envConfig };
   }
 
   // Validate config
@@ -613,6 +726,70 @@ function registerReporter(name, reporterFunction) {
 // Export the registerReporter function
 exports.registerReporter = registerReporter;
 
+async function reportResults({ apiConfig, results }) {
+  // Transform results into the required format for the API
+  // Extract contexts from the nested structure and format them
+  const contexts = [];
+
+  if (results.specs) {
+    results.specs.forEach((spec) => {
+      if (spec.tests) {
+        spec.tests.forEach((test) => {
+          if (test.contexts) {
+            test.contexts.forEach((context) => {
+              // Extract or generate contextId
+              const contextId =
+                context.contextId ||
+                context.id ||
+                `${spec.specId}-${test.testId}-${
+                  context.platform || "unknown"
+                }`;
+
+              // Convert result status to lowercase (PASS -> passed, FAIL -> failed, etc.)
+              let status;
+              if (context.result === "PASS") {
+                status = "passed";
+              } else if (context.result === "FAIL") {
+                status = "failed";
+              } else if (context.result === "WARNING") {
+                status = "warning";
+              } else if (context.result === "SKIPPED") {
+                status = "skipped";
+              } else {
+                status = "unknown";
+              }
+
+              // Build the context payload with the entire context object embedded
+              contexts.push({
+                contextId: contextId,
+                status: status,
+                result: context,
+              });
+            });
+          }
+        });
+      }
+    });
+  }
+
+  // POST to the /contexts endpoint
+  try {
+    const url = `${apiConfig.url}/contexts`;
+    const payload = { contexts };
+
+    const response = await axios.post(url, payload, {
+      headers: {
+        "x-runner-token": apiConfig.token,
+      },
+    });
+    console.log("Results reported successfully:", response.data);
+  } catch (error) {
+    console.error(
+      `Error reporting results to ${apiConfig.url}/contexts: ${error.message}`
+    );
+  }
+}
+
 async function outputResults(config = {}, outputPath, results, options = {}) {
   // Default to using both built-in reporters if none specified
   const defaultReporters = ["terminal", "json"];
@@ -678,7 +855,9 @@ async function spawnCommand(cmd, args) {
     }
   }
 
-  const runCommand = spawn(cmd, args);
+  const runCommand = spawn(cmd, args, {
+    env: process.env, // Explicitly pass environment variables
+  });
 
   // Capture stdout
   let stdout = "";

package/test/resolvedTests.test.js

@@ -0,0 +1,193 @@
+const { createServer } = require("./server");
+const path = require("path");
+const { spawnCommand } = require("../src/utils");
+const assert = require("assert").strict;
+const fs = require("fs");
+const artifactPath = path.resolve(__dirname, "./artifacts");
+const outputFile = path.resolve(`${artifactPath}/resolvedTestsResults.json`);
+
+// Create a server with custom options
+const server = createServer({
+  port: 8093,
+  staticDir: "./test/server/public",
+});
+
+// Start the server before tests
+before(async () => {
+  try {
+    await server.start();
+  } catch (error) {
+    console.error(`Failed to start test server: ${error.message}`);
+    throw error;
+  }
+});
+
+// Stop the server after tests
+after(async () => {
+  try {
+    await server.stop();
+  } catch (error) {
+    console.error(`Failed to stop test server: ${error.message}`);
+  }
+});
+
+describe("DOC_DETECTIVE_API environment variable", function () {
+  // Set indefinite timeout
+  this.timeout(0);
+
+  it("Should fetch and run resolved tests from API", async () => {
+    const apiConfig = {
+      accountId: "test-account",
+      url: "http://localhost:8093/api/resolved-tests",
+      token: "test-token-123",
+      contextIds: "test-context",
+    };
+
+    // Set environment variable
+    const originalEnv = process.env.DOC_DETECTIVE_API;
+    process.env.DOC_DETECTIVE_API = JSON.stringify(apiConfig);
+
+    try {
+      const result = await spawnCommand(
+        `node ./src/index.js -o ${outputFile}`
+      );
+
+      // Wait until the file is written
+      let waitCount = 0;
+      while (!fs.existsSync(outputFile) && waitCount < 50) {
+        await new Promise((resolve) => setTimeout(resolve, 100));
+        waitCount++;
+      }
+
+      if (fs.existsSync(outputFile)) {
+        const testResult = require(outputFile);
+        console.log(
+          "API Result summary:",
+          JSON.stringify(testResult.summary, null, 2)
+        );
+        // Clean up the require cache
+        delete require.cache[require.resolve(outputFile)];
+        fs.unlinkSync(outputFile);
+
+        // Check that tests were run
+        assert.ok(testResult.summary);
+        assert.ok(testResult.specs);
+      }
+    } finally {
+      // Restore original env
+      if (originalEnv !== undefined) {
+        process.env.DOC_DETECTIVE_API = originalEnv;
+      } else {
+        delete process.env.DOC_DETECTIVE_API;
+      }
+    }
+  });
+
+  it("Should reject API config without required fields", async () => {
+    const invalidApiConfig = {
+      accountId: "test-account",
+      // Missing url and token
+    };
+
+    const originalEnv = process.env.DOC_DETECTIVE_API;
+    process.env.DOC_DETECTIVE_API = JSON.stringify(invalidApiConfig);
+
+    try {
+      const result = await spawnCommand(
+        `node ./src/index.js -o ${outputFile}`
+      );
+
+      // Should exit with non-zero code
+      assert.notEqual(result.exitCode, 0);
+    } finally {
+      // Restore original env
+      if (originalEnv !== undefined) {
+        process.env.DOC_DETECTIVE_API = originalEnv;
+      } else {
+        delete process.env.DOC_DETECTIVE_API;
+      }
+    }
+  });
+
+  it("Should reject unauthorized API requests", async () => {
+    const apiConfigBadToken = {
+      accountId: "test-account",
+      url: "http://localhost:8093/api/resolved-tests",
+      token: "wrong-token",
+      contextIds: "test-context",
+    };
+
+    const originalEnv = process.env.DOC_DETECTIVE_API;
+    process.env.DOC_DETECTIVE_API = JSON.stringify(apiConfigBadToken);
+
+    try {
+      const result = await spawnCommand(
+        `node ./src/index.js -o ${outputFile}`
+      );
+
+      // Should exit with non-zero code due to 401 response
+      assert.notEqual(result.exitCode, 0);
+    } finally {
+      // Restore original env
+      if (originalEnv !== undefined) {
+        process.env.DOC_DETECTIVE_API = originalEnv;
+      } else {
+        delete process.env.DOC_DETECTIVE_API;
+      }
+    }
+  });
+
+  it("Should apply config overrides from DOC_DETECTIVE_CONFIG to API-fetched tests", async () => {
+    const apiConfig = {
+      accountId: "test-account",
+      url: "http://localhost:8093/api/resolved-tests",
+      token: "test-token-123",
+      contextIds: "test-context",
+    };
+
+    const configOverride = {
+      logLevel: "debug",
+    };
+
+    const originalApiEnv = process.env.DOC_DETECTIVE_API;
+    const originalConfigEnv = process.env.DOC_DETECTIVE_CONFIG;
+    process.env.DOC_DETECTIVE_API = JSON.stringify(apiConfig);
+    process.env.DOC_DETECTIVE_CONFIG = JSON.stringify(configOverride);
+
+    try {
+      await spawnCommand(
+        `node ./src/index.js -o ${outputFile}`
+      );
+
+      // Wait until the file is written
+      let waitCount = 0;
+      while (!fs.existsSync(outputFile) && waitCount < 50) {
+        await new Promise((resolve) => setTimeout(resolve, 100));
+        waitCount++;
+      }
+
+      if (fs.existsSync(outputFile)) {
+        const testResult = require(outputFile);
+        // Clean up the require cache
+        delete require.cache[require.resolve(outputFile)];
+        fs.unlinkSync(outputFile);
+
+        // Check that tests were run
+        assert.ok(testResult.summary);
+        assert.ok(testResult.specs);
+      }
+    } finally {
+      // Restore original env
+      if (originalApiEnv !== undefined) {
+        process.env.DOC_DETECTIVE_API = originalApiEnv;
+      } else {
+        delete process.env.DOC_DETECTIVE_API;
+      }
+      if (originalConfigEnv !== undefined) {
+        process.env.DOC_DETECTIVE_CONFIG = originalConfigEnv;
+      } else {
+        delete process.env.DOC_DETECTIVE_CONFIG;
+      }
+    }
+  });
+});

package/test/server/index.js

@@ -54,6 +54,52 @@ function createServer(options = {}) {
     }
   });
 
+  // Endpoint for testing DOC_DETECTIVE_API - returns resolved tests
+  app.get("/api/resolved-tests", (req, res) => {
+    try {
+      // Check for x-runner-token header
+      const token = req.headers['x-runner-token'];
+      
+      if (!token || token !== 'test-token-123') {
+        return res.status(401).json({ error: "Unauthorized" });
+      }
+
+      // Return a valid resolvedTests object
+      const resolvedTests = {
+        "resolvedTestsId": "api-resolved-tests-id",
+        "config": {
+          "logLevel": "info"
+        },
+        "specs": [
+          {
+            "specId": "api-spec",
+            "tests": [
+              {
+                "testId": "api-test",
+                "contexts": [
+                  {
+                    "contextId": "api-context",
+                    "steps": [
+                      {
+                        "stepId": "step-1",
+                        "checkLink": `http://localhost:${port}`
+                      }
+                    ]
+                  }
+                ]
+              }
+            ]
+          }
+        ]
+      };
+
+      res.json(resolvedTests);
+    } catch (error) {
+      console.error("Error processing resolved tests request:", error);
+      res.status(500).json({ error: "Internal server error" });
+    }
+  });
+
   return {
     /**
      * Start the server