@testdriverai/agent 7.8.0-canary.10

package/interfaces/junit-reporter.js

@@ -0,0 +1,290 @@
+const fs = require("fs");
+const path = require("path");
+const { events } = require("../agent/events.js");
+const builder = require("junit-report-builder");
+const stripAnsi = require("strip-ansi");
+
+/**
+ * JUnit XML Reporter for TestDriver
+ *
+ * Subscribes to TestDriver events and generates JUnit XML reports
+ * following the hierarchy:
+ * - Test Suite: folder path from "testdriver" root
+ * - Test Case: individual test file
+ *   - Contains system-out from log:* events (ANSI codes stripped)
+ *   - Contains system-err from error:* events (ANSI codes stripped)
+ *   - Properties for each step (prompt) and their status
+ */
+class JUnitReporter {
+  constructor(emitter, outputFilePath, mainTestFile) {
+    this.emitter = emitter;
+    this.outputFilePath = outputFilePath;
+    this.mainTestFile = mainTestFile;
+
+    // Current test state
+    this.currentTest = null;
+    this.currentTestCase = null;
+
+    // Single test suite for the run (based on main test file)
+    this.testSuite = null;
+
+    // Accumulate logs and errors for system-out/system-err
+    this.systemOut = [];
+    this.systemErr = [];
+
+    // Track step results for properties
+    this.stepResults = [];
+
+    // Track command results (including assertions)
+    this.commandResults = [];
+
+    // Track timing
+    this.testStartTime = null;
+
+    // Track final test result based on exit code
+    this.finalExitCode = null;
+
+    // Create test suite based on main test file
+    this.createTestSuite();
+
+    this.setupEventListeners();
+  }
+
+  createTestSuite() {
+    // Create test suite based on the main test file's folder
+    const suiteName = this.getTestSuiteName(this.mainTestFile);
+    this.testSuite = builder.testSuite().name(suiteName);
+  }
+
+  setupEventListeners() {
+    // Test lifecycle events
+    this.emitter.on(events.test.start, (data) => this.handleTestStart(data));
+
+    // Step lifecycle events
+    this.emitter.on(events.step.start, (data) => this.handleStepStart(data));
+    this.emitter.on(events.step.success, (data) =>
+      this.handleStepEnd(data, "passed"),
+    );
+    this.emitter.on(events.step.error, (data) =>
+      this.handleStepEnd(data, "failed"),
+    );
+
+    // Command lifecycle events (including assertions)
+    this.emitter.on(events.command.start, (data) =>
+      this.handleCommandStart(data),
+    );
+    this.emitter.on(events.command.success, (data) =>
+      this.handleCommandEnd(data, "passed"),
+    );
+    this.emitter.on(events.command.error, (data) =>
+      this.handleCommandEnd(data, "failed"),
+    );
+
+    // Log and error events for system-out/system-err
+    this.emitter.on("log:*", (message) => this.handleLogMessage(message));
+    this.emitter.on("error:*", (error) => this.handleErrorMessage(error));
+
+    // Handle exit to finalize report
+    this.emitter.on(events.exit, (exitCode) => this.finalizeReport(exitCode));
+  }
+
+  handleTestStart(data) {
+    const { filePath, timestamp } = data;
+    this.currentTest = { filePath, timestamp };
+    this.testStartTime = timestamp;
+
+    // Reset state for this test
+    this.systemOut = [];
+    this.systemErr = [];
+    this.stepResults = [];
+    this.commandResults = [];
+
+    // Create test case for the test file
+    const fileName = path.basename(filePath);
+    this.currentTestCase = this.testSuite
+      .testCase()
+      .className(this.getTestSuiteName(this.mainTestFile))
+      .name(fileName);
+  }
+
+  handleStepStart() {
+    // Step start is handled, the real work happens in handleStepEnd
+  }
+
+  handleStepEnd(data, status) {
+    // Only record steps if we have an active test
+    if (!this.currentTest) return;
+
+    // Extract step info from data
+    const prompt =
+      data.prompt || `Step ${data.stepIndex || this.stepResults.length}`;
+
+    // Record this step result
+    this.stepResults.push({
+      prompt: prompt,
+      status: status,
+      timestamp: data.timestamp,
+    });
+  }
+
+  handleCommandStart() {
+    // Commands are tracked but we wait for completion to record results
+  }
+
+  handleCommandEnd(data, status) {
+    // Only record commands if we have an active test
+    if (!this.currentTest) return;
+
+    // Extract command info from data
+    const command = data.command || "unknown";
+    const commandData = data.data || {};
+
+    // Record this command result
+    this.commandResults.push({
+      command: command,
+      status: status,
+      timestamp: data.timestamp,
+      expect: commandData.expect,
+      description: commandData.description || commandData.code || "",
+    });
+  }
+
+  handleLogMessage(message) {
+    // Only collect log messages if we have an active test running
+    if (!this.currentTest) return;
+
+    // Collect ALL log:* events for system-out, stripping ANSI codes
+    const cleanMessage = stripAnsi(message);
+    this.systemOut.push(`${cleanMessage}`);
+  }
+
+  handleErrorMessage(error) {
+    // Only collect error messages if we have an active test running
+    if (!this.currentTest) return;
+
+    // Collect ALL error:* events for system-err, stripping ANSI codes
+    const errorMessage =
+      typeof error === "string" ? error : JSON.stringify(error);
+    const cleanErrorMessage = stripAnsi(errorMessage);
+    this.systemErr.push(`${cleanErrorMessage}`);
+  }
+
+  getTestSuiteName(filePath) {
+    // Extract folder path from testdriver root to file
+    const relativePath = this.getRelativePathFromTestdriver(filePath);
+    const folderPath = path.dirname(relativePath);
+
+    // Create suite name from folder structure
+    return folderPath === "." ? "testdriver" : `testdriver/${folderPath}`;
+  }
+
+  getRelativePathFromTestdriver(filePath) {
+    // Find the "testdriver" directory in the path and return relative path from there
+    const testdriverIndex = filePath.toLowerCase().indexOf("/testdriver/");
+    if (testdriverIndex !== -1) {
+      return filePath.substring(testdriverIndex + "/testdriver/".length);
+    }
+
+    // Fallback: use just the filename
+    return path.basename(filePath);
+  }
+
+  finalizeReport(exitCode) {
+    // Store the exit code for determining test status
+    this.finalExitCode = exitCode;
+
+    // If we have a current test case that hasn't been finalized, finalize it now
+    if (this.currentTest && this.currentTestCase) {
+      const duration = this.testStartTime
+        ? (Date.now() - this.testStartTime) / 1000
+        : 0;
+
+      // Set test case duration
+      this.currentTestCase.time(duration);
+
+      // Add step results as properties
+      this.stepResults.forEach((step, index) => {
+        this.currentTestCase.property(
+          `step${index + 1}[${step.status}]`,
+          step.prompt,
+        );
+      });
+
+      // Add system-out and system-err
+      if (this.systemOut.length > 0) {
+        this.currentTestCase.standardOutput(this.systemOut.join("\n"));
+      }
+      if (this.systemErr.length > 0) {
+        this.currentTestCase.standardError(this.systemErr.join("\n"));
+      }
+
+      // Determine test result based on exit code (0 = success, non-zero = failure)
+      if (exitCode !== 0) {
+        // Test failed - collect failure information from steps and commands for detailed message
+        const failedSteps = this.stepResults.filter(
+          (step) => step.status === "failed",
+        );
+        const failedCommands = this.commandResults.filter(
+          (command) => command.status === "failed",
+        );
+
+        const failureMessages = [];
+        if (failedSteps.length > 0) {
+          failureMessages.push(
+            `Failed steps: ${failedSteps.map((s) => s.prompt).join(", ")}`,
+          );
+        }
+        if (failedCommands.length > 0) {
+          const failedAssertions = failedCommands.filter(
+            (c) => c.command === "assert",
+          );
+          const otherFailedCommands = failedCommands.filter(
+            (c) => c.command !== "assert",
+          );
+
+          if (failedAssertions.length > 0) {
+            failureMessages.push(
+              `Failed assertions: ${failedAssertions.map((c) => c.expect || "assertion").join(", ")}`,
+            );
+          }
+          if (otherFailedCommands.length > 0) {
+            failureMessages.push(
+              `Failed commands: ${otherFailedCommands.map((c) => c.command).join(", ")}`,
+            );
+          }
+        }
+
+        const failureMessage =
+          failureMessages.length > 0
+            ? failureMessages.join("; ")
+            : `Test failed with exit code ${exitCode}`;
+        this.currentTestCase.failure(failureMessage);
+      }
+    }
+
+    try {
+      // Generate the XML report
+      const xmlContent = builder.build();
+
+      // Ensure output directory exists
+      const outputDir = path.dirname(this.outputFilePath);
+      if (!fs.existsSync(outputDir)) {
+        fs.mkdirSync(outputDir, { recursive: true });
+      }
+
+      // Write to file
+      fs.writeFileSync(this.outputFilePath, xmlContent);
+    } catch (error) {
+      console.error("[JUnit Reporter] Failed to write JUnit report:", error);
+    }
+  }
+}
+
+/**
+ * Factory function to create and initialize JUnit reporter
+ */
+function createJUnitReporter(emitter, outputFilePath, mainTestFile) {
+  return new JUnitReporter(emitter, outputFilePath, mainTestFile);
+}
+
+module.exports = { JUnitReporter, createJUnitReporter };

package/interfaces/logger.js

@@ -0,0 +1,388 @@
+// central logger for the bot
+const winston = require("winston");
+const chalk = require("chalk");
+const theme = require("../agent/lib/theme");
+const Transport = require("winston-transport");
+const { events } = require("../agent/events");
+class CustomTransport extends Transport {
+  constructor(opts) {
+    super(opts);
+    this.name = opts.name || "customTransport";
+    this.level = opts.level || "info";
+    this.logStore = opts.logStore || []; // You could connect to a DB or API here
+    this.sandbox = null;
+  }
+
+  log(info, callback) {
+    try {
+      const { message } = info;
+
+      if (typeof message === "object") {
+        console.log(chalk.cyan("protecting against base64 error"));
+        console.log(message);
+        callback();
+        return;
+      }
+
+    } catch (e) {
+      console.error("Error in CustomTransport log method:", e);
+    }
+
+    callback();
+  }
+}
+
+// responsible for rendering ai markdown output
+const { marked } = require("marked");
+const { markedTerminal } = require("marked-terminal");
+const { censorSensitiveDataDeep } = require("../agent/lib/censorship");
+
+const { printf } = winston.format;
+
+const logFormat = printf(({ message }) => {
+  return `${message}`;
+});
+
+const logger = winston.createLogger({
+  format: winston.format.combine(
+    winston.format.splat(),
+    winston.format((info) => {
+      info.message = censorSensitiveDataDeep(info.message);
+      return info;
+    })(),
+    logFormat,
+  ),
+  transports: [
+    new winston.transports.Console(),
+    new CustomTransport({
+      level: "info",
+      logStore: [], // You could pass an external store or leave it empty
+    }),
+  ],
+});
+
+const plain = (codePart) => codePart;
+
+// marked is a markdown parser
+// markedTerminal allows us to render markdown in CLI
+marked.use(
+  markedTerminal(
+    {
+      width: 58, // 58 is the width of the terminal output on a 16" macbook pro
+      reflowText: true,
+      tab: 2,
+    },
+    {
+      theme: {
+        /**
+         * keyword in a regular Algol-style language
+         */
+        keyword: theme.blue,
+
+        /**
+         * built-in or library object (constant, class, function)
+         */
+        built_in: theme.cyan,
+
+        /**
+         * user-defined type in a language with first-class syntactically significant types, like
+         * Haskell
+         */
+        type: theme.cyan,
+
+        /**
+         * special identifier for a built-in value ("true", "false", "null")
+         */
+        literal: theme.blue,
+
+        /**
+         * number, including units and modifiers, if any.
+         */
+        number: theme.green,
+
+        /**
+         * literal regular expression
+         */
+        regexp: theme.red,
+
+        /**
+         * literal string, character
+         */
+        string: theme.green,
+
+        /**
+         * parsed section inside a literal string
+         */
+        subst: plain,
+
+        /**
+         * symbolic constant, interned string, goto label
+         */
+        symbol: plain,
+
+        /**
+         * class or class-level declaration (interfaces, traits, modules, etc)
+         */
+        class: theme.blue,
+
+        /**
+         * function or method declaration
+         */
+        function: theme.yellow,
+
+        /**
+         * name of a class or a function at the place of declaration
+         */
+        title: plain,
+
+        /**
+         * block of function arguments (parameters) at the place of declaration
+         */
+        params: plain,
+
+        /**
+         * comment
+         */
+        comment: theme.green,
+
+        /**
+         * documentation markup within comments
+         */
+        doctag: theme.green,
+
+        /**
+         * flags, modifiers, annotations, processing instructions, preprocessor directive, etc
+         */
+        meta: theme.grey,
+
+        /**
+         * keyword or built-in within meta construct
+         */
+        "meta-keyword": plain,
+
+        /**
+         * string within meta construct
+         */
+        "meta-string": plain,
+
+        /**
+         * heading of a section in a config file, heading in text markup
+         */
+        section: plain,
+
+        /**
+         * XML/HTML tag
+         */
+        tag: theme.grey,
+
+        /**
+         * name of an XML tag, the first word in an s-expression
+         */
+        name: theme.blue,
+
+        /**
+         * s-expression name from the language standard library
+         */
+        "builtin-name": plain,
+
+        /**
+         * name of an attribute with no language defined semantics (keys in JSON, setting names in
+         * .ini), also sub-attribute within another highlighted object, like XML tag
+         */
+        attr: theme.cyan,
+
+        /**
+         * name of an attribute followed by a structured value part, like CSS properties
+         */
+        attribute: plain,
+
+        /**
+         * variable in a config or a template file, environment var expansion in a script
+         */
+        variable: plain,
+
+        /**
+         * list item bullet in text markup
+         */
+        bullet: plain,
+
+        /**
+         * code block in text markup
+         */
+        code: plain,
+
+        /**
+         * emphasis in text markup
+         */
+        emphasis: chalk.italic,
+
+        /**
+         * strong emphasis in text markup
+         */
+        strong: chalk.bold,
+
+        /**
+         * mathematical formula in text markup
+         */
+        formula: plain,
+
+        /**
+         * hyperlink in text markup
+         */
+        link: chalk.underline,
+
+        /**
+         * quotation in text markup
+         */
+        quote: plain,
+
+        /**
+         * tag selector in CSS
+         */
+        "selector-tag": plain,
+
+        /**
+         * #id selector in CSS
+         */
+        "selector-id": plain,
+
+        /**
+         * .class selector in CSS
+         */
+        "selector-class": plain,
+
+        /**
+         * [attr] selector in CSS
+         */
+        "selector-attr": plain,
+
+        /**
+         * :pseudo selector in CSS
+         */
+        "selector-pseudo": plain,
+
+        /**
+         * tag of a template language
+         */
+        "template-tag": plain,
+
+        /**
+         * variable in a template language
+         */
+        "template-variable": plain,
+
+        /**
+         * added or changed line in a diff
+         */
+        addition: theme.green,
+
+        /**
+         * deleted line in a diff
+         */
+        deletion: theme.red,
+
+        /**
+         * things not matched by any token
+         */
+        default: plain,
+      },
+    },
+  ),
+);
+
+const createMarkdownLogger = (emitter) => {
+  // Indent prefix for streaming AI thoughts - makes it visually distinct and scoped
+  const streamIndent = "";
+
+  const markedParsePartial = (markdown, start = 0, end = 0) => {
+    let result = markdown.trimEnd().split("\n").slice(start, end);
+    if (end <= 0) {
+      end = result.length + end;
+    }
+    result = result.join("\n");
+
+    // Use streamIndent for streaming output to make it visually scoped
+    return marked.parse(result).replace(/^/gm, streamIndent).trimEnd();
+  };
+
+  // Event-based markdown streaming with buffering
+  const activeStreams = new Map();
+
+  // Handle streaming markdown events with proper buffering
+  emitter.on(events.log.markdown.start, (streamId) => {
+    activeStreams.set(streamId, {
+      buffer: "",
+      lastOutput: "",
+    });
+  });
+
+  emitter.on(events.log.markdown.chunk, (streamId, chunk) => {
+    if (!activeStreams.has(streamId)) {
+      return;
+    }
+
+    const stream = activeStreams.get(streamId);
+
+    if (typeof chunk !== "string") {
+      return;
+    }
+
+    const previousConsoleOutput = markedParsePartial(stream.buffer, 0, -1);
+
+    stream.buffer += chunk;
+
+    const consoleOutput = markedParsePartial(stream.buffer, 0, -1);
+
+    let diff = consoleOutput.replace(previousConsoleOutput, "");
+    if (diff) {
+      diff = censorSensitiveDataDeep(diff);
+      process.stdout.write(diff);
+    }
+  });
+
+  emitter.on(events.log.markdown.end, (streamId) => {
+    if (!activeStreams.has(streamId)) {
+      return;
+    }
+
+    const stream = activeStreams.get(streamId);
+
+    const previousConsoleOutput = markedParsePartial(stream.buffer, 0, -1);
+    const consoleOutput = markedParsePartial(stream.buffer, 0, Infinity);
+    let diff = consoleOutput.replace(previousConsoleOutput, "");
+
+    if (diff) {
+      diff = censorSensitiveDataDeep(diff);
+      process.stdout.write(diff);
+    }
+    // Use console.log for the final newlines so it gets captured by vitest
+    console.log("");
+
+    // Clean up the stream
+    activeStreams.delete(streamId);
+  });
+
+  // Handle static markdown logging (complete markdown blocks)
+  emitter.on(events.log.markdown.static, (markdown) => {
+    if (typeof markdown !== "string") {
+      logger.error("Static markdown requires a string");
+      logger.error(markdown);
+      return;
+    }
+
+    let consoleOutput = marked.parse(markdown);
+
+    // strip newlines at end of consoleOutput
+    consoleOutput = consoleOutput.replace(/\n$/, "");
+    consoleOutput = consoleOutput.replace(/^/gm, spaceChar);
+
+    logger.info(consoleOutput);
+  });
+};
+
+const spaceChar = "   ";
+
+module.exports = {
+  logger,
+  createMarkdownLogger,
+};