@testdriverai/agent 7.8.0-canary.10

package/agent/lib/system.js

@@ -0,0 +1,204 @@
+// utilities for getting information about the system
+const fs = require("fs");
+const os = require("os");
+const path = require("path");
+const { randomUUID } = require("crypto");
+const Jimp = require("jimp");
+const axios = require("axios");
+const { withRetry } = require("./sdk");
+const { events } = require("../events.js");
+
+const createSystem = (emitter, sandbox, config) => {
+
+  // Download a screenshot from S3 when the runner returns an s3Key
+  // (screenshots exceed Ably's 64KB message limit)
+  const downloadFromS3 = async (s3Key) => {
+    const apiRoot = config["TD_API_ROOT"] || sandbox.apiRoot;
+    const apiKey = sandbox.apiKey;
+
+    // Step 1: Get presigned download URL from API (with retry)
+    const response = await withRetry(
+      () => axios({
+        method: "post",
+        url: apiRoot + "/api/v7/runner/download-url",
+        data: { apiKey, s3Key },
+        headers: { "Content-Type": "application/json" },
+        timeout: 15000,
+      }),
+      {
+        retryConfig: { maxRetries: 3, baseDelayMs: 1000 },
+      },
+    );
+
+    const downloadUrl = response.data.downloadUrl;
+    if (!downloadUrl) {
+      throw new Error("No downloadUrl in response: " + JSON.stringify(response.data));
+    }
+
+    // Step 2: Download the image from S3
+    const imageResponse = await axios({
+      method: "get",
+      url: downloadUrl,
+      responseType: "arraybuffer",
+      timeout: 30000,
+    });
+
+    return Buffer.from(imageResponse.data).toString("base64");
+  };
+
+  const screenshot = async (options) => {
+    let response = await sandbox.send({
+      type: "system.screenshot",
+    });
+
+    let base64;
+
+    // Runner returns { s3Key } for Ably (screenshots too large for 64KB limit)
+    // Runner returns { base64 } for direct/local connections
+    if (response.s3Key) {
+      base64 = await downloadFromS3(response.s3Key);
+    } else {
+      base64 = response.base64;
+    }
+
+    if (!base64) {
+      throw new Error("Failed to take screenshot: sandbox returned empty data");
+    }
+    
+    let image = Buffer.from(base64, "base64");
+    
+    // Verify we got actual image data (PNG header starts with these bytes)
+    if (image.length < 100) {
+      throw new Error(`Failed to take screenshot: received only ${image.length} bytes`);
+    }
+    
+    fs.writeFileSync(options.filename, image);
+    return { filename: options.filename };
+  };
+
+  let primaryDisplay = null;
+
+  const getSystemInformationOsInfo = async () => {
+    return {
+      os: "windows",
+    };
+  };
+
+  let countImages = 0;
+  const tmpFilename = () => {
+    countImages = countImages + 1;
+    return path.join(os.tmpdir(), `td-${Date.now()}-${randomUUID().slice(0, 8)}-${countImages}.png`);
+  };
+
+  const captureAndResize = async (scale = 1, silent = false, mouse = false) => {
+    try {
+      if (!silent) {
+        emitter.emit(events.screenCapture.start, {
+          scale,
+          silent,
+          display: primaryDisplay,
+        });
+      }
+
+      let step1 = tmpFilename();
+      let step2 = tmpFilename();
+
+      await screenshot({ filename: step1, format: "png" });
+
+      // Load the screenshot image with Jimp
+      let image = await Jimp.read(step1);
+      
+      // Validate the image was loaded correctly (not a 1x1 or tiny placeholder)
+      if (image.getWidth() < 10 || image.getHeight() < 10) {
+        throw new Error(`Screenshot appears corrupted: got ${image.getWidth()}x${image.getHeight()} pixels`);
+      }
+
+      // Resize the image
+      image.resize(
+        Math.floor(config.TD_RESOLUTION[0] * scale),
+        Math.floor(config.TD_RESOLUTION[1] * scale),
+      );
+
+      if (mouse) {
+        // Only get mouse position when needed to avoid unnecessary websocket calls
+        const cursorPath = path.join(__dirname, "resources", "cursor-2.png");
+        const mousePos = await getMousePosition();
+        
+        // Load and composite the mouse cursor image if we have valid coordinates
+        if (mousePos && typeof mousePos.x === "number" && typeof mousePos.y === "number") {
+          const cursorImage = await Jimp.read(cursorPath);
+          image.composite(cursorImage, mousePos.x, mousePos.y);
+        }
+      }
+
+      await image.writeAsync(step2);
+
+      emitter.emit(events.screenCapture.end, {
+        scale,
+        silent,
+        display: primaryDisplay,
+      });
+
+      return step2;
+    } catch (error) {
+      emitter.emit(events.screenCapture.error, {
+        error,
+        scale,
+        silent,
+        display: primaryDisplay,
+      });
+      throw error;
+    }
+  };
+
+  // our handy screenshot function
+  const captureScreenBase64 = async (
+    scale = 1,
+    silent = false,
+    mouse = false,
+  ) => {
+    let step2 = await captureAndResize(scale, silent, mouse);
+    return fs.readFileSync(step2, "base64");
+  };
+
+  const captureScreenPNG = async (scale = 1, silent = false, mouse = false) => {
+    return await captureAndResize(scale, silent, mouse);
+  };
+
+  const platform = () => {
+    return "windows";
+  };
+
+  // this is the focused window
+  const activeWin = async () => {
+    // Get Mouse Position from command line
+    let result = await sandbox.send({
+      type: "system.get-active-window",
+    });
+
+    return result.out;
+  };
+
+  const getMousePosition = async () => {
+    // Get Mouse Position from command line
+    let result = await sandbox.send({
+      type: "system.get-mouse-position",
+    });
+
+    return result.out;
+  };
+
+  return {
+    captureScreenBase64,
+    captureScreenPNG,
+    getMousePosition,
+    primaryDisplay,
+    activeWin,
+    platform,
+    getSystemInformationOsInfo,
+  };
+};
+
+module.exports = {
+  createSystem,
+};

package/agent/lib/theme.js

@@ -0,0 +1,14 @@
+const chalk = require("chalk");
+chalk.level = 3; // Force color outputs when using chalk, was added for VS Code support
+
+module.exports = {
+  green: chalk.hex("#b3d334"),
+  blue: chalk.hex("#34a3d3"),
+  red: chalk.hex("#d33434"),
+  yellow: chalk.hex("#ced334"),
+  magenta: chalk.hex("#d334b3"),
+  cyan: chalk.hex("#34d3d3"),
+  dim: chalk.dim,
+  gray: chalk.gray,
+  white: chalk.white,
+};

package/agent/lib/valid-version.js

@@ -0,0 +1,21 @@
+const semver = require("semver");
+const packageJson = require("../../package.json");
+
+// Function to check if the new version's minor version is >= current version's minor version
+module.exports = (inputVersion) => {
+  const currentParsed = semver.parse(packageJson.version);
+  const inputParsed = semver.parse(inputVersion.replace("v", ""));
+
+  if (!currentParsed || !inputParsed) {
+    throw new Error("Invalid version format");
+  }
+
+  // Compare major and minor versions
+  if (
+    inputParsed.major === currentParsed.major &&
+    inputParsed.minor <= currentParsed.minor
+  ) {
+    return true;
+  }
+  return false;
+};

package/agent/lib/validation.js

@@ -0,0 +1,169 @@
+const types = () =>
+  import("arktype").then(({ scope }) =>
+    scope({
+      // - command: press-keys # Types a keyboard combination. Repeat the command to repeat the keypress.
+      //   keys: [command, space]
+      PressKeysCommand: {
+        command: '"press-keys"',
+        keys: "string[]",
+      },
+      // - command: hover-text # Hovers text matching the \`description\`. The text must be visible. This will also handle clicking or right clicking on the text if required.
+      //   text: Sign Up # The text to find on screen. The longer and more unique the better.
+      //   description: registration in the top right of the header # Describe the element so it can be identified in the future. Do not include the text itself here. Make sure to include the unique traits of this element.
+      //   action: click # What to do when text is found. Available actions are: click, right-click, double-click, hover
+      //   method: ai # Optional. Only try this if text match is not working.
+      HoverTextCommand: {
+        command: '"hover-text"',
+        text: "string",
+        description: "string",
+        action: '"click" | "right-click" | "double-click" | "hover"',
+        "method?": '"ai"',
+      },
+      // - command: type # Types the string into the active application. You must focus the correct field before typing.
+      //   text: Hello World
+      TypeCommand: {
+        command: '"type"',
+        text: "string",
+      },
+      // - command: wait # Waits a number of miliseconds before continuing.
+      //   timeout: 5000
+      WaitCommand: {
+        command: '"wait"',
+        timeout: "number",
+      },
+      // - command: hover-image # Hovers an icon, button, or image matching \`description\`. This will also handle handle clicking or right clicking on the icon or image if required.
+      //   description: search icon in the webpage content # Describe the icon or image and what it represents. Describe the element so it can be identified in the future. Do not include the image or icon itself here. Make sure to include the unique traits of this element.
+      //   action: click # What to do when text is found. Available actions are: click, right-click, double-click, hover
+      HoverImageCommand: {
+        command: '"hover-image"',
+        description: "string",
+        action: '"click" | "right-click" | "double-click" | "hover"',
+      },
+      // - command: focus-application # Focus an application by name.
+      //   name: Google Chrome # The name of the application to focus.
+      FocusApplicationCommand: {
+        command: '"focus-application"',
+        name: "string",
+      },
+      // - command: extract # Extract a string value from the screen without needing to interact with the desktop.
+      //   description: My dog's name # The key of the value to extract.
+      //   value: Roofus # The value to extract
+      ExtractCommand: {
+        command: '"extract"',
+        description: "string",
+        value: "string",
+      },
+      // - command: get-email-url # Retrieves the URL from a sign-up confirmation email in the background.
+      //   # This retrieves an email confirmation URL without opening an email client. Do not view the screen, just run this command when dealing with emails
+      //   username: testdriver # The username of the email address to check.
+      GetEmailUrlCommand: {
+        command: '"get-email-url"',
+        username: "string",
+      },
+      // - command: scroll # Scroll up or down. Make sure the correct portion of the page is focused before scrolling.
+      //   direction: down # Available directions are: up, down, left, right
+      //   amount: 300 # Optional. The amount of pixels to scroll. Defaults to 300.
+      ScrollCommand: {
+        command: '"scroll"',
+        direction: '"up" | "down" | "left" | "right"',
+        "amount?": "number",
+      },
+      // - command: scroll-until-text # Scroll until text is found
+      //   text: Sign Up # The text to find on screen. The longer and more unique the better.
+      //   direction: down # Available directions are: up, down, left, right
+      ScrollUntilTextCommand: {
+        command: '"scroll-until-text"',
+        text: "string",
+        direction: '"up" | "down" | "left" | "right"',
+      },
+      // - command: scroll-until-image # Scroll until icon or image is found
+      //   description: Submit at the bottom of the form
+      //   direction: down # Available directions are: up, down, left, rights
+      ScrollUntilImageCommand: {
+        command: '"scroll-until-image"',
+        description: "string",
+        direction: '"up" | "down" | "left" | "right"',
+      },
+      // - command: wait-for-text # Wait until text is seen on screen. Not recommended unless explicitly requested by user.
+      //   text: Copyright 2024 # The text to find on screen.
+      WaitForTextCommand: {
+        command: '"wait-for-text"',
+        text: "string",
+      },
+      // - command: wait-for-image # Wait until icon or image is seen on screen. Not recommended unless explicitly requested by user.
+      //   description: trash icon
+      WaitForImageCommand: {
+        command: '"wait-for-image"',
+        description: "string",
+      },
+      // - command: assert # Assert that a condition is true. This is used to validate that a task was successful. Only use this when the user asks to "assert", "check," or "make sure" of something.
+      //   expect: the video is playing # The condition to check. This should be a string that describes what you see on screen.
+      AssertCommand: {
+        command: '"assert"',
+        expect: "string",
+      },
+      // - command: if # Conditional block. If the condition is true, run the commands in the block. Otherwise, run the commands in the else block. Only use this if the user explicitly asks for a condition.
+      //   condition: the active window is "Google Chrome"
+      //   then:
+      //     - command: hover-text
+      //       text: Search Google or type a URL
+      //       description: main google search
+      //       action: click
+      //     - command: type
+      //       text: monster trucks
+      //       description: search for monster trucks
+      //   else:
+      //     - command: focus-application
+      //       name: Google Chrome
+      IfCommand: {
+        command: '"if"',
+        condition: "string",
+        // The unknown[] instead of Command[] is because of a weird error, although it does work
+        // on the other typescript repo
+        then: "unknown[]",
+        "else?": "unknown[]",
+      },
+
+      Command:
+        "PressKeysCommand | HoverTextCommand | TypeCommand | WaitCommand | HoverImageCommand | FocusApplicationCommand | ExtractCommand | GetEmailUrlCommand | ScrollCommand | ScrollUntilTextCommand | ScrollUntilImageCommand | WaitForTextCommand | WaitForImageCommand | AssertCommand | IfCommand",
+
+      CommandList: "Command[]",
+
+      Step: {
+        prompt: "string",
+        commands: "Command[]",
+      },
+
+      StepList: "Step[]",
+
+      File: {
+        version: "string",
+        session: "string | undefined",
+        steps: "Step[]",
+      },
+    }).export(),
+  );
+
+/**
+ *
+ * @param {unknown} data
+ */
+const getType = async (data) => {
+  const { type } = await import("arktype");
+  const t = await types();
+
+  /**
+   * @type {["File", "Step", "StepList", "Command", "CommandList"]}
+   */
+  const options = ["File", "Step", "StepList", "Command", "CommandList"];
+
+  for (const option of options) {
+    const result = t[option](data);
+    if (!(result instanceof type.errors)) {
+      return option;
+    }
+  }
+  return null;
+};
+
+module.exports = { getType };

package/ai/.claude-plugin/plugin.json

@@ -0,0 +1,9 @@
+{
+  "name": "testdriver",
+  "description": "TestDriver plugin: SDK guidance plus MCP access to test runs and results.",
+  "version": "1.0.0",
+  "author": {
+    "name": "TestDriver"
+  }
+}
+