@testdriverai/agent 7.8.0-canary.10

package/lib/sentry.js

@@ -0,0 +1,417 @@
+/**
+ * Sentry initialization for TestDriver CLI
+ *
+ * This module initializes Sentry for error tracking and performance monitoring.
+ * It should be required at the very beginning of the CLI entry point.
+ *
+ * Distributed Tracing:
+ * The CLI uses session-based trace IDs (MD5 hash of session ID) to link
+ * CLI traces with API traces. Call setSessionTraceContext() after establishing
+ * a session to ensure all CLI errors/logs are linked to the same trace.
+ */
+
+const Sentry = require("@sentry/node");
+const crypto = require("crypto");
+const os = require("os");
+const { version } = require("../package.json");
+const logger = require("../agent/lib/logger");
+
+// Store trace contexts per session so concurrent tests don't overwrite each other.
+// Keys are sessionIds, values are { traceId, sessionId }.
+const _traceContexts = new Map();
+
+// For backward compatibility, track the most recently set session
+let currentTraceId = null;
+let currentSessionId = null;
+
+// Track if we've attached listeners to avoid duplicates
+let emitterAttached = false;
+
+const isEnabled = () => {
+  // Disable if explicitly disabled
+  if (process.env.TD_TELEMETRY === "false") {
+    return false;
+  }
+  return true;
+};
+
+if (isEnabled()) {
+  logger.log("Analytics enabled. Set TD_TELEMETRY=false to disable.");
+  Sentry.init({
+    dsn:
+      process.env.SENTRY_DSN ||
+      "https://452bd5a00dbd83a38ee8813e11c57694@o4510262629236736.ingest.us.sentry.io/4510480443637760",
+    environment: "sdk",
+    release: version,
+    sampleRate: 1.0,
+    tracesSampleRate: 1.0, // Sample 20% of transactions for performance
+    enableLogs: true,
+    integrations: [Sentry.httpIntegration(), Sentry.nodeContextIntegration()],
+    // Set initial context
+    initialScope: {
+      tags: {
+        platform: os.platform(),
+        arch: os.arch(),
+        nodeVersion: process.version,
+      },
+    },
+    // Filter out common non-errors and expected test failures
+    // Only report actual exceptions and crashes, not failed tests or element lookups
+    beforeSend(event, hint) {
+      const error = hint.originalException;
+
+      if (error && error.message) {
+        const message = error.message;
+        
+        // Don't send user-initiated exits
+        if (message.includes("User cancelled")) {
+          return null;
+        }
+
+        // Don't send expected test/element failures - these are normal test outcomes, not crashes
+        if (
+          message.includes("Element not found") ||
+          message.includes("No elements found") ||
+          message.includes("No element found") ||
+          message.includes("Assertion failed") ||
+          message.includes("assertion failed")
+        ) {
+          return null;
+        }
+      }
+
+      // Filter out TestFailure errors (test failures, not crashes)
+      if (error && error.name === "TestFailure") {
+        return null;
+      }
+
+      // Filter out ElementNotFoundError - expected test outcome, not a crash
+      if (error && error.name === "ElementNotFoundError") {
+        return null;
+      }
+
+      return event;
+    },
+  });
+}
+
+/**
+ * Set user context for Sentry
+ * @param {Object} user - User object with id, email, etc.
+ */
+function setUser(user) {
+  if (!isEnabled()) return;
+  Sentry.setUser(user);
+}
+
+/**
+ * Set additional context
+ * @param {string} name - Context name
+ * @param {Object} context - Context data
+ */
+function setContext(name, context) {
+  if (!isEnabled()) return;
+  Sentry.setContext(name, context);
+}
+
+/**
+ * Set a tag
+ * @param {string} key - Tag key
+ * @param {string} value - Tag value
+ */
+function setTag(key, value) {
+  if (!isEnabled()) return;
+  Sentry.setTag(key, value);
+}
+
+/**
+ * Capture an exception
+ * @param {Error} error - The error to capture
+ * @param {Object} context - Additional context
+ */
+function captureException(error, context = {}) {
+  if (!isEnabled()) return;
+
+  Sentry.withScope((scope) => {
+    // Link to session trace if available
+    if (currentTraceId && currentSessionId) {
+      scope.setTag("session", currentSessionId);
+      scope.setContext("trace", {
+        trace_id: currentTraceId,
+        session_id: currentSessionId,
+      });
+    }
+
+    if (context.tags) {
+      Object.entries(context.tags).forEach(([key, value]) => {
+        scope.setTag(key, value);
+      });
+    }
+    if (context.extra) {
+      Object.entries(context.extra).forEach(([key, value]) => {
+        scope.setExtra(key, value);
+      });
+    }
+    Sentry.captureException(error);
+  });
+}
+
+/**
+ * Capture a message
+ * @param {string} message - The message to capture
+ * @param {string} level - Severity level (info, warning, error)
+ */
+function captureMessage(message, level = "info") {
+  if (!isEnabled()) return;
+
+  Sentry.withScope((scope) => {
+    // Link to session trace if available
+    if (currentTraceId && currentSessionId) {
+      scope.setTag("session", currentSessionId);
+      scope.setContext("trace", {
+        trace_id: currentTraceId,
+        session_id: currentSessionId,
+      });
+    }
+    Sentry.captureMessage(message, level);
+  });
+}
+
+/**
+ * Set the session trace context for distributed tracing
+ * This links CLI errors/logs to the same trace as API calls
+ * @param {string} sessionId - The session ID
+ */
+function setSessionTraceContext(sessionId) {
+  if (!isEnabled() || !sessionId) return;
+
+  // Derive trace ID from session ID (same algorithm as API)
+  const traceId = crypto.createHash("md5").update(sessionId).digest("hex");
+
+  // Store per-session trace context for concurrent safety
+  _traceContexts.set(sessionId, { traceId, sessionId });
+
+  // Also update the module-level "latest" for backward compatibility
+  currentTraceId = traceId;
+  currentSessionId = sessionId;
+
+  // Set as global tag so all events include it
+  Sentry.setTag("session", sessionId);
+  Sentry.setTag("trace_id", currentTraceId);
+
+  // Try to set propagation context for trace linking (may not be available in all versions)
+  try {
+    const scope = Sentry.getCurrentScope();
+    if (scope && typeof scope.setPropagationContext === "function") {
+      scope.setPropagationContext({
+        traceId: currentTraceId,
+        spanId: currentTraceId.substring(0, 16),
+        sampled: true,
+      });
+    }
+  } catch (e) {
+    // Ignore errors - propagation context may not be supported
+    logger.log("Could not set propagation context:", e.message);
+  }
+}
+
+/**
+ * Clear the session trace context
+ */
+function clearSessionTraceContext(sessionId) {
+  if (sessionId) {
+    _traceContexts.delete(sessionId);
+    // If the cleared session was the "latest", pick another or null
+    if (currentSessionId === sessionId) {
+      if (_traceContexts.size > 0) {
+        const last = Array.from(_traceContexts.values()).pop();
+        currentTraceId = last.traceId;
+        currentSessionId = last.sessionId;
+      } else {
+        currentTraceId = null;
+        currentSessionId = null;
+      }
+    }
+  } else {
+    // Clear all (backward compatibility)
+    _traceContexts.clear();
+    currentTraceId = null;
+    currentSessionId = null;
+  }
+}
+
+/**
+ * Get the current trace ID (for debugging)
+ * @returns {string|null} Current trace ID or null
+ */
+function getTraceId() {
+  return currentTraceId;
+}
+
+/**
+ * Attach log listeners to an emitter to capture CLI logs as Sentry breadcrumbs
+ * @param {EventEmitter} emitter - The event emitter to listen to
+ */
+function attachLogListeners(emitter) {
+  if (!isEnabled() || !emitter || emitterAttached) return;
+
+  // Check if Sentry.logger is available
+  if (!Sentry.logger) {
+    logger.log("Sentry.logger not available, skipping log listeners");
+    return;
+  }
+
+  emitterAttached = true;
+
+  // Helper to strip ANSI codes for cleaner logs
+  const stripAnsi = (str) => {
+    if (typeof str !== "string") return String(str);
+    return str.replace(/\x1B[[(?);]{0,2}(;?\d)*./g, "");
+  };
+
+  // Helper to get current log attributes with trace context
+  const getLogAttributes = (extra = {}) => {
+    const attrs = { ...extra };
+    if (currentSessionId) {
+      attrs["session.id"] = currentSessionId;
+    }
+    if (currentTraceId) {
+      attrs["sentry.trace.trace_id"] = currentTraceId;
+    }
+    // Get current user from Sentry scope
+    try {
+      const user = Sentry.getCurrentScope().getUser();
+      if (user) {
+        if (user.id) attrs["user.id"] = user.id;
+        if (user.email) attrs["user.email"] = user.email;
+        if (user.username) attrs["user.name"] = user.username;
+      }
+    } catch (e) {
+      // Ignore errors getting user
+    }
+    return attrs;
+  };
+
+  // Capture log:log as info logs
+  emitter.on("log:log", (message) => {
+    Sentry.logger.info(
+      stripAnsi(message),
+      getLogAttributes({ category: "cli.log" }),
+    );
+  });
+
+  // Capture log:warn as warning logs
+  emitter.on("log:warn", (message) => {
+    Sentry.logger.warn(
+      stripAnsi(message),
+      getLogAttributes({ category: "cli.warn" }),
+    );
+  });
+
+  // Capture log:debug as debug logs (only in verbose mode)
+  if (process.env.VERBOSE || process.env.TD_DEBUG) {
+    emitter.on("log:debug", (message) => {
+      Sentry.logger.debug(
+        stripAnsi(message),
+        getLogAttributes({ category: "cli.debug" }),
+      );
+    });
+  }
+
+  // Capture command events
+  emitter.on("command:start", (data) => {
+    Sentry.logger.info(
+      `Command started: ${data?.command || data?.name || "unknown"}`,
+      getLogAttributes({
+        category: "cli.command",
+        ...data,
+      }),
+    );
+  });
+
+  emitter.on("command:error", (data) => {
+    Sentry.logger.error(
+      `Command error: ${data?.message || data?.error || "unknown"}`,
+      getLogAttributes({
+        category: "cli.command",
+        ...data,
+      }),
+    );
+  });
+
+  // Capture step events
+  emitter.on("step:start", (data) => {
+    Sentry.logger.info(
+      `Step started: ${data?.step || data?.name || "unknown"}`,
+      getLogAttributes({
+        category: "cli.step",
+      }),
+    );
+  });
+
+  emitter.on("step:error", (data) => {
+    Sentry.logger.error(
+      `Step error: ${data?.message || data?.error || "unknown"}`,
+      getLogAttributes({
+        category: "cli.step",
+        ...data,
+      }),
+    );
+  });
+
+  // Capture test events
+  emitter.on("test:start", (data) => {
+    Sentry.logger.info(
+      `Test started: ${data?.name || "unknown"}`,
+      getLogAttributes({
+        category: "cli.test",
+      }),
+    );
+  });
+
+  emitter.on("test:error", (data) => {
+    Sentry.logger.error(
+      `Test error: ${data?.message || data?.error || "unknown"}`,
+      getLogAttributes({
+        category: "cli.test",
+        ...data,
+      }),
+    );
+  });
+}
+
+/**
+ * Start a new transaction for performance monitoring
+ * @param {string} name - Transaction name
+ * @param {string} op - Operation type
+ * @returns {Object} Transaction object
+ */
+function startTransaction(name, op = "cli") {
+  if (!isEnabled()) return null;
+  return Sentry.startSpan({ name, op });
+}
+
+/**
+ * Flush pending events before process exit
+ * @param {number} timeout - Timeout in milliseconds
+ */
+async function flush(timeout = 2000) {
+  if (!isEnabled()) return;
+  await Sentry.flush(timeout);
+}
+
+module.exports = {
+  Sentry,
+  isEnabled,
+  setUser,
+  setContext,
+  setTag,
+  captureException,
+  captureMessage,
+  setSessionTraceContext,
+  clearSessionTraceContext,
+  getTraceId,
+  attachLogListeners,
+  startTransaction,
+  flush,
+};

package/lib/vitest/hooks.d.ts

@@ -0,0 +1,57 @@
+/**
+ * TypeScript definitions for TestDriver Vitest Hooks
+ * @module testdriverai/vitest/hooks
+ */
+
+import type TestDriverSDK from '../../sdk';
+import type { TestDriverOptions } from '../../sdk';
+
+/**
+ * Vitest test context (from test function parameter)
+ */
+export interface VitestContext {
+  /**
+   * Current test task
+   */
+  task: any;
+  
+  /**
+   * Register cleanup handler
+   */
+  onTestFinished?: (fn: () => void | Promise<void>) => void;
+}
+
+/**
+ * Options for TestDriver hook (includes all TestDriverOptions)
+ */
+export interface TestDriverHookOptions extends TestDriverOptions {
+  // All TestDriverOptions are already included via extends
+}
+
+/**
+ * Create a TestDriver client for use in Vitest tests
+ * Manages lifecycle automatically (connects on first use, disconnects after test)
+ * 
+ * @param context - Vitest test context (from async (context) => {})
+ * @param options - TestDriver options
+ * @returns TestDriver SDK instance
+ * 
+ * @example
+ * import { describe, expect, it } from "vitest";
+ * import { TestDriver } from "testdriverai/lib/vitest/hooks.mjs";
+ * 
+ * describe("My Test Suite", () => {
+ *   it("should do something", async (context) => {
+ *     const testdriver = TestDriver(context, { newSandbox: true, headless: false });
+ *     
+ *     await testdriver.provision.chrome({ url: 'https://example.com' });
+ *     
+ *     const button = await testdriver.find("Sign In button");
+ *     await button.click();
+ *     
+ *     const result = await testdriver.assert("the dashboard is visible");
+ *     expect(result).toBeTruthy();
+ *   });
+ * });
+ */
+export function TestDriver(context: VitestContext, options?: TestDriverHookOptions): TestDriverSDK;