@wix/evalforge-evaluator 0.14.0 → 0.16.0

package/build/index.js

@@ -24,7 +24,7 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
 ));
 
 // src/index.ts
-var import_evalforge_types4 = require("@wix/evalforge-types");
+var import_evalforge_types5 = require("@wix/evalforge-types");
 
 // src/config.ts
 function loadConfig() {
@@ -87,8 +87,8 @@ function createApiClient(serverUrl, options = "") {
     }
     return headers;
   }
-  async function fetchJson(path9) {
-    const url = `${serverUrl}${apiPrefix}${pathPrefix}${path9}`;
+  async function fetchJson(path10) {
+    const url = `${serverUrl}${apiPrefix}${pathPrefix}${path10}`;
     console.error(`[API] GET ${url}`);
     const headers = buildHeaders();
     const response = await fetch(url, {
@@ -102,8 +102,8 @@ function createApiClient(serverUrl, options = "") {
     }
     return response.json();
   }
-  async function postJson(path9, body) {
-    const url = `${serverUrl}${apiPrefix}${pathPrefix}${path9}`;
+  async function postJson(path10, body) {
+    const url = `${serverUrl}${apiPrefix}${pathPrefix}${path10}`;
     console.error(`[API] POST ${url}`);
     const response = await fetch(url, {
       method: "POST",
@@ -117,8 +117,8 @@ function createApiClient(serverUrl, options = "") {
       );
     }
   }
-  async function deleteRequest(path9) {
-    const url = `${serverUrl}${apiPrefix}${pathPrefix}${path9}`;
+  async function deleteRequest(path10) {
+    const url = `${serverUrl}${apiPrefix}${pathPrefix}${path10}`;
     console.error(`[API] DELETE ${url}`);
     const headers = buildHeaders();
     const response = await fetch(url, {
@@ -132,8 +132,8 @@ function createApiClient(serverUrl, options = "") {
       );
     }
   }
-  async function putJson(path9, body) {
-    const url = `${serverUrl}${apiPrefix}${pathPrefix}${path9}`;
+  async function putJson(path10, body) {
+    const url = `${serverUrl}${apiPrefix}${pathPrefix}${path10}`;
     console.error(`[API] PUT ${url}`);
     const response = await fetch(url, {
       method: "PUT",
@@ -1157,17 +1157,17 @@ var ReadStream = class extends Minipass {
   [_size];
   [_remain];
   [_autoClose];
-  constructor(path9, opt) {
+  constructor(path10, opt) {
     opt = opt || {};
     super(opt);
     this.readable = true;
     this.writable = false;
-    if (typeof path9 !== "string") {
+    if (typeof path10 !== "string") {
       throw new TypeError("path must be a string");
     }
     this[_errored] = false;
     this[_fd] = typeof opt.fd === "number" ? opt.fd : void 0;
-    this[_path] = path9;
+    this[_path] = path10;
     this[_readSize] = opt.readSize || 16 * 1024 * 1024;
     this[_reading] = false;
     this[_size] = typeof opt.size === "number" ? opt.size : Infinity;
@@ -1330,10 +1330,10 @@ var WriteStream = class extends import_events.default {
   [_flags];
   [_finished] = false;
   [_pos];
-  constructor(path9, opt) {
+  constructor(path10, opt) {
     opt = opt || {};
     super(opt);
-    this[_path] = path9;
+    this[_path] = path10;
     this[_fd] = typeof opt.fd === "number" ? opt.fd : void 0;
     this[_mode] = opt.mode === void 0 ? 438 : opt.mode;
     this[_pos] = typeof opt.start === "number" ? opt.start : void 0;
@@ -2226,10 +2226,10 @@ var Header = class {
     }
     const prefixSize = this.ctime || this.atime ? 130 : 155;
     const split = splitPrefix(this.path || "", prefixSize);
-    const path9 = split[0];
+    const path10 = split[0];
     const prefix = split[1];
     this.needPax = !!split[2];
-    this.needPax = encString(buf, off, 100, path9) || this.needPax;
+    this.needPax = encString(buf, off, 100, path10) || this.needPax;
     this.needPax = encNumber(buf, off + 100, 8, this.mode) || this.needPax;
     this.needPax = encNumber(buf, off + 108, 8, this.uid) || this.needPax;
     this.needPax = encNumber(buf, off + 116, 8, this.gid) || this.needPax;
@@ -3205,16 +3205,16 @@ var modeFix = (mode, isDir, portable) => {
 // ../../node_modules/tar/dist/esm/strip-absolute-path.js
 var import_node_path3 = require("node:path");
 var { isAbsolute, parse: parse3 } = import_node_path3.win32;
-var stripAbsolutePath = (path9) => {
+var stripAbsolutePath = (path10) => {
   let r = "";
-  let parsed = parse3(path9);
-  while (isAbsolute(path9) || parsed.root) {
-    const root = path9.charAt(0) === "/" && path9.slice(0, 4) !== "//?/" ? "/" : parsed.root;
-    path9 = path9.slice(root.length);
+  let parsed = parse3(path10);
+  while (isAbsolute(path10) || parsed.root) {
+    const root = path10.charAt(0) === "/" && path10.slice(0, 4) !== "//?/" ? "/" : parsed.root;
+    path10 = path10.slice(root.length);
     r += root;
-    parsed = parse3(path9);
+    parsed = parse3(path10);
   }
-  return [r, path9];
+  return [r, path10];
 };
 
 // ../../node_modules/tar/dist/esm/winchars.js
@@ -3226,12 +3226,12 @@ var encode2 = (s) => raw.reduce((s2, c) => s2.split(c).join(toWin.get(c)), s);
 var decode = (s) => win.reduce((s2, c) => s2.split(c).join(toRaw.get(c)), s);
 
 // ../../node_modules/tar/dist/esm/write-entry.js
-var prefixPath = (path9, prefix) => {
+var prefixPath = (path10, prefix) => {
   if (!prefix) {
-    return normalizeWindowsPath(path9);
+    return normalizeWindowsPath(path10);
   }
-  path9 = normalizeWindowsPath(path9).replace(/^\.(\/|$)/, "");
-  return stripTrailingSlashes(prefix) + "/" + path9;
+  path10 = normalizeWindowsPath(path10).replace(/^\.(\/|$)/, "");
+  return stripTrailingSlashes(prefix) + "/" + path10;
 };
 var maxReadSize = 16 * 1024 * 1024;
 var PROCESS = /* @__PURE__ */ Symbol("process");
@@ -3376,8 +3376,8 @@ var WriteEntry = class extends Minipass {
   [MODE](mode) {
     return modeFix(mode, this.type === "Directory", this.portable);
   }
-  [PREFIX](path9) {
-    return prefixPath(path9, this.prefix);
+  [PREFIX](path10) {
+    return prefixPath(path10, this.prefix);
   }
   [HEADER]() {
     if (!this.stat) {
@@ -3758,8 +3758,8 @@ var WriteEntryTar = class extends Minipass {
     super.write(b);
     readEntry.pipe(this);
   }
-  [PREFIX](path9) {
-    return prefixPath(path9, this.prefix);
+  [PREFIX](path10) {
+    return prefixPath(path10, this.prefix);
   }
   [MODE](mode) {
     return modeFix(mode, this.type === "Directory", this.portable);
@@ -4183,8 +4183,8 @@ var PackJob = class {
   pending = false;
   ignore = false;
   piped = false;
-  constructor(path9, absolute) {
-    this.path = path9 || "./";
+  constructor(path10, absolute) {
+    this.path = path10 || "./";
     this.absolute = absolute;
   }
 };
@@ -4312,21 +4312,21 @@ var Pack = class extends Minipass {
   [WRITE](chunk) {
     return super.write(chunk);
   }
-  add(path9) {
-    this.write(path9);
+  add(path10) {
+    this.write(path10);
     return this;
   }
-  end(path9, encoding, cb) {
-    if (typeof path9 === "function") {
-      cb = path9;
-      path9 = void 0;
+  end(path10, encoding, cb) {
+    if (typeof path10 === "function") {
+      cb = path10;
+      path10 = void 0;
     }
     if (typeof encoding === "function") {
       cb = encoding;
       encoding = void 0;
     }
-    if (path9) {
-      this.add(path9);
+    if (path10) {
+      this.add(path10);
     }
     this[ENDED2] = true;
     this[PROCESS2]();
@@ -4334,14 +4334,14 @@ var Pack = class extends Minipass {
       cb();
     return this;
   }
-  write(path9) {
+  write(path10) {
     if (this[ENDED2]) {
       throw new Error("write after end");
     }
-    if (path9 instanceof ReadEntry) {
-      this[ADDTARENTRY](path9);
+    if (path10 instanceof ReadEntry) {
+      this[ADDTARENTRY](path10);
     } else {
-      this[ADDFSENTRY](path9);
+      this[ADDFSENTRY](path10);
     }
     return this.flowing;
   }
@@ -4684,9 +4684,9 @@ var getWriteFlag = !fMapEnabled ? () => "w" : (size) => size < fMapLimit ? fMapF
 // ../../node_modules/chownr/dist/esm/index.js
 var import_node_fs2 = __toESM(require("node:fs"), 1);
 var import_node_path5 = __toESM(require("node:path"), 1);
-var lchownSync = (path9, uid, gid) => {
+var lchownSync = (path10, uid, gid) => {
   try {
-    return import_node_fs2.default.lchownSync(path9, uid, gid);
+    return import_node_fs2.default.lchownSync(path10, uid, gid);
   } catch (er) {
     if (er?.code !== "ENOENT")
       throw er;
@@ -4769,9 +4769,9 @@ var CwdError = class extends Error {
   path;
   code;
   syscall = "chdir";
-  constructor(path9, code2) {
-    super(`${code2}: Cannot cd into '${path9}'`);
-    this.path = path9;
+  constructor(path10, code2) {
+    super(`${code2}: Cannot cd into '${path10}'`);
+    this.path = path10;
     this.code = code2;
   }
   get name() {
@@ -4785,10 +4785,10 @@ var SymlinkError = class extends Error {
   symlink;
   syscall = "symlink";
   code = "TAR_SYMLINK_ERROR";
-  constructor(symlink, path9) {
+  constructor(symlink, path10) {
     super("TAR_SYMLINK_ERROR: Cannot extract through symbolic link");
     this.symlink = symlink;
-    this.path = path9;
+    this.path = path10;
   }
   get name() {
     return "SymlinkError";
@@ -4970,13 +4970,13 @@ var normalizeUnicode = (s) => {
 // ../../node_modules/tar/dist/esm/path-reservations.js
 var platform3 = process.env.TESTING_TAR_FAKE_PLATFORM || process.platform;
 var isWindows2 = platform3 === "win32";
-var getDirs = (path9) => {
-  const dirs = path9.split("/").slice(0, -1).reduce((set, path10) => {
+var getDirs = (path10) => {
+  const dirs = path10.split("/").slice(0, -1).reduce((set, path11) => {
     const s = set[set.length - 1];
     if (s !== void 0) {
-      path10 = (0, import_node_path7.join)(s, path10);
+      path11 = (0, import_node_path7.join)(s, path11);
     }
-    set.push(path10 || "/");
+    set.push(path11 || "/");
     return set;
   }, []);
   return dirs;
@@ -4994,7 +4994,7 @@ var PathReservations = class {
     paths = isWindows2 ? ["win32 parallelization disabled"] : paths.map((p) => {
       return stripTrailingSlashes((0, import_node_path7.join)(normalizeUnicode(p))).toLowerCase();
     });
-    const dirs = new Set(paths.map((path9) => getDirs(path9)).reduce((a, b) => a.concat(b)));
+    const dirs = new Set(paths.map((path10) => getDirs(path10)).reduce((a, b) => a.concat(b)));
     this.#reservations.set(fn, { dirs, paths });
     for (const p of paths) {
       const q = this.#queues.get(p);
@@ -5027,8 +5027,8 @@ var PathReservations = class {
       throw new Error("function does not have any path reservations");
     }
     return {
-      paths: res.paths.map((path9) => this.#queues.get(path9)),
-      dirs: [...res.dirs].map((path9) => this.#queues.get(path9))
+      paths: res.paths.map((path10) => this.#queues.get(path10)),
+      dirs: [...res.dirs].map((path10) => this.#queues.get(path10))
     };
   }
   // check if fn is first in line for all its paths, and is
@@ -5056,14 +5056,14 @@ var PathReservations = class {
     }
     const { paths, dirs } = res;
     const next = /* @__PURE__ */ new Set();
-    for (const path9 of paths) {
-      const q = this.#queues.get(path9);
+    for (const path10 of paths) {
+      const q = this.#queues.get(path10);
       if (!q || q?.[0] !== fn) {
         continue;
       }
       const q0 = q[1];
       if (!q0) {
-        this.#queues.delete(path9);
+        this.#queues.delete(path10);
         continue;
       }
       q.shift();
@@ -5128,24 +5128,24 @@ var CHECKED_CWD = /* @__PURE__ */ Symbol("checkedCwd");
 var platform4 = process.env.TESTING_TAR_FAKE_PLATFORM || process.platform;
 var isWindows3 = platform4 === "win32";
 var DEFAULT_MAX_DEPTH = 1024;
-var unlinkFile = (path9, cb) => {
+var unlinkFile = (path10, cb) => {
   if (!isWindows3) {
-    return import_node_fs4.default.unlink(path9, cb);
+    return import_node_fs4.default.unlink(path10, cb);
   }
-  const name2 = path9 + ".DELETE." + (0, import_node_crypto.randomBytes)(16).toString("hex");
-  import_node_fs4.default.rename(path9, name2, (er) => {
+  const name2 = path10 + ".DELETE." + (0, import_node_crypto.randomBytes)(16).toString("hex");
+  import_node_fs4.default.rename(path10, name2, (er) => {
     if (er) {
       return cb(er);
     }
     import_node_fs4.default.unlink(name2, cb);
   });
 };
-var unlinkFileSync = (path9) => {
+var unlinkFileSync = (path10) => {
   if (!isWindows3) {
-    return import_node_fs4.default.unlinkSync(path9);
+    return import_node_fs4.default.unlinkSync(path10);
   }
-  const name2 = path9 + ".DELETE." + (0, import_node_crypto.randomBytes)(16).toString("hex");
-  import_node_fs4.default.renameSync(path9, name2);
+  const name2 = path10 + ".DELETE." + (0, import_node_crypto.randomBytes)(16).toString("hex");
+  import_node_fs4.default.renameSync(path10, name2);
   import_node_fs4.default.unlinkSync(name2);
 };
 var uint32 = (a, b, c) => a !== void 0 && a === a >>> 0 ? a : b !== void 0 && b === b >>> 0 ? b : c;
@@ -5243,24 +5243,24 @@ var Unpack = class extends Parser {
   // return false if we need to skip this file
   // return true if the field was successfully sanitized
   [STRIPABSOLUTEPATH](entry, field) {
-    const path9 = entry[field];
-    if (!path9 || this.preservePaths)
+    const path10 = entry[field];
+    if (!path10 || this.preservePaths)
       return true;
-    const parts = path9.split("/");
+    const parts = path10.split("/");
     if (parts.includes("..") || /* c8 ignore next */
     isWindows3 && /^[a-z]:\.\.$/i.test(parts[0] ?? "")) {
       this.warn("TAR_ENTRY_ERROR", `${field} contains '..'`, {
         entry,
-        [field]: path9
+        [field]: path10
       });
       return false;
     }
-    const [root, stripped] = stripAbsolutePath(path9);
+    const [root, stripped] = stripAbsolutePath(path10);
     if (root) {
       entry[field] = String(stripped);
       this.warn("TAR_ENTRY_INFO", `stripping ${root} from absolute ${field}`, {
         entry,
-        [field]: path9
+        [field]: path10
       });
     }
     return true;
@@ -6027,9 +6027,9 @@ var mtimeFilter = (opt) => {
   if (!opt.mtimeCache) {
     opt.mtimeCache = /* @__PURE__ */ new Map();
   }
-  opt.filter = filter ? (path9, stat) => filter(path9, stat) && !/* c8 ignore start */
-  ((opt.mtimeCache?.get(path9) ?? stat.mtime ?? 0) > (stat.mtime ?? 0)) : (path9, stat) => !/* c8 ignore start */
-  ((opt.mtimeCache?.get(path9) ?? stat.mtime ?? 0) > (stat.mtime ?? 0));
+  opt.filter = filter ? (path10, stat) => filter(path10, stat) && !/* c8 ignore start */
+  ((opt.mtimeCache?.get(path10) ?? stat.mtime ?? 0) > (stat.mtime ?? 0)) : (path10, stat) => !/* c8 ignore start */
+  ((opt.mtimeCache?.get(path10) ?? stat.mtime ?? 0) > (stat.mtime ?? 0));
 };
 
 // src/run-scenario/environment.ts
@@ -6324,6 +6324,37 @@ async function executeWithClaudeCode(skill, scenario, options) {
     queryOptions.mcpServers ? Object.keys(queryOptions.mcpServers) : "none"
   );
   console.log("[SDK-DEBUG] Calling SDK query()...");
+  if (traceContext) {
+    const preExecEvent = {
+      evalRunId: traceContext.evalRunId,
+      scenarioId: traceContext.scenarioId,
+      scenarioName: traceContext.scenarioName,
+      targetId: traceContext.targetId,
+      targetName: traceContext.targetName,
+      stepNumber: 0,
+      type: import_evalforge_types.LiveTraceEventType.DIAGNOSTIC,
+      outputPreview: JSON.stringify({
+        event: "pre-sdk-execution",
+        model: queryOptions.model,
+        maxTurns: queryOptions.maxTurns,
+        sdkEnv: {
+          ANTHROPIC_BASE_URL: sdkEnv.ANTHROPIC_BASE_URL,
+          hasANTHROPIC_API_KEY: !!sdkEnv.ANTHROPIC_API_KEY,
+          hasANTHROPIC_AUTH_TOKEN: !!sdkEnv.ANTHROPIC_AUTH_TOKEN,
+          hasANTHROPIC_CUSTOM_HEADERS: !!sdkEnv.ANTHROPIC_CUSTOM_HEADERS
+        },
+        timestamp: (/* @__PURE__ */ new Date()).toISOString()
+      }),
+      timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+      isComplete: false
+    };
+    emitTraceEvent(
+      preExecEvent,
+      traceContext.tracePushUrl,
+      traceContext.routeHeader,
+      traceContext.authToken
+    );
+  }
   try {
     for await (const message of query({
       prompt: scenario.triggerPrompt,
@@ -6504,6 +6535,38 @@ async function executeWithClaudeCode(skill, scenario, options) {
       sdkError: Object.keys(sdkSpecificInfo).length > 0 ? sdkSpecificInfo : void 0,
       cause: causeInfo
     };
+    if (traceContext) {
+      const errorTraceEvent = {
+        evalRunId: traceContext.evalRunId,
+        scenarioId: traceContext.scenarioId,
+        scenarioName: traceContext.scenarioName,
+        targetId: traceContext.targetId,
+        targetName: traceContext.targetName,
+        stepNumber: traceStepNumber + 1,
+        type: import_evalforge_types.LiveTraceEventType.DIAGNOSTIC,
+        outputPreview: JSON.stringify(
+          {
+            event: "sdk-execution-failed",
+            error: errorMessage,
+            errorName,
+            messageCount,
+            sdkEnv: sdkEnvDebug,
+            sdkError: sdkSpecificInfo,
+            cause: causeInfo
+          },
+          null,
+          2
+        ).slice(0, 2e3),
+        timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+        isComplete: true
+      };
+      emitTraceEvent(
+        errorTraceEvent,
+        traceContext.tracePushUrl,
+        traceContext.routeHeader,
+        traceContext.authToken
+      );
+    }
     throw new Error(
       `Claude SDK execution failed after ${messageCount} messages: ${errorMessage}
 Details: ${JSON.stringify(errorDetails, null, 2)}` + (errorStack ? `
@@ -6870,6 +6933,8 @@ ${stackLines.join("\n")}`);
   return parts.join(" ");
 }
 var ExecutionPhase = {
+  /** Environment diagnostics phase (runs before execution) */
+  DIAGNOSTICS: "diagnostics",
   CONFIG: "config-loading",
   API_CLIENT: "api-client-creation",
   FETCH_EVAL_RUN: "fetch-eval-run",
@@ -6886,6 +6951,585 @@ var ExecutionPhase = {
   UPDATE_STATUS: "update-status"
 };
 
+// src/diagnostics.ts
+var import_child_process = require("child_process");
+var fs11 = __toESM(require("fs"));
+var path9 = __toESM(require("path"));
+var import_evalforge_types4 = require("@wix/evalforge-types");
+async function execCommand(command, timeoutMs = 1e4) {
+  return new Promise((resolve) => {
+    try {
+      const proc2 = (0, import_child_process.spawn)("sh", ["-c", command], {
+        timeout: timeoutMs
+      });
+      let stdout = "";
+      let stderr = "";
+      proc2.stdout.on("data", (data) => {
+        stdout += data.toString();
+      });
+      proc2.stderr.on("data", (data) => {
+        stderr += data.toString();
+      });
+      proc2.on("close", (code2) => {
+        resolve({
+          stdout: stdout.trim(),
+          stderr: stderr.trim(),
+          exitCode: code2 ?? -1
+        });
+      });
+      proc2.on("error", (err) => {
+        resolve({
+          stdout: "",
+          stderr: err.message,
+          exitCode: -1
+        });
+      });
+    } catch (err) {
+      resolve({
+        stdout: "",
+        stderr: err instanceof Error ? err.message : String(err),
+        exitCode: -99
+      });
+    }
+  });
+}
+async function safeRunTest(testName, testFn) {
+  const start = Date.now();
+  try {
+    return await testFn();
+  } catch (err) {
+    const error = err instanceof Error ? err.message : String(err);
+    return {
+      name: testName,
+      passed: false,
+      details: {
+        testCrashed: true,
+        error,
+        stack: err instanceof Error ? err.stack : void 0
+      },
+      error: `Test crashed: ${error}`,
+      durationMs: Date.now() - start
+    };
+  }
+}
+async function testClaudeBinaryDiscovery() {
+  const start = Date.now();
+  const details = {};
+  const npmRootResult = await execCommand("npm root -g");
+  const npmBinResult = await execCommand("npm bin -g");
+  const npmRoot = npmRootResult.stdout;
+  const npmBin = npmBinResult.stdout;
+  details.npmRoot = npmRoot;
+  details.npmBin = npmBin;
+  const evaluatorBinPath = path9.join(
+    npmRoot,
+    "@wix",
+    "evalforge-evaluator",
+    "node_modules",
+    ".bin"
+  );
+  details.evaluatorBinPath = evaluatorBinPath;
+  const lsBinResult = await execCommand(`ls -la "${evaluatorBinPath}" 2>&1`);
+  details.evaluatorBinContents = lsBinResult.stdout || lsBinResult.stderr;
+  details.lsBinExitCode = lsBinResult.exitCode;
+  const claudePath = path9.join(evaluatorBinPath, "claude");
+  let claudeExists = false;
+  try {
+    claudeExists = fs11.existsSync(claudePath);
+  } catch {
+    claudeExists = false;
+  }
+  details.claudePath = claudePath;
+  details.claudeExists = claudeExists;
+  if (claudeExists) {
+    const readlinkResult = await execCommand(
+      `readlink -f "${claudePath}" 2>&1`
+    );
+    details.claudeRealPath = readlinkResult.stdout || readlinkResult.stderr;
+    const statResult = await execCommand(`stat "${claudePath}" 2>&1`);
+    details.claudeStat = statResult.stdout || statResult.stderr;
+    const lsClaudeResult = await execCommand(`ls -la "${claudePath}" 2>&1`);
+    details.claudeFileInfo = lsClaudeResult.stdout;
+  }
+  const whichResult = await execCommand("which claude 2>&1");
+  details.whichClaude = whichResult.stdout || "(not in PATH)";
+  details.whichExitCode = whichResult.exitCode;
+  const currentPath = process.env.PATH || "";
+  details.currentPATH = currentPath.split(":");
+  details.pathLength = currentPath.split(":").length;
+  const passed = claudeExists || whichResult.exitCode === 0;
+  return {
+    name: "claude-binary-discovery",
+    passed,
+    details,
+    error: passed ? void 0 : `Claude binary not found at ${claudePath}`,
+    durationMs: Date.now() - start
+  };
+}
+async function testClaudeExecution() {
+  const start = Date.now();
+  const details = {};
+  const npmRootResult = await execCommand("npm root -g");
+  const npmRoot = npmRootResult.stdout;
+  const claudePath = path9.join(
+    npmRoot,
+    "@wix",
+    "evalforge-evaluator",
+    "node_modules",
+    ".bin",
+    "claude"
+  );
+  details.claudePath = claudePath;
+  const versionResult = await execCommand(
+    `"${claudePath}" --version 2>&1`,
+    15e3
+  );
+  details.versionCommand = {
+    command: `"${claudePath}" --version`,
+    stdout: versionResult.stdout,
+    stderr: versionResult.stderr,
+    exitCode: versionResult.exitCode
+  };
+  const helpResult = await execCommand(
+    `"${claudePath}" --help 2>&1 | head -50`,
+    15e3
+  );
+  details.helpCommand = {
+    command: `"${claudePath}" --help | head -50`,
+    stdout: helpResult.stdout.slice(0, 1500),
+    stderr: helpResult.stderr.slice(0, 500),
+    exitCode: helpResult.exitCode
+  };
+  const whichClaudeResult = await execCommand("which claude 2>&1");
+  if (whichClaudeResult.exitCode === 0) {
+    const pathVersionResult = await execCommand("claude --version 2>&1", 15e3);
+    details.pathVersionCommand = {
+      whichClaude: whichClaudeResult.stdout,
+      stdout: pathVersionResult.stdout,
+      stderr: pathVersionResult.stderr,
+      exitCode: pathVersionResult.exitCode
+    };
+  }
+  const passed = versionResult.exitCode === 0 || helpResult.exitCode === 0;
+  return {
+    name: "claude-cli-execution",
+    passed,
+    details,
+    error: passed ? void 0 : `Claude CLI failed. Version exit: ${versionResult.exitCode}, Help exit: ${helpResult.exitCode}`,
+    durationMs: Date.now() - start
+  };
+}
+async function testEnvironmentDump() {
+  const start = Date.now();
+  const details = {};
+  const importantVars = [
+    "PATH",
+    "HOME",
+    "USER",
+    "SHELL",
+    "NODE_ENV",
+    "PWD",
+    "EVAL_SERVER_URL",
+    "AI_GATEWAY_URL",
+    "TRACE_PUSH_URL",
+    "EVAL_AUTH_TOKEN",
+    "ANTHROPIC_API_KEY",
+    "ANTHROPIC_AUTH_TOKEN",
+    "ANTHROPIC_BASE_URL",
+    "ANTHROPIC_CUSTOM_HEADERS"
+  ];
+  const capturedVars = {};
+  for (const key of importantVars) {
+    const value = process.env[key];
+    if (value) {
+      if (key.includes("SECRET") || key.includes("TOKEN") || key.includes("API_KEY")) {
+        capturedVars[key] = `[REDACTED - ${value.length} chars, starts: ${value.slice(0, 10)}...]`;
+      } else if (key === "ANTHROPIC_CUSTOM_HEADERS") {
+        capturedVars[key] = value.split("\n").map((h) => {
+          const [name2, val] = h.split(":");
+          return `${name2}: ${val ? "[" + val.length + " chars]" : "(empty)"}`;
+        }).join(" | ");
+      } else if (key === "PATH") {
+        const parts = value.split(":");
+        capturedVars[key] = `[${parts.length} entries] First: ${parts.slice(0, 3).join(":")} ... Last: ${parts.slice(-2).join(":")}`;
+      } else {
+        capturedVars[key] = value;
+      }
+    } else {
+      capturedVars[key] = "(NOT SET)";
+    }
+  }
+  details.importantVars = capturedVars;
+  const envResult = await execCommand("env | sort | head -50");
+  details.envCommandOutput = envResult.stdout;
+  details.envExitCode = envResult.exitCode;
+  details.nodeInfo = {
+    version: process.version,
+    platform: process.platform,
+    arch: process.arch,
+    pid: process.pid,
+    cwd: process.cwd(),
+    execPath: process.execPath
+  };
+  return {
+    name: "environment-dump",
+    passed: true,
+    // Info test, always passes
+    details,
+    durationMs: Date.now() - start
+  };
+}
+async function testFileSystemStructure() {
+  const start = Date.now();
+  const details = {};
+  const npmRootResult = await execCommand("npm root -g");
+  const npmRoot = npmRootResult.stdout;
+  const lsCwdResult = await execCommand("ls -la");
+  details.currentDirectory = {
+    path: process.cwd(),
+    contents: lsCwdResult.stdout
+  };
+  const lsNpmRootResult = await execCommand(
+    `ls -la "${npmRoot}" 2>&1 | head -30`
+  );
+  details.npmGlobalRoot = {
+    path: npmRoot,
+    contents: lsNpmRootResult.stdout
+  };
+  const wixPath = path9.join(npmRoot, "@wix");
+  const lsWixResult = await execCommand(`ls -la "${wixPath}" 2>&1`);
+  details.wixPackages = {
+    path: wixPath,
+    contents: lsWixResult.stdout
+  };
+  const evaluatorPath = path9.join(npmRoot, "@wix", "evalforge-evaluator");
+  const lsEvaluatorResult = await execCommand(`ls -la "${evaluatorPath}" 2>&1`);
+  details.evaluatorDir = {
+    path: evaluatorPath,
+    contents: lsEvaluatorResult.stdout
+  };
+  const nodeModulesPath = path9.join(evaluatorPath, "node_modules");
+  const lsNodeModulesResult = await execCommand(
+    `ls "${nodeModulesPath}" 2>&1 | head -30`
+  );
+  details.evaluatorNodeModules = {
+    path: nodeModulesPath,
+    contents: lsNodeModulesResult.stdout
+  };
+  const anthropicPath = path9.join(nodeModulesPath, "@anthropic-ai");
+  const lsAnthropicResult = await execCommand(`ls -la "${anthropicPath}" 2>&1`);
+  details.anthropicPackages = {
+    path: anthropicPath,
+    contents: lsAnthropicResult.stdout
+  };
+  const binPath = path9.join(nodeModulesPath, ".bin");
+  const lsBinResult = await execCommand(`ls -la "${binPath}" 2>&1`);
+  details.binDirectory = {
+    path: binPath,
+    contents: lsBinResult.stdout
+  };
+  return {
+    name: "file-system-structure",
+    passed: true,
+    // Info test, always passes
+    details,
+    durationMs: Date.now() - start
+  };
+}
+async function testNetworkConnectivity(config) {
+  const start = Date.now();
+  const details = {};
+  const dnsResult = await execCommand(
+    "nslookup manage.wix.com 2>&1 | head -10"
+  );
+  details.dnsLookup = {
+    command: "nslookup manage.wix.com",
+    output: dnsResult.stdout || dnsResult.stderr,
+    exitCode: dnsResult.exitCode
+  };
+  const pingResult = await execCommand("ping -c 2 manage.wix.com 2>&1");
+  details.pingTest = {
+    command: "ping -c 2 manage.wix.com",
+    output: pingResult.stdout || pingResult.stderr,
+    exitCode: pingResult.exitCode
+  };
+  const gatewayUrl = config.aiGatewayUrl || "https://manage.wix.com/_api/eval-wix-ai-gateway-proxy";
+  const curlGatewayResult = await execCommand(
+    `curl -v -s --connect-timeout 5 --max-time 10 "${gatewayUrl}" 2>&1 | tail -30`
+  );
+  details.aiGatewayTest = {
+    url: gatewayUrl,
+    output: curlGatewayResult.stdout,
+    exitCode: curlGatewayResult.exitCode
+  };
+  const serverUrl = config.serverUrl;
+  const curlServerResult = await execCommand(
+    `curl -v -s --connect-timeout 5 --max-time 10 "${serverUrl}/health" 2>&1 | tail -30`
+  );
+  details.backendServerTest = {
+    url: `${serverUrl}/health`,
+    output: curlServerResult.stdout,
+    exitCode: curlServerResult.exitCode
+  };
+  const httpsResult = await execCommand(
+    'curl -s --connect-timeout 5 -o /dev/null -w "HTTP_CODE:%{http_code} TIME:%{time_total}s" https://www.google.com 2>&1'
+  );
+  details.httpsBaseline = {
+    command: "curl https://www.google.com",
+    output: httpsResult.stdout,
+    exitCode: httpsResult.exitCode
+  };
+  const networkWorks = pingResult.exitCode === 0 || httpsResult.exitCode === 0;
+  const gatewayReachable = curlGatewayResult.exitCode === 0;
+  return {
+    name: "network-connectivity",
+    passed: networkWorks && gatewayReachable,
+    details,
+    error: networkWorks && gatewayReachable ? void 0 : `Network: ${networkWorks ? "OK" : "FAILED"}, Gateway: ${gatewayReachable ? "OK" : "FAILED"}`,
+    durationMs: Date.now() - start
+  };
+}
+async function testChildProcessSpawning() {
+  const start = Date.now();
+  const details = {};
+  const echoResult = await execCommand('echo "DIAGNOSTIC_TEST_SUCCESS_12345"');
+  details.echoTest = {
+    command: 'echo "DIAGNOSTIC_TEST_SUCCESS_12345"',
+    output: echoResult.stdout,
+    exitCode: echoResult.exitCode,
+    passed: echoResult.stdout === "DIAGNOSTIC_TEST_SUCCESS_12345"
+  };
+  const nodeResult = await execCommand(
+    'node -e "console.log(JSON.stringify({pid: process.pid, version: process.version, platform: process.platform}))"'
+  );
+  details.nodeTest = {
+    command: 'node -e "console.log(JSON.stringify({...}))"',
+    output: nodeResult.stdout,
+    exitCode: nodeResult.exitCode
+  };
+  const shellResult = await execCommand(
+    'echo "PID: $$"; pwd; whoami; date; uname -a'
+  );
+  details.shellTest = {
+    command: 'echo "PID: $$"; pwd; whoami; date; uname -a',
+    output: shellResult.stdout,
+    exitCode: shellResult.exitCode
+  };
+  const stderrResult = await execCommand(
+    `node -e "console.error('stderr test')"`
+  );
+  details.stderrTest = {
+    command: `node -e "console.error('stderr test')"`,
+    stderr: stderrResult.stderr,
+    exitCode: stderrResult.exitCode
+  };
+  const exitCodeResult = await execCommand("exit 42");
+  details.exitCodeTest = {
+    command: "exit 42",
+    exitCode: exitCodeResult.exitCode,
+    passed: exitCodeResult.exitCode === 42
+  };
+  const passed = echoResult.exitCode === 0 && echoResult.stdout === "DIAGNOSTIC_TEST_SUCCESS_12345";
+  return {
+    name: "child-process-spawning",
+    passed,
+    details,
+    error: passed ? void 0 : "Echo test failed",
+    durationMs: Date.now() - start
+  };
+}
+async function testSdkImport() {
+  const start = Date.now();
+  const details = {};
+  try {
+    const sdk = await import("@anthropic-ai/claude-agent-sdk");
+    details.sdkImported = true;
+    details.exportedKeys = Object.keys(sdk);
+    details.hasQuery = typeof sdk.query === "function";
+    if (typeof sdk.query === "function") {
+      details.queryFunctionExists = true;
+      details.queryFunctionType = typeof sdk.query;
+    }
+    return {
+      name: "sdk-import",
+      passed: true,
+      details,
+      durationMs: Date.now() - start
+    };
+  } catch (err) {
+    const error = err instanceof Error ? err.message : String(err);
+    return {
+      name: "sdk-import",
+      passed: false,
+      details: {
+        sdkImported: false,
+        error,
+        stack: err instanceof Error ? err.stack?.split("\n").slice(0, 5) : void 0
+      },
+      error: `Failed to import SDK: ${error}`,
+      durationMs: Date.now() - start
+    };
+  }
+}
+async function testFileSystemWrite() {
+  const start = Date.now();
+  const details = {};
+  const testDir = "/tmp/evalforge-diagnostics-test";
+  const testFile = path9.join(testDir, "test-file.txt");
+  const testContent = `Diagnostic test at ${(/* @__PURE__ */ new Date()).toISOString()}`;
+  try {
+    if (!fs11.existsSync(testDir)) {
+      fs11.mkdirSync(testDir, { recursive: true });
+    }
+    details.directoryCreated = true;
+    fs11.writeFileSync(testFile, testContent);
+    details.fileWritten = true;
+    const readContent = fs11.readFileSync(testFile, "utf8");
+    details.fileRead = true;
+    details.contentMatches = readContent === testContent;
+    const lsResult = await execCommand(`ls -la "${testDir}"`);
+    details.directoryContents = lsResult.stdout;
+    fs11.unlinkSync(testFile);
+    fs11.rmdirSync(testDir);
+    details.cleanedUp = true;
+    return {
+      name: "file-system-write",
+      passed: true,
+      details,
+      durationMs: Date.now() - start
+    };
+  } catch (err) {
+    const error = err instanceof Error ? err.message : String(err);
+    return {
+      name: "file-system-write",
+      passed: false,
+      details: {
+        ...details,
+        error,
+        testDir,
+        testFile
+      },
+      error: `File system write failed: ${error}`,
+      durationMs: Date.now() - start
+    };
+  }
+}
+function emitDiagnosticTraceEvent(evalRunId2, result, tracePushUrl, routeHeader, authToken) {
+  const truncatedResult = "summary" in result ? result : {
+    ...result,
+    details: JSON.parse(
+      JSON.stringify(
+        result.details,
+        (_, v) => typeof v === "string" && v.length > 500 ? v.slice(0, 500) + "... [truncated]" : v
+      )
+    )
+  };
+  const event = {
+    evalRunId: evalRunId2,
+    scenarioId: "diagnostics",
+    scenarioName: "Environment Diagnostics",
+    targetId: "system",
+    targetName: "name" in truncatedResult ? truncatedResult.name : "Summary",
+    stepNumber: 0,
+    type: import_evalforge_types4.LiveTraceEventType.DIAGNOSTIC,
+    outputPreview: JSON.stringify(truncatedResult, null, 2).slice(0, 3e3),
+    timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+    isComplete: "summary" in result
+  };
+  console.log(`TRACE_EVENT:${JSON.stringify(event)}`);
+  if (tracePushUrl) {
+    const headers = {
+      "Content-Type": "application/json"
+    };
+    if (routeHeader) {
+      headers["x-wix-route"] = routeHeader;
+    }
+    if (authToken) {
+      headers["Authorization"] = `Bearer ${authToken}`;
+    }
+    fetch(tracePushUrl, {
+      method: "POST",
+      headers,
+      body: JSON.stringify([event])
+    }).catch((err) => {
+      console.error(
+        "[DIAGNOSTICS] Failed to push trace event to backend:",
+        err
+      );
+    });
+  }
+}
+async function runDiagnostics(config, evalRunId2) {
+  const startedAt = (/* @__PURE__ */ new Date()).toISOString();
+  const startTime = Date.now();
+  console.error("");
+  console.error("\u2554\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2557");
+  console.error("\u2551           EVALFORGE ENVIRONMENT DIAGNOSTICS              \u2551");
+  console.error("\u2551   (Results sent to backend via trace events)             \u2551");
+  console.error("\u255A\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u255D");
+  console.error("");
+  const tests = [];
+  const runTest = async (testName, testFn) => {
+    console.error(`[DIAG] Running: ${testName}...`);
+    const result = await safeRunTest(testName, testFn);
+    tests.push(result);
+    const status = result.passed ? "\u2713 PASS" : "\u2717 FAIL";
+    console.error(`[DIAG] ${status}: ${result.name} (${result.durationMs}ms)`);
+    console.error("[DIAG] Details:");
+    console.error(JSON.stringify(result.details, null, 2));
+    console.error("");
+    if (!result.passed && result.error) {
+      console.error(`[DIAG] ERROR: ${result.error}`);
+    }
+    emitDiagnosticTraceEvent(
+      evalRunId2,
+      result,
+      config.tracePushUrl,
+      config.routeHeader,
+      config.authToken
+    );
+  };
+  await runTest("claude-binary-discovery", testClaudeBinaryDiscovery);
+  await runTest("claude-cli-execution", testClaudeExecution);
+  await runTest("environment-dump", testEnvironmentDump);
+  await runTest("file-system-structure", testFileSystemStructure);
+  await runTest("network-connectivity", () => testNetworkConnectivity(config));
+  await runTest("child-process-spawning", testChildProcessSpawning);
+  await runTest("sdk-import", testSdkImport);
+  await runTest("file-system-write", testFileSystemWrite);
+  const completedAt = (/* @__PURE__ */ new Date()).toISOString();
+  const totalDurationMs = Date.now() - startTime;
+  const report = {
+    startedAt,
+    completedAt,
+    totalDurationMs,
+    tests,
+    summary: {
+      total: tests.length,
+      passed: tests.filter((t) => t.passed).length,
+      failed: tests.filter((t) => !t.passed).length
+    }
+  };
+  emitDiagnosticTraceEvent(
+    evalRunId2,
+    report,
+    config.tracePushUrl,
+    config.routeHeader,
+    config.authToken
+  );
+  console.error("");
+  console.error("\u2554\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2557");
+  console.error(
+    `\u2551  DIAGNOSTICS COMPLETE: ${report.summary.passed}/${report.summary.total} passed, ${report.summary.failed} failed`.padEnd(
+      60
+    ) + "\u2551"
+  );
+  console.error(`\u2551  Total time: ${totalDurationMs}ms`.padEnd(60) + "\u2551");
+  console.error("\u255A\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u255D");
+  console.error("");
+  return report;
+}
+
 // src/index.ts
 console.error(
   "[EVALUATOR-BOOT] Module loading started",
@@ -6953,6 +7597,33 @@ async function runEvaluation(projectId2, evalRunId2) {
       `[${ExecutionPhase.API_CLIENT}] Failed to create API client: ${apiErr instanceof Error ? apiErr.message : String(apiErr)}`
     );
   }
+  state.currentPhase = ExecutionPhase.DIAGNOSTICS;
+  state.currentContext = { projectId: projectId2, evalRunId: evalRunId2, phase: "diagnostics" };
+  console.error("[DEBUG-H1.5] Running environment diagnostics...");
+  try {
+    const diagnosticReport = await runDiagnostics(config, evalRunId2);
+    console.error(
+      "[DEBUG-H1.5] Diagnostics completed",
+      JSON.stringify({
+        passed: diagnosticReport.summary.passed,
+        failed: diagnosticReport.summary.failed,
+        total: diagnosticReport.summary.total,
+        durationMs: diagnosticReport.totalDurationMs
+      })
+    );
+    const failedTests = diagnosticReport.tests.filter((t) => !t.passed);
+    if (failedTests.length > 0) {
+      console.error(
+        "[DEBUG-H1.5] FAILED DIAGNOSTIC TESTS:",
+        failedTests.map((t) => `${t.name}: ${t.error}`).join("\n")
+      );
+    }
+  } catch (diagErr) {
+    console.error(
+      "[DEBUG-H1.5] Diagnostics failed (non-fatal):",
+      diagErr instanceof Error ? diagErr.message : String(diagErr)
+    );
+  }
   state.currentPhase = ExecutionPhase.FETCH_EVAL_RUN;
   state.currentContext = { projectId: projectId2, evalRunId: evalRunId2, serverUrl: config.serverUrl };
   console.error(
@@ -7072,7 +7743,7 @@ async function runEvaluation(projectId2, evalRunId2) {
   };
   try {
     await api.updateEvalRun(projectId2, evalRunId2, {
-      status: import_evalforge_types4.EvalStatus.COMPLETED,
+      status: import_evalforge_types5.EvalStatus.COMPLETED,
       completedAt: (/* @__PURE__ */ new Date()).toISOString()
     });
   } catch (updateErr) {
@@ -7113,7 +7784,7 @@ runEvaluation(projectId, evalRunId).then(() => {
       authToken: config.authToken
     });
     await api.updateEvalRun(projectId, evalRunId, {
-      status: import_evalforge_types4.EvalStatus.FAILED,
+      status: import_evalforge_types5.EvalStatus.FAILED,
       completedAt: (/* @__PURE__ */ new Date()).toISOString(),
       jobError,
       jobStatus: "FAILED"
@@ -7136,7 +7807,7 @@ runEvaluation(projectId, evalRunId).then(() => {
           authToken
         });
         await api.updateEvalRun(projectId, evalRunId, {
-          status: import_evalforge_types4.EvalStatus.FAILED,
+          status: import_evalforge_types5.EvalStatus.FAILED,
           completedAt: (/* @__PURE__ */ new Date()).toISOString(),
           jobError: `Config load failed, then: ${jobError}`,
           jobStatus: "FAILED"