@wix/evalforge-evaluator 0.13.0 → 0.15.0

package/build/index.js

@@ -24,7 +24,7 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
 ));
 
 // src/index.ts
-var import_evalforge_types4 = require("@wix/evalforge-types");
+var import_evalforge_types5 = require("@wix/evalforge-types");
 
 // src/config.ts
 function loadConfig() {
@@ -87,8 +87,8 @@ function createApiClient(serverUrl, options = "") {
     }
     return headers;
   }
-  async function fetchJson(path9) {
-    const url = `${serverUrl}${apiPrefix}${pathPrefix}${path9}`;
+  async function fetchJson(path10) {
+    const url = `${serverUrl}${apiPrefix}${pathPrefix}${path10}`;
     console.error(`[API] GET ${url}`);
     const headers = buildHeaders();
     const response = await fetch(url, {
@@ -102,8 +102,8 @@ function createApiClient(serverUrl, options = "") {
     }
     return response.json();
   }
-  async function postJson(path9, body) {
-    const url = `${serverUrl}${apiPrefix}${pathPrefix}${path9}`;
+  async function postJson(path10, body) {
+    const url = `${serverUrl}${apiPrefix}${pathPrefix}${path10}`;
     console.error(`[API] POST ${url}`);
     const response = await fetch(url, {
       method: "POST",
@@ -117,8 +117,8 @@ function createApiClient(serverUrl, options = "") {
       );
     }
   }
-  async function deleteRequest(path9) {
-    const url = `${serverUrl}${apiPrefix}${pathPrefix}${path9}`;
+  async function deleteRequest(path10) {
+    const url = `${serverUrl}${apiPrefix}${pathPrefix}${path10}`;
     console.error(`[API] DELETE ${url}`);
     const headers = buildHeaders();
     const response = await fetch(url, {
@@ -132,8 +132,8 @@ function createApiClient(serverUrl, options = "") {
       );
     }
   }
-  async function putJson(path9, body) {
-    const url = `${serverUrl}${apiPrefix}${pathPrefix}${path9}`;
+  async function putJson(path10, body) {
+    const url = `${serverUrl}${apiPrefix}${pathPrefix}${path10}`;
     console.error(`[API] PUT ${url}`);
     const response = await fetch(url, {
       method: "PUT",
@@ -1157,17 +1157,17 @@ var ReadStream = class extends Minipass {
   [_size];
   [_remain];
   [_autoClose];
-  constructor(path9, opt) {
+  constructor(path10, opt) {
     opt = opt || {};
     super(opt);
     this.readable = true;
     this.writable = false;
-    if (typeof path9 !== "string") {
+    if (typeof path10 !== "string") {
       throw new TypeError("path must be a string");
     }
     this[_errored] = false;
     this[_fd] = typeof opt.fd === "number" ? opt.fd : void 0;
-    this[_path] = path9;
+    this[_path] = path10;
     this[_readSize] = opt.readSize || 16 * 1024 * 1024;
     this[_reading] = false;
     this[_size] = typeof opt.size === "number" ? opt.size : Infinity;
@@ -1330,10 +1330,10 @@ var WriteStream = class extends import_events.default {
   [_flags];
   [_finished] = false;
   [_pos];
-  constructor(path9, opt) {
+  constructor(path10, opt) {
     opt = opt || {};
     super(opt);
-    this[_path] = path9;
+    this[_path] = path10;
     this[_fd] = typeof opt.fd === "number" ? opt.fd : void 0;
     this[_mode] = opt.mode === void 0 ? 438 : opt.mode;
     this[_pos] = typeof opt.start === "number" ? opt.start : void 0;
@@ -2226,10 +2226,10 @@ var Header = class {
     }
     const prefixSize = this.ctime || this.atime ? 130 : 155;
     const split = splitPrefix(this.path || "", prefixSize);
-    const path9 = split[0];
+    const path10 = split[0];
     const prefix = split[1];
     this.needPax = !!split[2];
-    this.needPax = encString(buf, off, 100, path9) || this.needPax;
+    this.needPax = encString(buf, off, 100, path10) || this.needPax;
     this.needPax = encNumber(buf, off + 100, 8, this.mode) || this.needPax;
     this.needPax = encNumber(buf, off + 108, 8, this.uid) || this.needPax;
     this.needPax = encNumber(buf, off + 116, 8, this.gid) || this.needPax;
@@ -3205,16 +3205,16 @@ var modeFix = (mode, isDir, portable) => {
 // ../../node_modules/tar/dist/esm/strip-absolute-path.js
 var import_node_path3 = require("node:path");
 var { isAbsolute, parse: parse3 } = import_node_path3.win32;
-var stripAbsolutePath = (path9) => {
+var stripAbsolutePath = (path10) => {
   let r = "";
-  let parsed = parse3(path9);
-  while (isAbsolute(path9) || parsed.root) {
-    const root = path9.charAt(0) === "/" && path9.slice(0, 4) !== "//?/" ? "/" : parsed.root;
-    path9 = path9.slice(root.length);
+  let parsed = parse3(path10);
+  while (isAbsolute(path10) || parsed.root) {
+    const root = path10.charAt(0) === "/" && path10.slice(0, 4) !== "//?/" ? "/" : parsed.root;
+    path10 = path10.slice(root.length);
     r += root;
-    parsed = parse3(path9);
+    parsed = parse3(path10);
   }
-  return [r, path9];
+  return [r, path10];
 };
 
 // ../../node_modules/tar/dist/esm/winchars.js
@@ -3226,12 +3226,12 @@ var encode2 = (s) => raw.reduce((s2, c) => s2.split(c).join(toWin.get(c)), s);
 var decode = (s) => win.reduce((s2, c) => s2.split(c).join(toRaw.get(c)), s);
 
 // ../../node_modules/tar/dist/esm/write-entry.js
-var prefixPath = (path9, prefix) => {
+var prefixPath = (path10, prefix) => {
   if (!prefix) {
-    return normalizeWindowsPath(path9);
+    return normalizeWindowsPath(path10);
   }
-  path9 = normalizeWindowsPath(path9).replace(/^\.(\/|$)/, "");
-  return stripTrailingSlashes(prefix) + "/" + path9;
+  path10 = normalizeWindowsPath(path10).replace(/^\.(\/|$)/, "");
+  return stripTrailingSlashes(prefix) + "/" + path10;
 };
 var maxReadSize = 16 * 1024 * 1024;
 var PROCESS = /* @__PURE__ */ Symbol("process");
@@ -3376,8 +3376,8 @@ var WriteEntry = class extends Minipass {
   [MODE](mode) {
     return modeFix(mode, this.type === "Directory", this.portable);
   }
-  [PREFIX](path9) {
-    return prefixPath(path9, this.prefix);
+  [PREFIX](path10) {
+    return prefixPath(path10, this.prefix);
   }
   [HEADER]() {
     if (!this.stat) {
@@ -3758,8 +3758,8 @@ var WriteEntryTar = class extends Minipass {
     super.write(b);
     readEntry.pipe(this);
   }
-  [PREFIX](path9) {
-    return prefixPath(path9, this.prefix);
+  [PREFIX](path10) {
+    return prefixPath(path10, this.prefix);
   }
   [MODE](mode) {
     return modeFix(mode, this.type === "Directory", this.portable);
@@ -4183,8 +4183,8 @@ var PackJob = class {
   pending = false;
   ignore = false;
   piped = false;
-  constructor(path9, absolute) {
-    this.path = path9 || "./";
+  constructor(path10, absolute) {
+    this.path = path10 || "./";
     this.absolute = absolute;
   }
 };
@@ -4312,21 +4312,21 @@ var Pack = class extends Minipass {
   [WRITE](chunk) {
     return super.write(chunk);
   }
-  add(path9) {
-    this.write(path9);
+  add(path10) {
+    this.write(path10);
     return this;
   }
-  end(path9, encoding, cb) {
-    if (typeof path9 === "function") {
-      cb = path9;
-      path9 = void 0;
+  end(path10, encoding, cb) {
+    if (typeof path10 === "function") {
+      cb = path10;
+      path10 = void 0;
     }
     if (typeof encoding === "function") {
       cb = encoding;
       encoding = void 0;
     }
-    if (path9) {
-      this.add(path9);
+    if (path10) {
+      this.add(path10);
     }
     this[ENDED2] = true;
     this[PROCESS2]();
@@ -4334,14 +4334,14 @@ var Pack = class extends Minipass {
       cb();
     return this;
   }
-  write(path9) {
+  write(path10) {
     if (this[ENDED2]) {
       throw new Error("write after end");
     }
-    if (path9 instanceof ReadEntry) {
-      this[ADDTARENTRY](path9);
+    if (path10 instanceof ReadEntry) {
+      this[ADDTARENTRY](path10);
     } else {
-      this[ADDFSENTRY](path9);
+      this[ADDFSENTRY](path10);
     }
     return this.flowing;
   }
@@ -4684,9 +4684,9 @@ var getWriteFlag = !fMapEnabled ? () => "w" : (size) => size < fMapLimit ? fMapF
 // ../../node_modules/chownr/dist/esm/index.js
 var import_node_fs2 = __toESM(require("node:fs"), 1);
 var import_node_path5 = __toESM(require("node:path"), 1);
-var lchownSync = (path9, uid, gid) => {
+var lchownSync = (path10, uid, gid) => {
   try {
-    return import_node_fs2.default.lchownSync(path9, uid, gid);
+    return import_node_fs2.default.lchownSync(path10, uid, gid);
   } catch (er) {
     if (er?.code !== "ENOENT")
       throw er;
@@ -4769,9 +4769,9 @@ var CwdError = class extends Error {
   path;
   code;
   syscall = "chdir";
-  constructor(path9, code2) {
-    super(`${code2}: Cannot cd into '${path9}'`);
-    this.path = path9;
+  constructor(path10, code2) {
+    super(`${code2}: Cannot cd into '${path10}'`);
+    this.path = path10;
     this.code = code2;
   }
   get name() {
@@ -4785,10 +4785,10 @@ var SymlinkError = class extends Error {
   symlink;
   syscall = "symlink";
   code = "TAR_SYMLINK_ERROR";
-  constructor(symlink, path9) {
+  constructor(symlink, path10) {
     super("TAR_SYMLINK_ERROR: Cannot extract through symbolic link");
     this.symlink = symlink;
-    this.path = path9;
+    this.path = path10;
   }
   get name() {
     return "SymlinkError";
@@ -4970,13 +4970,13 @@ var normalizeUnicode = (s) => {
 // ../../node_modules/tar/dist/esm/path-reservations.js
 var platform3 = process.env.TESTING_TAR_FAKE_PLATFORM || process.platform;
 var isWindows2 = platform3 === "win32";
-var getDirs = (path9) => {
-  const dirs = path9.split("/").slice(0, -1).reduce((set, path10) => {
+var getDirs = (path10) => {
+  const dirs = path10.split("/").slice(0, -1).reduce((set, path11) => {
     const s = set[set.length - 1];
     if (s !== void 0) {
-      path10 = (0, import_node_path7.join)(s, path10);
+      path11 = (0, import_node_path7.join)(s, path11);
     }
-    set.push(path10 || "/");
+    set.push(path11 || "/");
     return set;
   }, []);
   return dirs;
@@ -4994,7 +4994,7 @@ var PathReservations = class {
     paths = isWindows2 ? ["win32 parallelization disabled"] : paths.map((p) => {
       return stripTrailingSlashes((0, import_node_path7.join)(normalizeUnicode(p))).toLowerCase();
     });
-    const dirs = new Set(paths.map((path9) => getDirs(path9)).reduce((a, b) => a.concat(b)));
+    const dirs = new Set(paths.map((path10) => getDirs(path10)).reduce((a, b) => a.concat(b)));
     this.#reservations.set(fn, { dirs, paths });
     for (const p of paths) {
       const q = this.#queues.get(p);
@@ -5027,8 +5027,8 @@ var PathReservations = class {
       throw new Error("function does not have any path reservations");
     }
     return {
-      paths: res.paths.map((path9) => this.#queues.get(path9)),
-      dirs: [...res.dirs].map((path9) => this.#queues.get(path9))
+      paths: res.paths.map((path10) => this.#queues.get(path10)),
+      dirs: [...res.dirs].map((path10) => this.#queues.get(path10))
     };
   }
   // check if fn is first in line for all its paths, and is
@@ -5056,14 +5056,14 @@ var PathReservations = class {
     }
     const { paths, dirs } = res;
     const next = /* @__PURE__ */ new Set();
-    for (const path9 of paths) {
-      const q = this.#queues.get(path9);
+    for (const path10 of paths) {
+      const q = this.#queues.get(path10);
       if (!q || q?.[0] !== fn) {
         continue;
       }
       const q0 = q[1];
       if (!q0) {
-        this.#queues.delete(path9);
+        this.#queues.delete(path10);
         continue;
       }
       q.shift();
@@ -5128,24 +5128,24 @@ var CHECKED_CWD = /* @__PURE__ */ Symbol("checkedCwd");
 var platform4 = process.env.TESTING_TAR_FAKE_PLATFORM || process.platform;
 var isWindows3 = platform4 === "win32";
 var DEFAULT_MAX_DEPTH = 1024;
-var unlinkFile = (path9, cb) => {
+var unlinkFile = (path10, cb) => {
   if (!isWindows3) {
-    return import_node_fs4.default.unlink(path9, cb);
+    return import_node_fs4.default.unlink(path10, cb);
   }
-  const name2 = path9 + ".DELETE." + (0, import_node_crypto.randomBytes)(16).toString("hex");
-  import_node_fs4.default.rename(path9, name2, (er) => {
+  const name2 = path10 + ".DELETE." + (0, import_node_crypto.randomBytes)(16).toString("hex");
+  import_node_fs4.default.rename(path10, name2, (er) => {
     if (er) {
       return cb(er);
     }
     import_node_fs4.default.unlink(name2, cb);
   });
 };
-var unlinkFileSync = (path9) => {
+var unlinkFileSync = (path10) => {
   if (!isWindows3) {
-    return import_node_fs4.default.unlinkSync(path9);
+    return import_node_fs4.default.unlinkSync(path10);
   }
-  const name2 = path9 + ".DELETE." + (0, import_node_crypto.randomBytes)(16).toString("hex");
-  import_node_fs4.default.renameSync(path9, name2);
+  const name2 = path10 + ".DELETE." + (0, import_node_crypto.randomBytes)(16).toString("hex");
+  import_node_fs4.default.renameSync(path10, name2);
   import_node_fs4.default.unlinkSync(name2);
 };
 var uint32 = (a, b, c) => a !== void 0 && a === a >>> 0 ? a : b !== void 0 && b === b >>> 0 ? b : c;
@@ -5243,24 +5243,24 @@ var Unpack = class extends Parser {
   // return false if we need to skip this file
   // return true if the field was successfully sanitized
   [STRIPABSOLUTEPATH](entry, field) {
-    const path9 = entry[field];
-    if (!path9 || this.preservePaths)
+    const path10 = entry[field];
+    if (!path10 || this.preservePaths)
       return true;
-    const parts = path9.split("/");
+    const parts = path10.split("/");
     if (parts.includes("..") || /* c8 ignore next */
     isWindows3 && /^[a-z]:\.\.$/i.test(parts[0] ?? "")) {
       this.warn("TAR_ENTRY_ERROR", `${field} contains '..'`, {
         entry,
-        [field]: path9
+        [field]: path10
       });
       return false;
     }
-    const [root, stripped] = stripAbsolutePath(path9);
+    const [root, stripped] = stripAbsolutePath(path10);
     if (root) {
       entry[field] = String(stripped);
       this.warn("TAR_ENTRY_INFO", `stripping ${root} from absolute ${field}`, {
         entry,
-        [field]: path9
+        [field]: path10
       });
     }
     return true;
@@ -6027,9 +6027,9 @@ var mtimeFilter = (opt) => {
   if (!opt.mtimeCache) {
     opt.mtimeCache = /* @__PURE__ */ new Map();
   }
-  opt.filter = filter ? (path9, stat) => filter(path9, stat) && !/* c8 ignore start */
-  ((opt.mtimeCache?.get(path9) ?? stat.mtime ?? 0) > (stat.mtime ?? 0)) : (path9, stat) => !/* c8 ignore start */
-  ((opt.mtimeCache?.get(path9) ?? stat.mtime ?? 0) > (stat.mtime ?? 0));
+  opt.filter = filter ? (path10, stat) => filter(path10, stat) && !/* c8 ignore start */
+  ((opt.mtimeCache?.get(path10) ?? stat.mtime ?? 0) > (stat.mtime ?? 0)) : (path10, stat) => !/* c8 ignore start */
+  ((opt.mtimeCache?.get(path10) ?? stat.mtime ?? 0) > (stat.mtime ?? 0));
 };
 
 // src/run-scenario/environment.ts
@@ -6324,6 +6324,37 @@ async function executeWithClaudeCode(skill, scenario, options) {
     queryOptions.mcpServers ? Object.keys(queryOptions.mcpServers) : "none"
   );
   console.log("[SDK-DEBUG] Calling SDK query()...");
+  if (traceContext) {
+    const preExecEvent = {
+      evalRunId: traceContext.evalRunId,
+      scenarioId: traceContext.scenarioId,
+      scenarioName: traceContext.scenarioName,
+      targetId: traceContext.targetId,
+      targetName: traceContext.targetName,
+      stepNumber: 0,
+      type: import_evalforge_types.LiveTraceEventType.DIAGNOSTIC,
+      outputPreview: JSON.stringify({
+        event: "pre-sdk-execution",
+        model: queryOptions.model,
+        maxTurns: queryOptions.maxTurns,
+        sdkEnv: {
+          ANTHROPIC_BASE_URL: sdkEnv.ANTHROPIC_BASE_URL,
+          hasANTHROPIC_API_KEY: !!sdkEnv.ANTHROPIC_API_KEY,
+          hasANTHROPIC_AUTH_TOKEN: !!sdkEnv.ANTHROPIC_AUTH_TOKEN,
+          hasANTHROPIC_CUSTOM_HEADERS: !!sdkEnv.ANTHROPIC_CUSTOM_HEADERS
+        },
+        timestamp: (/* @__PURE__ */ new Date()).toISOString()
+      }),
+      timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+      isComplete: false
+    };
+    emitTraceEvent(
+      preExecEvent,
+      traceContext.tracePushUrl,
+      traceContext.routeHeader,
+      traceContext.authToken
+    );
+  }
   try {
     for await (const message of query({
       prompt: scenario.triggerPrompt,
@@ -6381,10 +6412,10 @@ async function executeWithClaudeCode(skill, scenario, options) {
       console.error(errorStack);
     }
     if (sdkError && typeof sdkError === "object") {
-      const errObj = sdkError;
+      const errObj2 = sdkError;
       console.error("[SDK-ERROR] Error object properties:");
-      for (const key of Object.keys(errObj)) {
-        const value = errObj[key];
+      for (const key of Object.keys(errObj2)) {
+        const value = errObj2[key];
         if (value !== void 0 && key !== "stack") {
           try {
             const valueStr = typeof value === "object" ? JSON.stringify(value, null, 2) : String(value);
@@ -6394,7 +6425,7 @@ async function executeWithClaudeCode(skill, scenario, options) {
           }
         }
       }
-      const sdkErrorKeys = [
+      const sdkErrorKeys2 = [
         "code",
         "status",
         "statusCode",
@@ -6417,19 +6448,19 @@ async function executeWithClaudeCode(skill, scenario, options) {
         "spawnargs"
       ];
       const extraInfo = {};
-      for (const key of sdkErrorKeys) {
-        if (key in errObj && errObj[key] !== void 0) {
-          extraInfo[key] = errObj[key];
+      for (const key of sdkErrorKeys2) {
+        if (key in errObj2 && errObj2[key] !== void 0) {
+          extraInfo[key] = errObj2[key];
         }
       }
       if (Object.keys(extraInfo).length > 0) {
         console.error("[SDK-ERROR] SDK-specific error details:");
         console.error(JSON.stringify(extraInfo, null, 2));
       }
-      if (errObj.cause && typeof errObj.cause === "object") {
+      if (errObj2.cause && typeof errObj2.cause === "object") {
         console.error("[SDK-ERROR] Error cause:");
         try {
-          console.error(JSON.stringify(errObj.cause, null, 2));
+          console.error(JSON.stringify(errObj2.cause, null, 2));
         } catch {
           console.error("[SDK-ERROR] Error cause: [cannot serialize]");
         }
@@ -6453,17 +6484,92 @@ async function executeWithClaudeCode(skill, scenario, options) {
     console.error("[SDK-ERROR]   USER:", process.env.USER);
     console.error("[SDK-ERROR]   SHELL:", process.env.SHELL);
     console.error("[SDK-ERROR] ==========================================");
+    const errObj = sdkError;
+    const sdkSpecificInfo = {};
+    const sdkErrorKeys = [
+      "exitCode",
+      "stderr",
+      "stdout",
+      "signal",
+      "killed",
+      "code",
+      "status",
+      "errno",
+      "syscall",
+      "spawnargs"
+    ];
+    for (const key of sdkErrorKeys) {
+      if (errObj && key in errObj && errObj[key] !== void 0) {
+        const val = errObj[key];
+        if (typeof val === "string" && val.length > 500) {
+          sdkSpecificInfo[key] = val.substring(0, 500) + "... [truncated]";
+        } else {
+          sdkSpecificInfo[key] = val;
+        }
+      }
+    }
+    let causeInfo;
+    if (errObj?.cause && typeof errObj.cause === "object") {
+      try {
+        const causeStr = JSON.stringify(errObj.cause, null, 2);
+        causeInfo = causeStr.length > 500 ? causeStr.substring(0, 500) + "... [truncated]" : causeStr;
+      } catch {
+        causeInfo = "[cannot serialize cause]";
+      }
+    }
+    const sdkEnvDebug = {
+      ANTHROPIC_BASE_URL: sdkEnv.ANTHROPIC_BASE_URL,
+      hasANTHROPIC_API_KEY: !!sdkEnv.ANTHROPIC_API_KEY,
+      hasANTHROPIC_AUTH_TOKEN: !!sdkEnv.ANTHROPIC_AUTH_TOKEN,
+      hasANTHROPIC_CUSTOM_HEADERS: !!sdkEnv.ANTHROPIC_CUSTOM_HEADERS,
+      ANTHROPIC_CUSTOM_HEADERS_preview: sdkEnv.ANTHROPIC_CUSTOM_HEADERS ? sdkEnv.ANTHROPIC_CUSTOM_HEADERS.split("\n").map((h) => h.split(":")[0]).join(", ") : void 0
+    };
     const errorDetails = {
       messageCount,
       errorName,
       errorMessage,
       skillId: skill.id,
       scenarioId: scenario.id,
-      model: options.model || DEFAULT_MODEL
+      model: options.model || DEFAULT_MODEL,
+      sdkEnv: sdkEnvDebug,
+      sdkError: Object.keys(sdkSpecificInfo).length > 0 ? sdkSpecificInfo : void 0,
+      cause: causeInfo
     };
+    if (traceContext) {
+      const errorTraceEvent = {
+        evalRunId: traceContext.evalRunId,
+        scenarioId: traceContext.scenarioId,
+        scenarioName: traceContext.scenarioName,
+        targetId: traceContext.targetId,
+        targetName: traceContext.targetName,
+        stepNumber: traceStepNumber + 1,
+        type: import_evalforge_types.LiveTraceEventType.DIAGNOSTIC,
+        outputPreview: JSON.stringify(
+          {
+            event: "sdk-execution-failed",
+            error: errorMessage,
+            errorName,
+            messageCount,
+            sdkEnv: sdkEnvDebug,
+            sdkError: sdkSpecificInfo,
+            cause: causeInfo
+          },
+          null,
+          2
+        ).slice(0, 2e3),
+        timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+        isComplete: true
+      };
+      emitTraceEvent(
+        errorTraceEvent,
+        traceContext.tracePushUrl,
+        traceContext.routeHeader,
+        traceContext.authToken
+      );
+    }
     throw new Error(
       `Claude SDK execution failed after ${messageCount} messages: ${errorMessage}
-Details: ${JSON.stringify(errorDetails)}` + (errorStack ? `
+Details: ${JSON.stringify(errorDetails, null, 2)}` + (errorStack ? `
 Stack: ${errorStack.split("\n").slice(0, 5).join("\n")}` : "")
     );
   }
@@ -6827,6 +6933,8 @@ ${stackLines.join("\n")}`);
   return parts.join(" ");
 }
 var ExecutionPhase = {
+  /** Environment diagnostics phase (runs before execution) */
+  DIAGNOSTICS: "diagnostics",
   CONFIG: "config-loading",
   API_CLIENT: "api-client-creation",
   FETCH_EVAL_RUN: "fetch-eval-run",
@@ -6843,6 +6951,402 @@ var ExecutionPhase = {
   UPDATE_STATUS: "update-status"
 };
 
+// src/diagnostics.ts
+var import_child_process = require("child_process");
+var fs11 = __toESM(require("fs"));
+var path9 = __toESM(require("path"));
+var import_evalforge_types4 = require("@wix/evalforge-types");
+async function execCommand(command, timeoutMs = 5e3) {
+  return new Promise((resolve) => {
+    const proc2 = (0, import_child_process.spawn)("sh", ["-c", command], {
+      timeout: timeoutMs
+    });
+    let stdout = "";
+    let stderr = "";
+    proc2.stdout.on("data", (data) => {
+      stdout += data.toString();
+    });
+    proc2.stderr.on("data", (data) => {
+      stderr += data.toString();
+    });
+    proc2.on("close", (code2) => {
+      resolve({
+        stdout: stdout.trim(),
+        stderr: stderr.trim(),
+        exitCode: code2 ?? -1
+      });
+    });
+    proc2.on("error", (err) => {
+      resolve({
+        stdout: "",
+        stderr: err.message,
+        exitCode: -1
+      });
+    });
+  });
+}
+async function testEnvironmentVariables() {
+  const start = Date.now();
+  const envVars = [
+    "PATH",
+    "HOME",
+    "USER",
+    "SHELL",
+    "NODE_ENV",
+    "EVAL_SERVER_URL",
+    "AI_GATEWAY_URL",
+    "ANTHROPIC_API_KEY",
+    "ANTHROPIC_AUTH_TOKEN",
+    "ANTHROPIC_BASE_URL",
+    "ANTHROPIC_CUSTOM_HEADERS"
+  ];
+  const details = {};
+  const missing = [];
+  for (const key of envVars) {
+    const value = process.env[key];
+    if (value) {
+      if (key.includes("SECRET") || key.includes("TOKEN") || key.includes("API_KEY")) {
+        details[key] = `[SET - ${value.length} chars]`;
+      } else if (key === "PATH") {
+        details[key] = value.split(":");
+      } else if (key === "ANTHROPIC_CUSTOM_HEADERS") {
+        details[key] = value.split("\n").map((h) => h.split(":")[0]).join(", ");
+      } else {
+        details[key] = value;
+      }
+    } else {
+      missing.push(key);
+    }
+  }
+  details.missingEnvVars = missing;
+  return {
+    name: "environment-variables",
+    passed: true,
+    // Info only, doesn't fail
+    details,
+    durationMs: Date.now() - start
+  };
+}
+async function testNodeEnvironment() {
+  const start = Date.now();
+  const details = {
+    nodeVersion: process.version,
+    platform: process.platform,
+    arch: process.arch,
+    cwd: process.cwd(),
+    pid: process.pid,
+    uptime: process.uptime(),
+    memoryUsage: process.memoryUsage(),
+    execPath: process.execPath
+  };
+  return {
+    name: "node-environment",
+    passed: true,
+    details,
+    durationMs: Date.now() - start
+  };
+}
+async function testNpmGlobalDirectory() {
+  const start = Date.now();
+  const npmRootResult = await execCommand("npm root -g");
+  const npmBinResult = await execCommand("npm bin -g");
+  const npmRoot = npmRootResult.stdout;
+  const npmBin = npmBinResult.stdout;
+  const details = {
+    npmRootGlobal: npmRoot,
+    npmBinGlobal: npmBin,
+    npmRootExitCode: npmRootResult.exitCode,
+    npmBinExitCode: npmBinResult.exitCode
+  };
+  if (npmRoot) {
+    const evaluatorPath = path9.join(npmRoot, "@wix", "evalforge-evaluator");
+    const evaluatorExists = fs11.existsSync(evaluatorPath);
+    details.evaluatorInstalled = evaluatorExists;
+    if (evaluatorExists) {
+      try {
+        const files = fs11.readdirSync(evaluatorPath);
+        details.evaluatorFiles = files;
+      } catch {
+        details.evaluatorFiles = "Failed to list files";
+      }
+    }
+  }
+  if (npmRoot) {
+    const sdkPath = path9.join(
+      npmRoot,
+      "@wix",
+      "evalforge-evaluator",
+      "node_modules",
+      "@anthropic-ai",
+      "claude-agent-sdk"
+    );
+    const sdkExists = fs11.existsSync(sdkPath);
+    details.claudeAgentSdkInstalled = sdkExists;
+  }
+  const passed = npmRootResult.exitCode === 0 && npmBinResult.exitCode === 0;
+  return {
+    name: "npm-global-directory",
+    passed,
+    details,
+    error: passed ? void 0 : npmRootResult.stderr || npmBinResult.stderr,
+    durationMs: Date.now() - start
+  };
+}
+async function testClaudeBinary() {
+  const start = Date.now();
+  const whichResult = await execCommand("which claude");
+  const versionResult = await execCommand("claude --version");
+  const npmBinResult = await execCommand("npm bin -g");
+  const npmBin = npmBinResult.stdout;
+  let claudeInNpmBin = false;
+  if (npmBin) {
+    const claudePath = path9.join(npmBin, "claude");
+    claudeInNpmBin = fs11.existsSync(claudePath);
+  }
+  const details = {
+    whichClaude: whichResult.stdout || "(not found)",
+    whichExitCode: whichResult.exitCode,
+    claudeVersion: versionResult.stdout || versionResult.stderr,
+    versionExitCode: versionResult.exitCode,
+    claudeInNpmGlobalBin: claudeInNpmBin,
+    npmGlobalBin: npmBin
+  };
+  const pathDirs = (process.env.PATH || "").split(":");
+  const claudeFoundIn = [];
+  for (const dir of pathDirs) {
+    const claudePath = path9.join(dir, "claude");
+    if (fs11.existsSync(claudePath)) {
+      claudeFoundIn.push(dir);
+    }
+  }
+  details.claudeFoundInPathDirs = claudeFoundIn;
+  const passed = whichResult.exitCode === 0 || claudeInNpmBin;
+  return {
+    name: "claude-cli-binary",
+    passed,
+    details,
+    error: passed ? void 0 : "Claude CLI binary not found in PATH. The SDK will fail to spawn it.",
+    durationMs: Date.now() - start
+  };
+}
+async function testChildProcess() {
+  const start = Date.now();
+  const echoResult = await execCommand('echo "diagnostic-test-success"');
+  const nodeResult = await execCommand('node -e "console.log(process.pid)"');
+  const details = {
+    echoResult: echoResult.stdout,
+    echoExitCode: echoResult.exitCode,
+    nodeResult: nodeResult.stdout,
+    nodeExitCode: nodeResult.exitCode
+  };
+  const passed = echoResult.exitCode === 0 && echoResult.stdout === "diagnostic-test-success";
+  return {
+    name: "child-process-spawning",
+    passed,
+    details,
+    error: passed ? void 0 : "Failed to spawn child process",
+    durationMs: Date.now() - start
+  };
+}
+async function testNetworkConnectivity(config) {
+  const start = Date.now();
+  const aiGatewayUrl = config.aiGatewayUrl;
+  if (!aiGatewayUrl) {
+    return {
+      name: "network-connectivity",
+      passed: false,
+      details: { error: "No AI_GATEWAY_URL configured" },
+      error: "No AI_GATEWAY_URL configured",
+      durationMs: Date.now() - start
+    };
+  }
+  const curlResult = await execCommand(
+    `curl -s -o /dev/null -w "%{http_code}" --connect-timeout 5 "${aiGatewayUrl}" 2>&1`
+  );
+  const serverUrl = config.serverUrl;
+  const serverResult = await execCommand(
+    `curl -s -o /dev/null -w "%{http_code}" --connect-timeout 5 "${serverUrl}/health" 2>&1`
+  );
+  const details = {
+    aiGatewayUrl,
+    aiGatewayHttpCode: curlResult.stdout,
+    aiGatewayExitCode: curlResult.exitCode,
+    serverUrl,
+    serverHttpCode: serverResult.stdout,
+    serverExitCode: serverResult.exitCode
+  };
+  const gatewayReachable = curlResult.exitCode === 0 && curlResult.stdout !== "000";
+  return {
+    name: "network-connectivity",
+    passed: gatewayReachable,
+    details,
+    error: gatewayReachable ? void 0 : "Failed to reach AI Gateway",
+    durationMs: Date.now() - start
+  };
+}
+async function testSdkImport() {
+  const start = Date.now();
+  try {
+    const sdk = await import("@anthropic-ai/claude-agent-sdk");
+    const details = {
+      sdkImported: true,
+      hasQuery: typeof sdk.query === "function",
+      exportedKeys: Object.keys(sdk)
+    };
+    return {
+      name: "sdk-import",
+      passed: true,
+      details,
+      durationMs: Date.now() - start
+    };
+  } catch (err) {
+    const error = err instanceof Error ? err.message : String(err);
+    return {
+      name: "sdk-import",
+      passed: false,
+      details: {
+        sdkImported: false,
+        error
+      },
+      error: `Failed to import Claude Agent SDK: ${error}`,
+      durationMs: Date.now() - start
+    };
+  }
+}
+async function testFileSystemAccess() {
+  const start = Date.now();
+  const testDir = "/tmp/evalforge-diagnostics";
+  const testFile = path9.join(testDir, "test.txt");
+  try {
+    if (!fs11.existsSync(testDir)) {
+      fs11.mkdirSync(testDir, { recursive: true });
+    }
+    fs11.writeFileSync(testFile, "diagnostic-test");
+    const content = fs11.readFileSync(testFile, "utf8");
+    fs11.unlinkSync(testFile);
+    fs11.rmdirSync(testDir);
+    const details = {
+      canCreateDirectory: true,
+      canWriteFile: true,
+      canReadFile: content === "diagnostic-test",
+      testDir,
+      cwd: process.cwd(),
+      cwdContents: fs11.readdirSync(process.cwd()).slice(0, 20)
+      // First 20 files
+    };
+    return {
+      name: "file-system-access",
+      passed: true,
+      details,
+      durationMs: Date.now() - start
+    };
+  } catch (err) {
+    const error = err instanceof Error ? err.message : String(err);
+    return {
+      name: "file-system-access",
+      passed: false,
+      details: {
+        error,
+        testDir,
+        cwd: process.cwd()
+      },
+      error: `File system access failed: ${error}`,
+      durationMs: Date.now() - start
+    };
+  }
+}
+function emitDiagnosticTraceEvent(evalRunId2, result, tracePushUrl, routeHeader, authToken) {
+  const event = {
+    evalRunId: evalRunId2,
+    scenarioId: "diagnostics",
+    scenarioName: "Environment Diagnostics",
+    targetId: "system",
+    targetName: "System",
+    stepNumber: 0,
+    type: import_evalforge_types4.LiveTraceEventType.DIAGNOSTIC,
+    outputPreview: JSON.stringify(result, null, 2).slice(0, 2e3),
+    // Limit size
+    timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+    isComplete: "summary" in result
+    // Complete if it's the full report
+  };
+  console.log(`TRACE_EVENT:${JSON.stringify(event)}`);
+  if (tracePushUrl) {
+    const headers = {
+      "Content-Type": "application/json"
+    };
+    if (routeHeader) {
+      headers["x-wix-route"] = routeHeader;
+    }
+    if (authToken) {
+      headers["Authorization"] = `Bearer ${authToken}`;
+    }
+    fetch(tracePushUrl, {
+      method: "POST",
+      headers,
+      body: JSON.stringify([event])
+    }).catch((err) => {
+      console.error("[DIAGNOSTICS] Failed to push trace event:", err);
+    });
+  }
+}
+async function runDiagnostics(config, evalRunId2) {
+  const startedAt = (/* @__PURE__ */ new Date()).toISOString();
+  const startTime = Date.now();
+  console.error("[DIAGNOSTICS] Starting environment diagnostics...");
+  const tests = [];
+  const runTest = async (testFn) => {
+    const result = await testFn();
+    tests.push(result);
+    emitDiagnosticTraceEvent(
+      evalRunId2,
+      result,
+      config.tracePushUrl,
+      config.routeHeader,
+      config.authToken
+    );
+    const status = result.passed ? "\u2713" : "\u2717";
+    console.error(
+      `[DIAGNOSTICS] ${status} ${result.name} (${result.durationMs}ms)`
+    );
+    if (!result.passed && result.error) {
+      console.error(`[DIAGNOSTICS]   Error: ${result.error}`);
+    }
+  };
+  await runTest(testEnvironmentVariables);
+  await runTest(testNodeEnvironment);
+  await runTest(testNpmGlobalDirectory);
+  await runTest(testClaudeBinary);
+  await runTest(testChildProcess);
+  await runTest(() => testNetworkConnectivity(config));
+  await runTest(testSdkImport);
+  await runTest(testFileSystemAccess);
+  const completedAt = (/* @__PURE__ */ new Date()).toISOString();
+  const totalDurationMs = Date.now() - startTime;
+  const report = {
+    startedAt,
+    completedAt,
+    totalDurationMs,
+    tests,
+    summary: {
+      total: tests.length,
+      passed: tests.filter((t) => t.passed).length,
+      failed: tests.filter((t) => !t.passed).length
+    }
+  };
+  emitDiagnosticTraceEvent(
+    evalRunId2,
+    report,
+    config.tracePushUrl,
+    config.routeHeader,
+    config.authToken
+  );
+  console.error(
+    `[DIAGNOSTICS] Completed: ${report.summary.passed}/${report.summary.total} tests passed (${totalDurationMs}ms)`
+  );
+  return report;
+}
+
 // src/index.ts
 console.error(
   "[EVALUATOR-BOOT] Module loading started",
@@ -6910,6 +7414,33 @@ async function runEvaluation(projectId2, evalRunId2) {
       `[${ExecutionPhase.API_CLIENT}] Failed to create API client: ${apiErr instanceof Error ? apiErr.message : String(apiErr)}`
     );
   }
+  state.currentPhase = ExecutionPhase.DIAGNOSTICS;
+  state.currentContext = { projectId: projectId2, evalRunId: evalRunId2, phase: "diagnostics" };
+  console.error("[DEBUG-H1.5] Running environment diagnostics...");
+  try {
+    const diagnosticReport = await runDiagnostics(config, evalRunId2);
+    console.error(
+      "[DEBUG-H1.5] Diagnostics completed",
+      JSON.stringify({
+        passed: diagnosticReport.summary.passed,
+        failed: diagnosticReport.summary.failed,
+        total: diagnosticReport.summary.total,
+        durationMs: diagnosticReport.totalDurationMs
+      })
+    );
+    const failedTests = diagnosticReport.tests.filter((t) => !t.passed);
+    if (failedTests.length > 0) {
+      console.error(
+        "[DEBUG-H1.5] FAILED DIAGNOSTIC TESTS:",
+        failedTests.map((t) => `${t.name}: ${t.error}`).join("\n")
+      );
+    }
+  } catch (diagErr) {
+    console.error(
+      "[DEBUG-H1.5] Diagnostics failed (non-fatal):",
+      diagErr instanceof Error ? diagErr.message : String(diagErr)
+    );
+  }
   state.currentPhase = ExecutionPhase.FETCH_EVAL_RUN;
   state.currentContext = { projectId: projectId2, evalRunId: evalRunId2, serverUrl: config.serverUrl };
   console.error(
@@ -7029,7 +7560,7 @@ async function runEvaluation(projectId2, evalRunId2) {
   };
   try {
     await api.updateEvalRun(projectId2, evalRunId2, {
-      status: import_evalforge_types4.EvalStatus.COMPLETED,
+      status: import_evalforge_types5.EvalStatus.COMPLETED,
       completedAt: (/* @__PURE__ */ new Date()).toISOString()
     });
   } catch (updateErr) {
@@ -7070,7 +7601,7 @@ runEvaluation(projectId, evalRunId).then(() => {
       authToken: config.authToken
     });
     await api.updateEvalRun(projectId, evalRunId, {
-      status: import_evalforge_types4.EvalStatus.FAILED,
+      status: import_evalforge_types5.EvalStatus.FAILED,
       completedAt: (/* @__PURE__ */ new Date()).toISOString(),
       jobError,
       jobStatus: "FAILED"
@@ -7093,7 +7624,7 @@ runEvaluation(projectId, evalRunId).then(() => {
           authToken
         });
         await api.updateEvalRun(projectId, evalRunId, {
-          status: import_evalforge_types4.EvalStatus.FAILED,
+          status: import_evalforge_types5.EvalStatus.FAILED,
           completedAt: (/* @__PURE__ */ new Date()).toISOString(),
           jobError: `Config load failed, then: ${jobError}`,
           jobStatus: "FAILED"