@wix/evalforge-evaluator 0.186.0 → 0.188.0

package/README.md

@@ -50,6 +50,10 @@ Backend calls go through the evalforge ambassador packages (gRPC via
 user-editable fields, not the system state transitions
 (`status`/`completedAt`/`jobError`/`jobStatus`) the evaluator writes.
 
+## Live trace during environment setup
+
+For templated runs, the evaluator emits `PROGRESS` trace events during environment setup — "Setting up environment", "Fetching template files", "Installing dependencies", "Environment ready" — via the shared `emitTraceEvent` helper. Because `emitTraceEvent` writes to stdout (captured by the backend for local runs) and also calls the `pushEvent` callback (used for remote jobs via `tracePushUrl`), these events appear in the live trace in both local and remote runs. Without them, the trace panel stays blank during the often multi-minute setup phase before the agent starts.
+
 ## Scripts
 
 ```bash

package/build/index.js

@@ -1277,7 +1277,7 @@ var require_error = __commonJS({
       "toJSON",
       "__CANCEL__"
     ];
-    var HttpError = class _HttpError extends Error {
+    var HttpError2 = class _HttpError extends Error {
       constructor(error) {
         var _a;
         super(error.message);
@@ -1294,9 +1294,9 @@ var require_error = __commonJS({
         return (0, headers_1.requestIdOrEmptyString)(this.response);
       }
     };
-    exports2.HttpError = HttpError;
+    exports2.HttpError = HttpError2;
     function createHttpError(...args) {
-      return new HttpError(...args);
+      return new HttpError2(...args);
     }
     exports2.createHttpError = createHttpError;
   }
@@ -6351,6 +6351,133 @@ function getLatestCapabilityVersion(payload) {
   return __getLatestCapabilityVersion;
 }
 
+// ../../node_modules/@wix/ambassador-evalforge-v1-site-provisioning/es/build/http.impl.js
+var _deleteProvisionedSiteRequest = {};
+var _deleteProvisionedSiteResponse = {};
+var _provisionScenarioSiteRequest = {};
+var _provisionScenarioSiteResponse = {};
+function resolveWixEvalforgeV1SiteProvisioningServiceUrl(opts) {
+  var domainToMappings = {
+    "dev._base_domain_": [
+      {
+        srcPath: "/_api/evalforge-backend",
+        destPath: ""
+      }
+    ],
+    "api._api_base_domain_": [
+      {
+        srcPath: "/evalforge-backend",
+        destPath: ""
+      }
+    ],
+    "bo._base_domain_": [
+      {
+        srcPath: "/_api/evalforge-backend",
+        destPath: ""
+      }
+    ],
+    "wixbo.ai": [
+      {
+        srcPath: "/_api/evalforge-backend",
+        destPath: ""
+      }
+    ],
+    "wix-bo.com": [
+      {
+        srcPath: "/_api/evalforge-backend",
+        destPath: ""
+      }
+    ],
+    "manage._base_domain_": [
+      {
+        srcPath: "/_api/evalforge-backend",
+        destPath: ""
+      }
+    ]
+  };
+  return resolveUrl(Object.assign(opts, { domainToMappings }));
+}
+function provisionScenarioSite(payload) {
+  var _a = serializer(_provisionScenarioSiteRequest, {}), toReq = _a.toJSON, fromReq = _a.fromJSON;
+  var fromRes = serializer(_provisionScenarioSiteResponse, {}).fromJSON;
+  function __provisionScenarioSite(_a2) {
+    var host = _a2.host;
+    var serializedData = toReq(payload);
+    var metadata = {
+      entityFqdn: "wix.evalforge.v1.site_provisioning",
+      method: "POST",
+      methodFqn: "wix.evalforge.v1.SiteProvisioningService.ProvisionScenarioSite",
+      migrationOptions: {
+        optInTransformResponse: true
+      },
+      url: resolveWixEvalforgeV1SiteProvisioningServiceUrl({
+        protoPath: "/v1/projects/{projectId}/site-provisioning/provision-site",
+        data: serializedData,
+        host
+      }),
+      data: serializedData,
+      transformResponse: fromRes
+    };
+    return metadata;
+  }
+  __provisionScenarioSite.fromReq = fromReq;
+  __provisionScenarioSite.__isAmbassador = true;
+  return __provisionScenarioSite;
+}
+function deleteProvisionedSite(payload) {
+  var _a = serializer(_deleteProvisionedSiteRequest, {}), toReq = _a.toJSON, fromReq = _a.fromJSON;
+  var fromRes = serializer(_deleteProvisionedSiteResponse, {}).fromJSON;
+  function __deleteProvisionedSite(_a2) {
+    var host = _a2.host;
+    var serializedData = toReq(payload);
+    var metadata = {
+      entityFqdn: "wix.evalforge.v1.site_provisioning",
+      method: "POST",
+      methodFqn: "wix.evalforge.v1.SiteProvisioningService.DeleteProvisionedSite",
+      migrationOptions: {
+        optInTransformResponse: true
+      },
+      url: resolveWixEvalforgeV1SiteProvisioningServiceUrl({
+        protoPath: "/v1/projects/{projectId}/site-provisioning/delete-site",
+        data: serializedData,
+        host
+      }),
+      data: serializedData,
+      transformResponse: fromRes
+    };
+    return metadata;
+  }
+  __deleteProvisionedSite.fromReq = fromReq;
+  __deleteProvisionedSite.__isAmbassador = true;
+  return __deleteProvisionedSite;
+}
+
+// ../../node_modules/@wix/ambassador-evalforge-v1-test-scenario/es/build/types.impl.js
+var WebhookIdentityType;
+(function(WebhookIdentityType2) {
+  WebhookIdentityType2["UNKNOWN"] = "UNKNOWN";
+  WebhookIdentityType2["ANONYMOUS_VISITOR"] = "ANONYMOUS_VISITOR";
+  WebhookIdentityType2["MEMBER"] = "MEMBER";
+  WebhookIdentityType2["WIX_USER"] = "WIX_USER";
+  WebhookIdentityType2["APP"] = "APP";
+})(WebhookIdentityType || (WebhookIdentityType = {}));
+var SiteBootstrapHttpMethod;
+(function(SiteBootstrapHttpMethod2) {
+  SiteBootstrapHttpMethod2["SITE_BOOTSTRAP_HTTP_METHOD_UNSPECIFIED"] = "SITE_BOOTSTRAP_HTTP_METHOD_UNSPECIFIED";
+  SiteBootstrapHttpMethod2["GET"] = "GET";
+  SiteBootstrapHttpMethod2["POST"] = "POST";
+  SiteBootstrapHttpMethod2["PUT"] = "PUT";
+  SiteBootstrapHttpMethod2["PATCH"] = "PATCH";
+  SiteBootstrapHttpMethod2["DELETE"] = "DELETE";
+})(SiteBootstrapHttpMethod || (SiteBootstrapHttpMethod = {}));
+var Mode;
+(function(Mode2) {
+  Mode2["UNKNOWN_MODE"] = "UNKNOWN_MODE";
+  Mode2["NONE"] = "NONE";
+  Mode2["CLONE"] = "CLONE";
+  Mode2["TEMPLATE"] = "TEMPLATE";
+})(Mode || (Mode = {}));
+
 // src/ambassador-converters.ts
 function toProtoEnum(prefix, value) {
   return `${prefix}${value.toUpperCase()}`;
@@ -6657,9 +6784,37 @@ function testScenarioFromProto(wire) {
       })
     ),
     createdAt: fromProtoDate(wire.createdAt) ?? "",
-    updatedAt: fromProtoDate(wire.updatedAt) ?? ""
+    updatedAt: fromProtoDate(wire.updatedAt) ?? "",
+    siteSetup: siteSetupFromAmbassador(wire.siteSetup)
   };
 }
+function siteSetupFromAmbassador(wire) {
+  if (!wire) return void 0;
+  const steps = (wire.bootstrap?.steps ?? []).filter(
+    (step) => step.method && step.method !== SiteBootstrapHttpMethod.SITE_BOOTSTRAP_HTTP_METHOD_UNSPECIFIED && step.url
+  ).map((step) => ({
+    label: step.label ?? void 0,
+    method: step.method.toLowerCase(),
+    url: step.url ?? "",
+    body: step.body ?? void 0
+  }));
+  const bootstrap = steps.length > 0 ? { steps } : void 0;
+  if (wire.mode === Mode.CLONE) {
+    return {
+      mode: "clone",
+      sourceSiteId: wire.cloneOptions?.sourceSiteId ?? "",
+      bootstrap
+    };
+  }
+  if (wire.mode === Mode.TEMPLATE) {
+    return {
+      mode: "template",
+      templateId: wire.templateOptions?.templateId ?? "",
+      bootstrap
+    };
+  }
+  return void 0;
+}
 function templateFromProto(wire) {
   return {
     id: wire.id ?? "",
@@ -6713,8 +6868,36 @@ function capabilityVersionFromProto(wire, projectId2) {
     createdAt: fromProtoDate(wire.createdAt) ?? ""
   };
 }
+function provisionedSiteFromProto(proto) {
+  return {
+    id: proto.id ?? "",
+    url: proto.url ?? void 0,
+    editorUrl: proto.editorUrl ?? void 0
+  };
+}
+function siteBootstrapResultFromProto(proto) {
+  if (!proto) return void 0;
+  return {
+    steps: (proto.steps ?? []).map((step) => ({
+      label: step.label ?? void 0,
+      statusCode: step.statusCode ?? 0,
+      ok: step.ok ?? false,
+      error: step.error ?? void 0
+    }))
+  };
+}
 
 // src/api-client.ts
+function rethrowWithRequestId(err, action) {
+  if (err instanceof import_http_client.HttpError) {
+    const status = err.response?.status;
+    const requestId = err.requestId;
+    throw new Error(
+      `Failed to ${action}` + (status !== void 0 ? ` (HTTP ${status})` : "") + (requestId ? ` [request id: ${requestId}]` : "") + `: ${err.message}`
+    );
+  }
+  throw err;
+}
 function resolveAmbassadorBaseUrl(serverUrl) {
   try {
     return new URL(serverUrl).origin;
@@ -6890,6 +7073,29 @@ function createApiClient(serverUrl, options = "") {
     },
     updateEvalRun(projectId2, evalRunId2, update) {
       return putJson(`/projects/${projectId2}/eval-runs/${evalRunId2}`, update);
+    },
+    async provisionScenarioSite(projectId2, evalRunId2, scenarioId) {
+      const res = await httpClient.request(provisionScenarioSite({ projectId: projectId2, evalRunId: evalRunId2, scenarioId })).catch(
+        (err) => rethrowWithRequestId(err, `provision a site for scenario ${scenarioId}`)
+      );
+      const site = res.data.site;
+      if (!site) {
+        throw new Error(
+          `Site provisioning for scenario ${scenarioId} returned no site.`
+        );
+      }
+      return {
+        ...provisionedSiteFromProto(site),
+        bootstrapResult: siteBootstrapResultFromProto(res.data.bootstrapResult)
+      };
+    },
+    async deleteProvisionedSite(projectId2, siteId) {
+      await httpClient.request(deleteProvisionedSite({ projectId: projectId2, siteId })).catch((err) => {
+        console.warn(
+          "[site-provisioning] deleteProvisionedSite failed \u2014 site may remain:",
+          err
+        );
+      });
     }
   };
 }
@@ -7256,10 +7462,13 @@ function installWithCache(workDir, exec, cacheBase, pm) {
     );
   }
 }
-async function installDependencies(workDir, exec = import_child_process.execFileSync, cacheBase) {
+async function installDependencies(workDir, onProgress, options = {}) {
   if (!(0, import_fs.existsSync)(import_path2.default.join(workDir, "package.json"))) {
     return;
   }
+  const exec = options.exec ?? import_child_process.execFileSync;
+  const cacheBase = options.cacheBase;
+  onProgress("Installing dependencies...");
   const pm = detectPackageManager(workDir);
   if (cacheBase) {
     installWithCache(workDir, exec, cacheBase, pm);
@@ -7335,7 +7544,8 @@ function writeWixEnvFile(workDir) {
     console.warn("[environment] Failed to read wix.config.json");
   }
 }
-async function prepareWorkingDirectory(config, evalRunId2, targetId, scenarioId, template) {
+async function prepareWorkingDirectory(config, evalRunId2, targetId, scenarioId, onProgress, options = {}) {
+  const template = options.template;
   const baseDir = config.evaluationsDir ?? import_path3.default.join((0, import_os.tmpdir)(), "evalforge-evaluations");
   const nodeModulesCacheDir = import_path3.default.join(baseDir, "_node_modules_cache");
   if (template) {
@@ -7349,10 +7559,14 @@ async function prepareWorkingDirectory(config, evalRunId2, targetId, scenarioId,
       (0, import_fs2.rmSync)(workDir2, { recursive: true });
     }
     (0, import_fs2.mkdirSync)(workDir2, { recursive: true });
+    onProgress("Fetching template files...");
     await fetchAndWriteTemplateFiles(template, workDir2);
     console.log(`Template files written to ${workDir2}`);
     writeWixEnvFile(workDir2);
-    await installDependencies(workDir2, void 0, nodeModulesCacheDir);
+    await installDependencies(workDir2, onProgress, {
+      cacheBase: nodeModulesCacheDir
+    });
+    onProgress("Environment ready");
     return workDir2;
   }
   const workDir = import_path3.default.join(baseDir, `${evalRunId2}_${targetId}_${scenarioId}`);
@@ -7364,6 +7578,13 @@ async function prepareWorkingDirectory(config, evalRunId2, targetId, scenarioId,
   return workDir;
 }
 
+// src/run-scenario/agents/shared/trace-emit.ts
+var import_evalforge_types2 = require("@wix/evalforge-types");
+function emitTraceEvent(event, pushEvent) {
+  console.log(`${import_evalforge_types2.TRACE_EVENT_PREFIX}${JSON.stringify(event)}`);
+  pushEvent?.(event);
+}
+
 // src/run-scenario/run-agent-with-context.ts
 var import_crypto5 = require("crypto");
 
@@ -7528,7 +7749,7 @@ var import_crypto2 = require("crypto");
 // src/run-scenario/agents/claude-code/write-mcp.ts
 var import_promises5 = require("fs/promises");
 var import_path6 = require("path");
-var import_evalforge_types2 = require("@wix/evalforge-types");
+var import_evalforge_types3 = require("@wix/evalforge-types");
 
 // src/run-scenario/agents/shared/resolve-mcp-placeholders.ts
 var import_promises4 = require("fs/promises");
@@ -7589,7 +7810,7 @@ async function writeMcpToFilesystem(cwd, mcps) {
   }
   const resolvedServers = await resolveMcpPlaceholders(mcpServers, { cwd });
   const content = JSON.stringify(
-    { [import_evalforge_types2.MCP_SERVERS_JSON_KEY]: resolvedServers },
+    { [import_evalforge_types3.MCP_SERVERS_JSON_KEY]: resolvedServers },
     null,
     2
   );
@@ -7815,13 +8036,6 @@ function buildConversation(timestampedMessages) {
   return messages;
 }
 
-// src/run-scenario/agents/shared/trace-emit.ts
-var import_evalforge_types3 = require("@wix/evalforge-types");
-function emitTraceEvent(event, pushEvent) {
-  console.log(`${import_evalforge_types3.TRACE_EVENT_PREFIX}${JSON.stringify(event)}`);
-  pushEvent?.(event);
-}
-
 // src/run-scenario/agents/claude-code/execute.ts
 var DEFAULT_MODEL = import_evalforge_types4.ClaudeModel.CLAUDE_4_5_SONNET_1_0;
 async function* buildPromptStream(triggerPrompt, images) {
@@ -11783,87 +11997,124 @@ async function runAgentWithContext(config, evalRunId2, scenario, evalData, workD
 }
 
 // src/run-scenario/index.ts
-async function runScenario(config, evalRunId2, scenario, evalData, template, resolvedAssertions, pushEvent) {
+async function runScenario(config, evalRunId2, scenario, evalData, template, resolvedAssertions, pushEvent, apiClient, projectId2) {
   const targetId = evalData.evalRun.presetId ?? evalData.agent?.id ?? evalData.evalRun.id;
   const targetName = evalData.presetName ?? evalData.agent?.name ?? "";
-  if (template) {
-    console.log(
-      (0, import_evalforge_types13.formatTraceEventLine)({
-        evalRunId: evalRunId2,
-        scenarioId: scenario.id,
-        scenarioName: scenario.name,
-        targetId,
-        targetName,
-        stepNumber: 0,
-        type: import_evalforge_types13.LiveTraceEventType.PROGRESS,
-        outputPreview: "Setting up environment (installing dependencies)...",
-        elapsedMs: 0,
-        timestamp: (/* @__PURE__ */ new Date()).toISOString(),
-        isComplete: false
-      })
-    );
-  }
-  const workDir = await prepareWorkingDirectory(
-    config,
-    evalRunId2,
-    targetId,
-    scenario.id,
-    template
-  );
-  const partialResult = await runAgentWithContext(
-    config,
-    evalRunId2,
-    scenario,
-    evalData,
-    workDir,
+  const emitSetupProgress = (outputPreview) => emitTraceEvent(
+    {
+      evalRunId: evalRunId2,
+      scenarioId: scenario.id,
+      scenarioName: scenario.name,
+      targetId,
+      targetName,
+      stepNumber: 0,
+      type: import_evalforge_types13.LiveTraceEventType.PROGRESS,
+      outputPreview,
+      elapsedMs: 0,
+      timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+      isComplete: false
+    },
     pushEvent
   );
-  const inlineAssertions = scenario.assertions ?? [];
-  const assertions = [
-    ...inlineAssertions,
-    ...resolvedAssertions ?? []
-  ];
-  const templateFilesMap = new Map(
-    (partialResult.templateFiles ?? []).map((f) => [f.path, f.status])
-  );
-  const evaluationInput = {
-    outputText: partialResult.outputText,
-    llmTrace: partialResult.llmTrace,
-    fileDiffs: partialResult.fileDiffs?.map((d) => ({
-      path: d.path,
-      status: templateFilesMap.get(d.path)
-    })),
-    durationMs: partialResult.duration
-  };
-  const defaultJudgeModel = import_evalforge_types13.DEFAULT_JUDGE_MODEL;
-  const assertionContext = {
-    workDir,
-    defaultJudgeModel,
-    llmConfig: {
-      baseUrl: config.aiGatewayUrl,
-      headers: config.aiGatewayHeaders
+  let provisionedSite;
+  if (apiClient && projectId2 && scenario.siteSetup && scenario.siteSetup.mode !== "none") {
+    provisionedSite = await apiClient.provisionScenarioSite(
+      projectId2,
+      evalRunId2,
+      scenario.id
+    );
+  }
+  const failedStep = provisionedSite?.bootstrapResult?.steps.find((s) => !s.ok);
+  if (failedStep) {
+    const message = `Site bootstrap step ${failedStep.label ? `"${failedStep.label}" ` : ""}failed (HTTP ${failedStep.statusCode}): ${failedStep.error ?? "unknown error"}`;
+    console.warn(`[run-scenario] ${message}`);
+    pushEvent?.({
+      evalRunId: evalRunId2,
+      scenarioId: scenario.id,
+      scenarioName: scenario.name,
+      targetId,
+      targetName,
+      stepNumber: 0,
+      type: import_evalforge_types13.LiveTraceEventType.PROGRESS,
+      outputPreview: message,
+      elapsedMs: 0,
+      timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+      isComplete: false
+    });
+  }
+  const effectiveTriggerPrompt = provisionedSite ? `${scenario.triggerPrompt}
+
+Site ID: ${provisionedSite.id}` : scenario.triggerPrompt;
+  try {
+    emitSetupProgress("Setting up environment...");
+    const workDir = await prepareWorkingDirectory(
+      config,
+      evalRunId2,
+      targetId,
+      scenario.id,
+      emitSetupProgress,
+      { template }
+    );
+    const partialResult = await runAgentWithContext(
+      config,
+      evalRunId2,
+      { ...scenario, triggerPrompt: effectiveTriggerPrompt },
+      evalData,
+      workDir,
+      pushEvent
+    );
+    const inlineAssertions = scenario.assertions ?? [];
+    const assertions = [
+      ...inlineAssertions,
+      ...resolvedAssertions ?? []
+    ];
+    const templateFilesMap = new Map(
+      (partialResult.templateFiles ?? []).map((f) => [f.path, f.status])
+    );
+    const evaluationInput = {
+      outputText: partialResult.outputText,
+      llmTrace: partialResult.llmTrace,
+      fileDiffs: partialResult.fileDiffs?.map((d) => ({
+        path: d.path,
+        status: templateFilesMap.get(d.path)
+      })),
+      durationMs: partialResult.duration
+    };
+    const defaultJudgeModel = import_evalforge_types13.DEFAULT_JUDGE_MODEL;
+    const assertionContext = {
+      workDir,
+      defaultJudgeModel,
+      llmConfig: {
+        baseUrl: config.aiGatewayUrl,
+        headers: config.aiGatewayHeaders
+      }
+    };
+    const assertionResults = assertions.length > 0 ? await (0, import_eval_assertions.evaluateAssertions)(
+      evaluationInput,
+      assertions,
+      assertionContext
+    ) : [];
+    const passed = assertionResults.filter(
+      (r) => r.status === import_evalforge_types13.AssertionResultStatus.PASSED
+    ).length;
+    const failed = assertionResults.filter(
+      (r) => r.status === import_evalforge_types13.AssertionResultStatus.FAILED
+    ).length;
+    const total = assertionResults.length;
+    const passRate = total > 0 ? Math.round(passed / total * 100) : 100;
+    return {
+      ...partialResult,
+      assertionResults,
+      passed,
+      failed,
+      passRate,
+      provisionedSite
+    };
+  } finally {
+    if (provisionedSite && apiClient && projectId2) {
+      await apiClient.deleteProvisionedSite(projectId2, provisionedSite.id);
     }
-  };
-  const assertionResults = assertions.length > 0 ? await (0, import_eval_assertions.evaluateAssertions)(
-    evaluationInput,
-    assertions,
-    assertionContext
-  ) : [];
-  const passed = assertionResults.filter(
-    (r) => r.status === import_evalforge_types13.AssertionResultStatus.PASSED
-  ).length;
-  const failed = assertionResults.filter(
-    (r) => r.status === import_evalforge_types13.AssertionResultStatus.FAILED
-  ).length;
-  const total = assertionResults.length;
-  const passRate = total > 0 ? Math.round(passed / total * 100) : 100;
-  return {
-    ...partialResult,
-    assertionResults,
-    passed,
-    failed,
-    passRate
-  };
+  }
 }
 
 // src/evaluation-loop.ts
@@ -12143,7 +12394,9 @@ async function runEvaluation(projectId2, evalRunId2) {
         evalData,
         template,
         resolvedAssertions,
-        pushEvent
+        pushEvent,
+        api,
+        projectId2
       );
     },
     addResult: async (result) => {