@wix/evalforge-evaluator 0.185.0 → 0.187.0

package/build/index.mjs

@@ -1282,7 +1282,7 @@ var require_error = __commonJS({
       "toJSON",
       "__CANCEL__"
     ];
-    var HttpError = class _HttpError extends Error {
+    var HttpError2 = class _HttpError extends Error {
       constructor(error) {
         var _a;
         super(error.message);
@@ -1299,9 +1299,9 @@ var require_error = __commonJS({
         return (0, headers_1.requestIdOrEmptyString)(this.response);
       }
     };
-    exports.HttpError = HttpError;
+    exports.HttpError = HttpError2;
     function createHttpError(...args) {
-      return new HttpError(...args);
+      return new HttpError2(...args);
     }
     exports.createHttpError = createHttpError;
   }
@@ -6356,6 +6356,133 @@ function getLatestCapabilityVersion(payload) {
   return __getLatestCapabilityVersion;
 }
 
+// ../../node_modules/@wix/ambassador-evalforge-v1-site-provisioning/es/build/http.impl.js
+var _deleteProvisionedSiteRequest = {};
+var _deleteProvisionedSiteResponse = {};
+var _provisionScenarioSiteRequest = {};
+var _provisionScenarioSiteResponse = {};
+function resolveWixEvalforgeV1SiteProvisioningServiceUrl(opts) {
+  var domainToMappings = {
+    "dev._base_domain_": [
+      {
+        srcPath: "/_api/evalforge-backend",
+        destPath: ""
+      }
+    ],
+    "api._api_base_domain_": [
+      {
+        srcPath: "/evalforge-backend",
+        destPath: ""
+      }
+    ],
+    "bo._base_domain_": [
+      {
+        srcPath: "/_api/evalforge-backend",
+        destPath: ""
+      }
+    ],
+    "wixbo.ai": [
+      {
+        srcPath: "/_api/evalforge-backend",
+        destPath: ""
+      }
+    ],
+    "wix-bo.com": [
+      {
+        srcPath: "/_api/evalforge-backend",
+        destPath: ""
+      }
+    ],
+    "manage._base_domain_": [
+      {
+        srcPath: "/_api/evalforge-backend",
+        destPath: ""
+      }
+    ]
+  };
+  return resolveUrl(Object.assign(opts, { domainToMappings }));
+}
+function provisionScenarioSite(payload) {
+  var _a = serializer(_provisionScenarioSiteRequest, {}), toReq = _a.toJSON, fromReq = _a.fromJSON;
+  var fromRes = serializer(_provisionScenarioSiteResponse, {}).fromJSON;
+  function __provisionScenarioSite(_a2) {
+    var host = _a2.host;
+    var serializedData = toReq(payload);
+    var metadata = {
+      entityFqdn: "wix.evalforge.v1.site_provisioning",
+      method: "POST",
+      methodFqn: "wix.evalforge.v1.SiteProvisioningService.ProvisionScenarioSite",
+      migrationOptions: {
+        optInTransformResponse: true
+      },
+      url: resolveWixEvalforgeV1SiteProvisioningServiceUrl({
+        protoPath: "/v1/projects/{projectId}/site-provisioning/provision-site",
+        data: serializedData,
+        host
+      }),
+      data: serializedData,
+      transformResponse: fromRes
+    };
+    return metadata;
+  }
+  __provisionScenarioSite.fromReq = fromReq;
+  __provisionScenarioSite.__isAmbassador = true;
+  return __provisionScenarioSite;
+}
+function deleteProvisionedSite(payload) {
+  var _a = serializer(_deleteProvisionedSiteRequest, {}), toReq = _a.toJSON, fromReq = _a.fromJSON;
+  var fromRes = serializer(_deleteProvisionedSiteResponse, {}).fromJSON;
+  function __deleteProvisionedSite(_a2) {
+    var host = _a2.host;
+    var serializedData = toReq(payload);
+    var metadata = {
+      entityFqdn: "wix.evalforge.v1.site_provisioning",
+      method: "POST",
+      methodFqn: "wix.evalforge.v1.SiteProvisioningService.DeleteProvisionedSite",
+      migrationOptions: {
+        optInTransformResponse: true
+      },
+      url: resolveWixEvalforgeV1SiteProvisioningServiceUrl({
+        protoPath: "/v1/projects/{projectId}/site-provisioning/delete-site",
+        data: serializedData,
+        host
+      }),
+      data: serializedData,
+      transformResponse: fromRes
+    };
+    return metadata;
+  }
+  __deleteProvisionedSite.fromReq = fromReq;
+  __deleteProvisionedSite.__isAmbassador = true;
+  return __deleteProvisionedSite;
+}
+
+// ../../node_modules/@wix/ambassador-evalforge-v1-test-scenario/es/build/types.impl.js
+var WebhookIdentityType;
+(function(WebhookIdentityType2) {
+  WebhookIdentityType2["UNKNOWN"] = "UNKNOWN";
+  WebhookIdentityType2["ANONYMOUS_VISITOR"] = "ANONYMOUS_VISITOR";
+  WebhookIdentityType2["MEMBER"] = "MEMBER";
+  WebhookIdentityType2["WIX_USER"] = "WIX_USER";
+  WebhookIdentityType2["APP"] = "APP";
+})(WebhookIdentityType || (WebhookIdentityType = {}));
+var SiteBootstrapHttpMethod;
+(function(SiteBootstrapHttpMethod2) {
+  SiteBootstrapHttpMethod2["SITE_BOOTSTRAP_HTTP_METHOD_UNSPECIFIED"] = "SITE_BOOTSTRAP_HTTP_METHOD_UNSPECIFIED";
+  SiteBootstrapHttpMethod2["GET"] = "GET";
+  SiteBootstrapHttpMethod2["POST"] = "POST";
+  SiteBootstrapHttpMethod2["PUT"] = "PUT";
+  SiteBootstrapHttpMethod2["PATCH"] = "PATCH";
+  SiteBootstrapHttpMethod2["DELETE"] = "DELETE";
+})(SiteBootstrapHttpMethod || (SiteBootstrapHttpMethod = {}));
+var Mode;
+(function(Mode2) {
+  Mode2["UNKNOWN_MODE"] = "UNKNOWN_MODE";
+  Mode2["NONE"] = "NONE";
+  Mode2["CLONE"] = "CLONE";
+  Mode2["TEMPLATE"] = "TEMPLATE";
+})(Mode || (Mode = {}));
+
 // src/ambassador-converters.ts
 function toProtoEnum(prefix, value) {
   return `${prefix}${value.toUpperCase()}`;
@@ -6662,9 +6789,37 @@ function testScenarioFromProto(wire) {
       })
     ),
     createdAt: fromProtoDate(wire.createdAt) ?? "",
-    updatedAt: fromProtoDate(wire.updatedAt) ?? ""
+    updatedAt: fromProtoDate(wire.updatedAt) ?? "",
+    siteSetup: siteSetupFromAmbassador(wire.siteSetup)
   };
 }
+function siteSetupFromAmbassador(wire) {
+  if (!wire) return void 0;
+  const steps = (wire.bootstrap?.steps ?? []).filter(
+    (step) => step.method && step.method !== SiteBootstrapHttpMethod.SITE_BOOTSTRAP_HTTP_METHOD_UNSPECIFIED && step.url
+  ).map((step) => ({
+    label: step.label ?? void 0,
+    method: step.method.toLowerCase(),
+    url: step.url ?? "",
+    body: step.body ?? void 0
+  }));
+  const bootstrap = steps.length > 0 ? { steps } : void 0;
+  if (wire.mode === Mode.CLONE) {
+    return {
+      mode: "clone",
+      sourceSiteId: wire.cloneOptions?.sourceSiteId ?? "",
+      bootstrap
+    };
+  }
+  if (wire.mode === Mode.TEMPLATE) {
+    return {
+      mode: "template",
+      templateId: wire.templateOptions?.templateId ?? "",
+      bootstrap
+    };
+  }
+  return void 0;
+}
 function templateFromProto(wire) {
   return {
     id: wire.id ?? "",
@@ -6718,8 +6873,36 @@ function capabilityVersionFromProto(wire, projectId2) {
     createdAt: fromProtoDate(wire.createdAt) ?? ""
   };
 }
+function provisionedSiteFromProto(proto) {
+  return {
+    id: proto.id ?? "",
+    url: proto.url ?? void 0,
+    editorUrl: proto.editorUrl ?? void 0
+  };
+}
+function siteBootstrapResultFromProto(proto) {
+  if (!proto) return void 0;
+  return {
+    steps: (proto.steps ?? []).map((step) => ({
+      label: step.label ?? void 0,
+      statusCode: step.statusCode ?? 0,
+      ok: step.ok ?? false,
+      error: step.error ?? void 0
+    }))
+  };
+}
 
 // src/api-client.ts
+function rethrowWithRequestId(err, action) {
+  if (err instanceof import_http_client.HttpError) {
+    const status = err.response?.status;
+    const requestId = err.requestId;
+    throw new Error(
+      `Failed to ${action}` + (status !== void 0 ? ` (HTTP ${status})` : "") + (requestId ? ` [request id: ${requestId}]` : "") + `: ${err.message}`
+    );
+  }
+  throw err;
+}
 function resolveAmbassadorBaseUrl(serverUrl) {
   try {
     return new URL(serverUrl).origin;
@@ -6821,21 +7004,37 @@ function createApiClient(serverUrl, options = "") {
     // The legacy REST endpoint enriched the capability with its latest version
     // server-side; ambassador's GetCapability returns the bare entity, so we
     // compose it with GetLatestCapabilityVersion in parallel here.
+    //
+    // The latest-version fetch is BEST-EFFORT: a failure must not drop the whole
+    // capability. Otherwise one broken snapshot fetch makes the capability (e.g.
+    // an MCP) silently vanish from the run. Runs that pin a version still resolve
+    // their content via getCapabilityVersion downstream.
     async getCapability(projectId2, id) {
-      const [capRes, versionRes] = await Promise.all([
+      const [capResult, versionResult] = await Promise.allSettled([
         httpClient.request(getCapability({ projectId: projectId2, capabilityId: id })),
         httpClient.request(
           getLatestCapabilityVersion({ projectId: projectId2, capabilityId: id })
         )
       ]);
-      const capability = capRes.data.capability;
+      if (capResult.status === "rejected") {
+        throw capResult.reason;
+      }
+      const capability = capResult.value.data.capability;
       if (!capability) {
         throw new Error(`Capability ${id} not found in project ${projectId2}`);
       }
-      const latestVersion = versionRes.data.capabilityVersion ? capabilityVersionFromProto(
-        versionRes.data.capabilityVersion,
-        projectId2
-      ) : void 0;
+      let latestVersion;
+      if (versionResult.status === "fulfilled" && versionResult.value.data.capabilityVersion) {
+        latestVersion = capabilityVersionFromProto(
+          versionResult.value.data.capabilityVersion,
+          projectId2
+        );
+      } else if (versionResult.status === "rejected") {
+        const reason = versionResult.reason instanceof Error ? versionResult.reason.message : String(versionResult.reason);
+        console.warn(
+          `[Capabilities] getLatestCapabilityVersion(${id}) failed; loading capability without a snapshot (pinned versions still resolve): ${reason}`
+        );
+      }
       return { ...capabilityFromProto(capability), latestVersion };
     },
     async getCapabilityVersion(projectId2, capabilityId, versionId) {
@@ -6879,6 +7078,29 @@ function createApiClient(serverUrl, options = "") {
     },
     updateEvalRun(projectId2, evalRunId2, update) {
       return putJson(`/projects/${projectId2}/eval-runs/${evalRunId2}`, update);
+    },
+    async provisionScenarioSite(projectId2, evalRunId2, scenarioId) {
+      const res = await httpClient.request(provisionScenarioSite({ projectId: projectId2, evalRunId: evalRunId2, scenarioId })).catch(
+        (err) => rethrowWithRequestId(err, `provision a site for scenario ${scenarioId}`)
+      );
+      const site = res.data.site;
+      if (!site) {
+        throw new Error(
+          `Site provisioning for scenario ${scenarioId} returned no site.`
+        );
+      }
+      return {
+        ...provisionedSiteFromProto(site),
+        bootstrapResult: siteBootstrapResultFromProto(res.data.bootstrapResult)
+      };
+    },
+    async deleteProvisionedSite(projectId2, siteId) {
+      await httpClient.request(deleteProvisionedSite({ projectId: projectId2, siteId })).catch((err) => {
+        console.warn(
+          "[site-provisioning] deleteProvisionedSite failed \u2014 site may remain:",
+          err
+        );
+      });
     }
   };
 }
@@ -11820,87 +12042,123 @@ async function runAgentWithContext(config, evalRunId2, scenario, evalData, workD
 }
 
 // src/run-scenario/index.ts
-async function runScenario(config, evalRunId2, scenario, evalData, template, resolvedAssertions, pushEvent) {
+async function runScenario(config, evalRunId2, scenario, evalData, template, resolvedAssertions, pushEvent, apiClient, projectId2) {
   const targetId = evalData.evalRun.presetId ?? evalData.agent?.id ?? evalData.evalRun.id;
   const targetName = evalData.presetName ?? evalData.agent?.name ?? "";
-  if (template) {
-    console.log(
-      formatTraceEventLine({
-        evalRunId: evalRunId2,
-        scenarioId: scenario.id,
-        scenarioName: scenario.name,
-        targetId,
-        targetName,
-        stepNumber: 0,
-        type: LiveTraceEventType4.PROGRESS,
-        outputPreview: "Setting up environment (installing dependencies)...",
-        elapsedMs: 0,
-        timestamp: (/* @__PURE__ */ new Date()).toISOString(),
-        isComplete: false
-      })
+  let provisionedSite;
+  if (apiClient && projectId2 && scenario.siteSetup && scenario.siteSetup.mode !== "none") {
+    provisionedSite = await apiClient.provisionScenarioSite(
+      projectId2,
+      evalRunId2,
+      scenario.id
     );
   }
-  const workDir = await prepareWorkingDirectory(
-    config,
-    evalRunId2,
-    targetId,
-    scenario.id,
-    template
-  );
-  const partialResult = await runAgentWithContext(
-    config,
-    evalRunId2,
-    scenario,
-    evalData,
-    workDir,
-    pushEvent
-  );
-  const inlineAssertions = scenario.assertions ?? [];
-  const assertions = [
-    ...inlineAssertions,
-    ...resolvedAssertions ?? []
-  ];
-  const templateFilesMap = new Map(
-    (partialResult.templateFiles ?? []).map((f) => [f.path, f.status])
-  );
-  const evaluationInput = {
-    outputText: partialResult.outputText,
-    llmTrace: partialResult.llmTrace,
-    fileDiffs: partialResult.fileDiffs?.map((d) => ({
-      path: d.path,
-      status: templateFilesMap.get(d.path)
-    })),
-    durationMs: partialResult.duration
-  };
-  const defaultJudgeModel = DEFAULT_JUDGE_MODEL;
-  const assertionContext = {
-    workDir,
-    defaultJudgeModel,
-    llmConfig: {
-      baseUrl: config.aiGatewayUrl,
-      headers: config.aiGatewayHeaders
+  const failedStep = provisionedSite?.bootstrapResult?.steps.find((s) => !s.ok);
+  if (failedStep) {
+    const message = `Site bootstrap step ${failedStep.label ? `"${failedStep.label}" ` : ""}failed (HTTP ${failedStep.statusCode}): ${failedStep.error ?? "unknown error"}`;
+    console.warn(`[run-scenario] ${message}`);
+    pushEvent?.({
+      evalRunId: evalRunId2,
+      scenarioId: scenario.id,
+      scenarioName: scenario.name,
+      targetId,
+      targetName,
+      stepNumber: 0,
+      type: LiveTraceEventType4.PROGRESS,
+      outputPreview: message,
+      elapsedMs: 0,
+      timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+      isComplete: false
+    });
+  }
+  const effectiveTriggerPrompt = provisionedSite ? `${scenario.triggerPrompt}
+
+Site ID: ${provisionedSite.id}` : scenario.triggerPrompt;
+  try {
+    if (template) {
+      console.log(
+        formatTraceEventLine({
+          evalRunId: evalRunId2,
+          scenarioId: scenario.id,
+          scenarioName: scenario.name,
+          targetId,
+          targetName,
+          stepNumber: 0,
+          type: LiveTraceEventType4.PROGRESS,
+          outputPreview: "Setting up environment (installing dependencies)...",
+          elapsedMs: 0,
+          timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+          isComplete: false
+        })
+      );
     }
-  };
-  const assertionResults = assertions.length > 0 ? await evaluateAssertionsBase(
-    evaluationInput,
-    assertions,
-    assertionContext
-  ) : [];
-  const passed = assertionResults.filter(
-    (r) => r.status === AssertionResultStatus.PASSED
-  ).length;
-  const failed = assertionResults.filter(
-    (r) => r.status === AssertionResultStatus.FAILED
-  ).length;
-  const total = assertionResults.length;
-  const passRate = total > 0 ? Math.round(passed / total * 100) : 100;
-  return {
-    ...partialResult,
-    assertionResults,
-    passed,
-    failed,
-    passRate
-  };
+    const workDir = await prepareWorkingDirectory(
+      config,
+      evalRunId2,
+      targetId,
+      scenario.id,
+      template
+    );
+    const partialResult = await runAgentWithContext(
+      config,
+      evalRunId2,
+      { ...scenario, triggerPrompt: effectiveTriggerPrompt },
+      evalData,
+      workDir,
+      pushEvent
+    );
+    const inlineAssertions = scenario.assertions ?? [];
+    const assertions = [
+      ...inlineAssertions,
+      ...resolvedAssertions ?? []
+    ];
+    const templateFilesMap = new Map(
+      (partialResult.templateFiles ?? []).map((f) => [f.path, f.status])
+    );
+    const evaluationInput = {
+      outputText: partialResult.outputText,
+      llmTrace: partialResult.llmTrace,
+      fileDiffs: partialResult.fileDiffs?.map((d) => ({
+        path: d.path,
+        status: templateFilesMap.get(d.path)
+      })),
+      durationMs: partialResult.duration
+    };
+    const defaultJudgeModel = DEFAULT_JUDGE_MODEL;
+    const assertionContext = {
+      workDir,
+      defaultJudgeModel,
+      llmConfig: {
+        baseUrl: config.aiGatewayUrl,
+        headers: config.aiGatewayHeaders
+      }
+    };
+    const assertionResults = assertions.length > 0 ? await evaluateAssertionsBase(
+      evaluationInput,
+      assertions,
+      assertionContext
+    ) : [];
+    const passed = assertionResults.filter(
+      (r) => r.status === AssertionResultStatus.PASSED
+    ).length;
+    const failed = assertionResults.filter(
+      (r) => r.status === AssertionResultStatus.FAILED
+    ).length;
+    const total = assertionResults.length;
+    const passRate = total > 0 ? Math.round(passed / total * 100) : 100;
+    return {
+      ...partialResult,
+      assertionResults,
+      passed,
+      failed,
+      passRate,
+      provisionedSite
+    };
+  } finally {
+    if (provisionedSite && apiClient && projectId2) {
+      await apiClient.deleteProvisionedSite(projectId2, provisionedSite.id);
+    }
+  }
 }
 
 // src/evaluation-loop.ts
@@ -12180,7 +12438,9 @@ async function runEvaluation(projectId2, evalRunId2) {
         evalData,
         template,
         resolvedAssertions,
-        pushEvent
+        pushEvent,
+        api,
+        projectId2
       );
     },
     addResult: async (result) => {