langwatch 0.23.0 → 0.25.0

package/dist/index.mjs

@@ -5,51 +5,51 @@ import {
   DatasetPlanLimitError,
   DatasetValidationError,
   DatasetsFacade
-} from "./chunk-3CRFCUOQ.mjs";
+} from "./chunk-ILU7DFFC.mjs";
+import {
+  ScenariosApiService
+} from "./chunk-7G6VZNKU.mjs";
 import {
   SuitesApiService
-} from "./chunk-IFCABR3D.mjs";
+} from "./chunk-NAW5BKCY.mjs";
 import {
-  AnalyticsApiService
-} from "./chunk-XIHOAUIU.mjs";
-import "./chunk-WF54BYWA.mjs";
+  AnnotationsApiService
+} from "./chunk-IFHIS7NO.mjs";
 import {
-  ScenariosApiService
-} from "./chunk-LIUQYWHM.mjs";
+  AnalyticsApiService
+} from "./chunk-C3ASNRX5.mjs";
+import "./chunk-XRZSZSZN.mjs";
 import {
   DashboardsApiService
-} from "./chunk-YDRETRB6.mjs";
+} from "./chunk-5CPABRWB.mjs";
 import {
   ModelProvidersApiService
-} from "./chunk-YD7SA3ZL.mjs";
-import {
-  AnnotationsApiService
-} from "./chunk-Y5PPNQXM.mjs";
+} from "./chunk-BRBFTP25.mjs";
 import {
   AgentsApiService
-} from "./chunk-HJARRXXA.mjs";
+} from "./chunk-AN2NF2FO.mjs";
 import {
   WorkflowsApiService
-} from "./chunk-QJQMCRLL.mjs";
+} from "./chunk-VDRY4UC2.mjs";
 import {
   EvaluatorsApiError,
   EvaluatorsApiService
-} from "./chunk-7FCVMMAW.mjs";
+} from "./chunk-3ABCXA7Q.mjs";
 import {
   ensureSetup
-} from "./chunk-N6O7634G.mjs";
+} from "./chunk-ZTLHGFWB.mjs";
 import {
   FetchPolicy,
   LocalPromptsService,
   PromptsApiService,
   PromptsFacade
-} from "./chunk-GSZ4QKFH.mjs";
+} from "./chunk-Y6O4YPT7.mjs";
 import {
   createTracingProxy
 } from "./chunk-FG3JQBJB.mjs";
 import {
   createLangWatchApiClient
-} from "./chunk-SAR52TH6.mjs";
+} from "./chunk-PM6QYGUE.mjs";
 import {
   FilterableBatchSpanProcessor
 } from "./chunk-FARZEN6T.mjs";
@@ -60,20 +60,25 @@ import {
 import {
   LangWatchTraceExporter,
   getLangWatchLogger
-} from "./chunk-PWBHSOGK.mjs";
+} from "./chunk-TOGBUNQC.mjs";
 import {
   DEFAULT_ENDPOINT,
   LANGWATCH_SDK_NAME_CLIENT,
   LANGWATCH_SDK_VERSION
-} from "./chunk-LUBBPBV3.mjs";
+} from "./chunk-FPNXFYXA.mjs";
 import "./chunk-RKJDDUGQ.mjs";
 import {
   ConsoleLogger,
   NoOpLogger,
   attributes_exports
 } from "./chunk-M4IBRWUC.mjs";
-import "./chunk-O6D3W3PG.mjs";
+import "./chunk-OETMLMNQ.mjs";
 import "./chunk-BTBSMJTY.mjs";
+import {
+  extractStatusFromResponse,
+  formatApiErrorForOperation,
+  formatApiErrorMessage
+} from "./chunk-J3KM7ZO7.mjs";
 import {
   __privateAdd,
   __privateGet,
@@ -246,6 +251,50 @@ var EvaluatorError = class extends ExperimentError {
   }
 };
 
+// src/client-sdk/services/experiments/printSummary.ts
+function printSummary(result) {
+  const { runId, status, passed, failed, passRate, duration, runUrl, summary } = result;
+  console.log("\n" + "\u2550".repeat(60));
+  console.log("  EXPERIMENT RESULTS");
+  console.log("\u2550".repeat(60));
+  console.log(`  Run ID:     ${runId}`);
+  console.log(`  Status:     ${status.toUpperCase()}`);
+  console.log(`  Duration:   ${(duration / 1e3).toFixed(1)}s`);
+  console.log("\u2500".repeat(60));
+  console.log(`  Passed:     ${passed}`);
+  console.log(`  Failed:     ${failed}`);
+  console.log(`  Pass Rate:  ${passRate.toFixed(1)}%`);
+  if (summary.totalCost) {
+    console.log(`  Total Cost: $${summary.totalCost.toFixed(4)}`);
+  }
+  if (summary.targets && summary.targets.length > 0) {
+    console.log("\u2500".repeat(60));
+    console.log("  TARGETS:");
+    for (const target of summary.targets) {
+      console.log(`    ${target.name}: ${target.passed} passed, ${target.failed} failed`);
+      if (target.avgLatency) {
+        console.log(`      Avg latency: ${target.avgLatency.toFixed(0)}ms`);
+      }
+      if (target.totalCost) {
+        console.log(`      Total cost: $${target.totalCost.toFixed(4)}`);
+      }
+    }
+  }
+  if (summary.evaluators && summary.evaluators.length > 0) {
+    console.log("\u2500".repeat(60));
+    console.log("  EVALUATORS:");
+    for (const evaluator of summary.evaluators) {
+      console.log(`    ${evaluator.name}: ${evaluator.passRate.toFixed(1)}% pass rate`);
+      if (evaluator.avgScore !== void 0) {
+        console.log(`      Avg score: ${evaluator.avgScore.toFixed(2)}`);
+      }
+    }
+  }
+  console.log("\u2500".repeat(60));
+  console.log(`  View details: ${runUrl}`);
+  console.log("\u2550".repeat(60) + "\n");
+}
+
 // src/client-sdk/services/experiments/experiment.ts
 var DEFAULT_CONCURRENCY = 4;
 var DEBOUNCE_INTERVAL_MS = 1e3;
@@ -261,6 +310,12 @@ var Experiment = class _Experiment {
     this.lastSentMs = 0;
     this.pendingFlush = null;
     this.flushTimeout = null;
+    // Cumulative record for printSummary() — never cleared, mirrors batch.
+    // Store only the fields printSummary() needs, not the full payloads —
+    // inputs/tracebacks/outputs can be large and don't belong in retained summary state.
+    this.cumulativeEvaluations = [];
+    this.cumulativeEntries = [];
+    this.runUrl = "";
     // Target registry
     this.targets = /* @__PURE__ */ new Map();
     // Current iteration context (for log/evaluate calls)
@@ -325,7 +380,8 @@ var Experiment = class _Experiment {
       const data = await response.json();
       this.experimentSlug = data.slug;
       const encodedRunId = encodeURIComponent(this.runId);
-      console.log(`Follow results at: ${this.endpoint.replace(/\/$/, "")}${data.path}?runId=${encodedRunId}`);
+      this.runUrl = `${this.endpoint.replace(/\/$/, "")}${data.path}?runId=${encodedRunId}`;
+      console.log(`Follow results at: ${this.runUrl}`);
       this.initialized = true;
     } catch (error) {
       if (error instanceof ExperimentInitError) {
@@ -465,6 +521,12 @@ var Experiment = class _Experiment {
         trace_id: capturedTraceId != null ? capturedTraceId : this.getTraceIdFromContext()
       };
       this.batch.dataset.push(entry);
+      this.cumulativeEntries.push({
+        duration: entry.duration,
+        error: entry.error,
+        cost: entry.cost,
+        target_id: entry.target_id
+      });
     }
     this.iterationUsedWithTarget.delete(index);
     this.progress++;
@@ -531,6 +593,15 @@ var Experiment = class _Experiment {
       target_id: targetId != null ? targetId : null
     };
     this.batch.evaluations.push(result);
+    this.cumulativeEvaluations.push({
+      name: result.name,
+      evaluator: result.evaluator,
+      status: result.status,
+      passed: result.passed,
+      score: result.score,
+      cost: result.cost,
+      target_id: result.target_id
+    });
     this.scheduleSend();
   }
   /**
@@ -725,6 +796,12 @@ var Experiment = class _Experiment {
       predicted
     };
     this.batch.dataset.push(entry);
+    this.cumulativeEntries.push({
+      duration: entry.duration,
+      error: entry.error,
+      cost: entry.cost,
+      target_id: entry.target_id
+    });
     this.scheduleSend();
     return {
       result,
@@ -773,6 +850,128 @@ var Experiment = class _Experiment {
       }, DEBOUNCE_INTERVAL_MS - (now - this.lastSentMs));
     }
   }
+  /**
+   * Print a CI-friendly summary and optionally exit with code 1 on failure.
+   *
+   * Mirrors `ExperimentRunResult.printSummary` for parity — SDK-driven experiments
+   * can now fail CI builds the same way platform experiments do.
+   *
+   * @param exitOnFailure - If true (default), calls `process.exit(1)` when any evaluation failed.
+   *
+   * @example
+   * ```typescript
+   * const experiment = await langwatch.experiments.init("ci-quality-check");
+   * await experiment.run(dataset, async ({ item, index }) => {
+   *   const response = await myLLM(item.input);
+   *   await experiment.evaluate("ragas/faithfulness", { index, data: { input: item.input, output: response } });
+   * });
+   * experiment.printSummary();
+   * ```
+   */
+  printSummary(exitOnFailure = true) {
+    var _a, _b;
+    const evaluators = /* @__PURE__ */ new Map();
+    let totalPassed = 0;
+    let totalFailed = 0;
+    let totalCost = 0;
+    for (const e of this.cumulativeEvaluations) {
+      const name = (_b = (_a = e.name) != null ? _a : e.evaluator) != null ? _b : "unknown";
+      if (!evaluators.has(name)) {
+        evaluators.set(name, { passed: 0, failed: 0, scoreSum: 0, scoreCount: 0 });
+      }
+      const stats = evaluators.get(name);
+      if (e.status === "error" || e.passed === false) {
+        stats.failed += 1;
+        totalFailed += 1;
+      } else if (e.passed === true) {
+        stats.passed += 1;
+        totalPassed += 1;
+      }
+      if (typeof e.score === "number") {
+        stats.scoreSum += e.score;
+        stats.scoreCount += 1;
+      }
+      if (typeof e.cost === "number") {
+        totalCost += e.cost;
+      }
+    }
+    const targets = /* @__PURE__ */ new Map();
+    for (const entry of this.cumulativeEntries) {
+      const tid = entry.target_id;
+      if (!tid) continue;
+      if (!targets.has(tid)) {
+        targets.set(tid, { passed: 0, failed: 0, latencySum: 0, latencyCount: 0, cost: 0 });
+      }
+      const stats = targets.get(tid);
+      if (typeof entry.duration === "number") {
+        stats.latencySum += entry.duration;
+        stats.latencyCount += 1;
+      }
+      if (typeof entry.cost === "number") {
+        stats.cost += entry.cost;
+      }
+    }
+    for (const e of this.cumulativeEvaluations) {
+      const tid = e.target_id;
+      if (!tid) continue;
+      if (!targets.has(tid)) {
+        targets.set(tid, { passed: 0, failed: 0, latencySum: 0, latencyCount: 0, cost: 0 });
+      }
+      const stats = targets.get(tid);
+      if (e.status === "error" || e.passed === false) stats.failed += 1;
+      else if (e.passed === true) stats.passed += 1;
+      if (typeof e.cost === "number") stats.cost += e.cost;
+    }
+    const total = totalPassed + totalFailed;
+    const passRate = total > 0 ? totalPassed / total * 100 : 0;
+    const duration = Math.max(0, Date.now() - this.createdAtMs);
+    let failedCells = 0;
+    for (const entry of this.cumulativeEntries) {
+      if (typeof entry.cost === "number") totalCost += entry.cost;
+      if (entry.error) failedCells += 1;
+    }
+    const hasFailures = totalFailed > 0 || failedCells > 0;
+    printSummary({
+      runId: this.runId,
+      status: hasFailures ? "failed" : "completed",
+      passed: totalPassed,
+      failed: totalFailed,
+      passRate,
+      duration,
+      runUrl: this.runUrl,
+      summary: {
+        runId: this.runId,
+        totalCells: this.cumulativeEntries.length || total,
+        completedCells: Math.max(0, (this.cumulativeEntries.length || total) - failedCells),
+        failedCells,
+        duration,
+        runUrl: this.runUrl,
+        totalPassed,
+        totalFailed,
+        passRate,
+        totalCost,
+        targets: Array.from(targets.entries()).map(([targetId, s]) => ({
+          targetId,
+          name: targetId,
+          passed: s.passed,
+          failed: s.failed,
+          avgLatency: s.latencyCount > 0 ? s.latencySum / s.latencyCount : 0,
+          totalCost: s.cost
+        })),
+        evaluators: Array.from(evaluators.entries()).map(([name, s]) => ({
+          evaluatorId: name,
+          name,
+          passed: s.passed,
+          failed: s.failed,
+          passRate: s.passed + s.failed > 0 ? s.passed / (s.passed + s.failed) * 100 : 0,
+          avgScore: s.scoreCount > 0 ? s.scoreSum / s.scoreCount : void 0
+        }))
+      }
+    });
+    if (exitOnFailure && hasFailures) {
+      process.exit(1);
+    }
+  }
   /**
    * Send current batch to the API
    */
@@ -1112,7 +1311,7 @@ var ExperimentsFacade = class {
       summary,
       printSummary: (exitOnFailure = true) => {
         var _a2;
-        this.printSummary({
+        printSummary({
           runId,
           status,
           passed: totalPassed,
@@ -1131,50 +1330,6 @@ var ExperimentsFacade = class {
     result[Symbol.for("nodejs.util.inspect.custom")] = () => result.toString();
     return result;
   }
-  /**
-   * Print a CI-friendly summary of the experiment results
-   */
-  printSummary(result) {
-    const { runId, status, passed, failed, passRate, duration, runUrl, summary } = result;
-    console.log("\n" + "\u2550".repeat(60));
-    console.log("  EXPERIMENT RESULTS");
-    console.log("\u2550".repeat(60));
-    console.log(`  Run ID:     ${runId}`);
-    console.log(`  Status:     ${status.toUpperCase()}`);
-    console.log(`  Duration:   ${(duration / 1e3).toFixed(1)}s`);
-    console.log("\u2500".repeat(60));
-    console.log(`  Passed:     ${passed}`);
-    console.log(`  Failed:     ${failed}`);
-    console.log(`  Pass Rate:  ${passRate.toFixed(1)}%`);
-    if (summary.targets && summary.targets.length > 0) {
-      console.log("\u2500".repeat(60));
-      console.log("  TARGETS:");
-      for (const target of summary.targets) {
-        console.log(`    ${target.name}: ${target.passed} passed, ${target.failed} failed`);
-        if (target.avgLatency) {
-          console.log(`      Avg latency: ${target.avgLatency.toFixed(0)}ms`);
-        }
-        if (target.totalCost) {
-          console.log(`      Total cost: $${target.totalCost.toFixed(4)}`);
-        }
-      }
-    }
-    if (summary.evaluators && summary.evaluators.length > 0) {
-      console.log("\u2500".repeat(60));
-      console.log("  EVALUATORS:");
-      for (const evaluator of summary.evaluators) {
-        console.log(
-          `    ${evaluator.name}: ${evaluator.passRate.toFixed(1)}% pass rate`
-        );
-        if (evaluator.avgScore !== void 0) {
-          console.log(`      Avg score: ${evaluator.avgScore.toFixed(2)}`);
-        }
-      }
-    }
-    console.log("\u2500".repeat(60));
-    console.log(`  View details: ${runUrl}`);
-    console.log("\u2550".repeat(60) + "\n");
-  }
   sleep(ms) {
     return new Promise((resolve) => setTimeout(resolve, ms));
   }
@@ -1394,13 +1549,10 @@ var TriggersApiService = class {
     this.apiClient = (_a = config == null ? void 0 : config.langwatchApiClient) != null ? _a : createLangWatchApiClient();
   }
   handleApiError(operation, error) {
-    var _a;
-    const errorMessage = typeof error === "string" ? error : error != null && typeof error === "object" && "error" in error && error.error != null ? typeof error.error === "string" ? error.error : (_a = error.error.message) != null ? _a : JSON.stringify(error.error) : error instanceof Error ? error.message : "Unknown error occurred";
-    throw new TriggersApiError(
-      `Failed to ${operation}: ${errorMessage}`,
-      operation,
-      error
-    );
+    const message = formatApiErrorForOperation({ operation, error, options: {
+      status: extractStatusFromResponse(error)
+    } });
+    throw new TriggersApiError(message, operation, error);
   }
   async getAll() {
     const { data, error } = await this.apiClient.GET("/api/triggers");
@@ -1453,13 +1605,10 @@ var GraphsApiService = class {
     this.apiClient = (_a = config == null ? void 0 : config.langwatchApiClient) != null ? _a : createLangWatchApiClient();
   }
   handleApiError(operation, error) {
-    var _a;
-    const errorMessage = typeof error === "string" ? error : error != null && typeof error === "object" && "error" in error && error.error != null ? typeof error.error === "string" ? error.error : (_a = error.error.message) != null ? _a : JSON.stringify(error.error) : error instanceof Error ? error.message : "Unknown error occurred";
-    throw new GraphsApiError(
-      `Failed to ${operation}: ${errorMessage}`,
-      operation,
-      error
-    );
+    const message = formatApiErrorForOperation({ operation, error, options: {
+      status: extractStatusFromResponse(error)
+    } });
+    throw new GraphsApiError(message, operation, error);
   }
   async getAll(dashboardId) {
     const { data, error } = await this.apiClient.GET("/api/graphs", {
@@ -1514,13 +1663,10 @@ var SimulationRunsApiService = class {
     this.apiClient = (_a = config == null ? void 0 : config.langwatchApiClient) != null ? _a : createLangWatchApiClient();
   }
   handleApiError(operation, error) {
-    var _a;
-    const errorMessage = typeof error === "string" ? error : error != null && typeof error === "object" && "error" in error && error.error != null ? typeof error.error === "string" ? error.error : (_a = error.error.message) != null ? _a : JSON.stringify(error.error) : error instanceof Error ? error.message : "Unknown error occurred";
-    throw new SimulationRunsApiError(
-      `Failed to ${operation}: ${errorMessage}`,
-      operation,
-      error
-    );
+    const message = formatApiErrorForOperation({ operation, error, options: {
+      status: extractStatusFromResponse(error)
+    } });
+    throw new SimulationRunsApiError(message, operation, error);
   }
   async getAll(params) {
     const { data, error } = await this.apiClient.GET("/api/simulation-runs", {
@@ -1577,9 +1723,16 @@ var MonitorsApiService = class {
     }));
     if (!response.ok) {
       const errorText = await response.text();
+      let parsed = errorText;
+      try {
+        parsed = JSON.parse(errorText);
+      } catch (e) {
+      }
+      const message = formatApiErrorMessage({ error: parsed, options: { status: response.status } });
       throw new MonitorsApiError(
-        `HTTP ${response.status}: ${errorText}`,
-        (_a = options == null ? void 0 : options.method) != null ? _a : "GET"
+        `HTTP ${response.status}: ${message}`,
+        (_a = options == null ? void 0 : options.method) != null ? _a : "GET",
+        parsed
       );
     }
     return response.json();
@@ -1640,9 +1793,16 @@ var SecretsApiService = class {
     }));
     if (!response.ok) {
       const errorText = await response.text();
+      let parsed = errorText;
+      try {
+        parsed = JSON.parse(errorText);
+      } catch (e) {
+      }
+      const message = formatApiErrorMessage({ error: parsed, options: { status: response.status } });
       throw new SecretsApiError(
-        `HTTP ${response.status}: ${errorText}`,
-        (_a = options == null ? void 0 : options.method) != null ? _a : "GET"
+        `HTTP ${response.status}: ${message}`,
+        (_a = options == null ? void 0 : options.method) != null ? _a : "GET",
+        parsed
       );
     }
     return response.json();