langwatch 0.24.0 → 0.25.0

package/dist/index.mjs

@@ -5,51 +5,51 @@ import {
   DatasetPlanLimitError,
   DatasetValidationError,
   DatasetsFacade
-} from "./chunk-RV7L6G3P.mjs";
+} from "./chunk-ILU7DFFC.mjs";
+import {
+  ScenariosApiService
+} from "./chunk-7G6VZNKU.mjs";
 import {
   SuitesApiService
-} from "./chunk-ZWYTD4AG.mjs";
+} from "./chunk-NAW5BKCY.mjs";
 import {
-  AnalyticsApiService
-} from "./chunk-L3OKUC5R.mjs";
-import "./chunk-F5FCFJ2U.mjs";
+  AnnotationsApiService
+} from "./chunk-IFHIS7NO.mjs";
 import {
-  ScenariosApiService
-} from "./chunk-RVC7GPWG.mjs";
+  AnalyticsApiService
+} from "./chunk-C3ASNRX5.mjs";
+import "./chunk-XRZSZSZN.mjs";
 import {
   DashboardsApiService
-} from "./chunk-YSL4VG4Z.mjs";
+} from "./chunk-5CPABRWB.mjs";
 import {
   ModelProvidersApiService
-} from "./chunk-6CTXWSV2.mjs";
-import {
-  AnnotationsApiService
-} from "./chunk-SFLXNUFP.mjs";
+} from "./chunk-BRBFTP25.mjs";
 import {
   AgentsApiService
-} from "./chunk-WZN7CXPM.mjs";
+} from "./chunk-AN2NF2FO.mjs";
 import {
   WorkflowsApiService
-} from "./chunk-RFWHE6KS.mjs";
+} from "./chunk-VDRY4UC2.mjs";
 import {
   EvaluatorsApiError,
   EvaluatorsApiService
-} from "./chunk-TAKK42QB.mjs";
+} from "./chunk-3ABCXA7Q.mjs";
 import {
   ensureSetup
-} from "./chunk-DR74R3QU.mjs";
+} from "./chunk-ZTLHGFWB.mjs";
 import {
   FetchPolicy,
   LocalPromptsService,
   PromptsApiService,
   PromptsFacade
-} from "./chunk-7EGQA7EX.mjs";
+} from "./chunk-Y6O4YPT7.mjs";
 import {
   createTracingProxy
 } from "./chunk-FG3JQBJB.mjs";
 import {
   createLangWatchApiClient
-} from "./chunk-KBZ44J7F.mjs";
+} from "./chunk-PM6QYGUE.mjs";
 import {
   FilterableBatchSpanProcessor
 } from "./chunk-FARZEN6T.mjs";
@@ -60,19 +60,19 @@ import {
 import {
   LangWatchTraceExporter,
   getLangWatchLogger
-} from "./chunk-MQGDZNNP.mjs";
+} from "./chunk-TOGBUNQC.mjs";
 import {
   DEFAULT_ENDPOINT,
   LANGWATCH_SDK_NAME_CLIENT,
   LANGWATCH_SDK_VERSION
-} from "./chunk-ILRPI7OH.mjs";
+} from "./chunk-FPNXFYXA.mjs";
 import "./chunk-RKJDDUGQ.mjs";
 import {
   ConsoleLogger,
   NoOpLogger,
   attributes_exports
 } from "./chunk-M4IBRWUC.mjs";
-import "./chunk-DBNSF7FL.mjs";
+import "./chunk-OETMLMNQ.mjs";
 import "./chunk-BTBSMJTY.mjs";
 import {
   extractStatusFromResponse,
@@ -251,6 +251,50 @@ var EvaluatorError = class extends ExperimentError {
   }
 };
 
+// src/client-sdk/services/experiments/printSummary.ts
+function printSummary(result) {
+  const { runId, status, passed, failed, passRate, duration, runUrl, summary } = result;
+  console.log("\n" + "\u2550".repeat(60));
+  console.log("  EXPERIMENT RESULTS");
+  console.log("\u2550".repeat(60));
+  console.log(`  Run ID:     ${runId}`);
+  console.log(`  Status:     ${status.toUpperCase()}`);
+  console.log(`  Duration:   ${(duration / 1e3).toFixed(1)}s`);
+  console.log("\u2500".repeat(60));
+  console.log(`  Passed:     ${passed}`);
+  console.log(`  Failed:     ${failed}`);
+  console.log(`  Pass Rate:  ${passRate.toFixed(1)}%`);
+  if (summary.totalCost) {
+    console.log(`  Total Cost: $${summary.totalCost.toFixed(4)}`);
+  }
+  if (summary.targets && summary.targets.length > 0) {
+    console.log("\u2500".repeat(60));
+    console.log("  TARGETS:");
+    for (const target of summary.targets) {
+      console.log(`    ${target.name}: ${target.passed} passed, ${target.failed} failed`);
+      if (target.avgLatency) {
+        console.log(`      Avg latency: ${target.avgLatency.toFixed(0)}ms`);
+      }
+      if (target.totalCost) {
+        console.log(`      Total cost: $${target.totalCost.toFixed(4)}`);
+      }
+    }
+  }
+  if (summary.evaluators && summary.evaluators.length > 0) {
+    console.log("\u2500".repeat(60));
+    console.log("  EVALUATORS:");
+    for (const evaluator of summary.evaluators) {
+      console.log(`    ${evaluator.name}: ${evaluator.passRate.toFixed(1)}% pass rate`);
+      if (evaluator.avgScore !== void 0) {
+        console.log(`      Avg score: ${evaluator.avgScore.toFixed(2)}`);
+      }
+    }
+  }
+  console.log("\u2500".repeat(60));
+  console.log(`  View details: ${runUrl}`);
+  console.log("\u2550".repeat(60) + "\n");
+}
+
 // src/client-sdk/services/experiments/experiment.ts
 var DEFAULT_CONCURRENCY = 4;
 var DEBOUNCE_INTERVAL_MS = 1e3;
@@ -266,6 +310,12 @@ var Experiment = class _Experiment {
     this.lastSentMs = 0;
     this.pendingFlush = null;
     this.flushTimeout = null;
+    // Cumulative record for printSummary() — never cleared, mirrors batch.
+    // Store only the fields printSummary() needs, not the full payloads —
+    // inputs/tracebacks/outputs can be large and don't belong in retained summary state.
+    this.cumulativeEvaluations = [];
+    this.cumulativeEntries = [];
+    this.runUrl = "";
     // Target registry
     this.targets = /* @__PURE__ */ new Map();
     // Current iteration context (for log/evaluate calls)
@@ -330,7 +380,8 @@ var Experiment = class _Experiment {
       const data = await response.json();
       this.experimentSlug = data.slug;
       const encodedRunId = encodeURIComponent(this.runId);
-      console.log(`Follow results at: ${this.endpoint.replace(/\/$/, "")}${data.path}?runId=${encodedRunId}`);
+      this.runUrl = `${this.endpoint.replace(/\/$/, "")}${data.path}?runId=${encodedRunId}`;
+      console.log(`Follow results at: ${this.runUrl}`);
       this.initialized = true;
     } catch (error) {
       if (error instanceof ExperimentInitError) {
@@ -470,6 +521,12 @@ var Experiment = class _Experiment {
         trace_id: capturedTraceId != null ? capturedTraceId : this.getTraceIdFromContext()
       };
       this.batch.dataset.push(entry);
+      this.cumulativeEntries.push({
+        duration: entry.duration,
+        error: entry.error,
+        cost: entry.cost,
+        target_id: entry.target_id
+      });
     }
     this.iterationUsedWithTarget.delete(index);
     this.progress++;
@@ -536,6 +593,15 @@ var Experiment = class _Experiment {
       target_id: targetId != null ? targetId : null
     };
     this.batch.evaluations.push(result);
+    this.cumulativeEvaluations.push({
+      name: result.name,
+      evaluator: result.evaluator,
+      status: result.status,
+      passed: result.passed,
+      score: result.score,
+      cost: result.cost,
+      target_id: result.target_id
+    });
     this.scheduleSend();
   }
   /**
@@ -730,6 +796,12 @@ var Experiment = class _Experiment {
       predicted
     };
     this.batch.dataset.push(entry);
+    this.cumulativeEntries.push({
+      duration: entry.duration,
+      error: entry.error,
+      cost: entry.cost,
+      target_id: entry.target_id
+    });
     this.scheduleSend();
     return {
       result,
@@ -778,6 +850,128 @@ var Experiment = class _Experiment {
       }, DEBOUNCE_INTERVAL_MS - (now - this.lastSentMs));
     }
   }
+  /**
+   * Print a CI-friendly summary and optionally exit with code 1 on failure.
+   *
+   * Mirrors `ExperimentRunResult.printSummary` for parity — SDK-driven experiments
+   * can now fail CI builds the same way platform experiments do.
+   *
+   * @param exitOnFailure - If true (default), calls `process.exit(1)` when any evaluation failed.
+   *
+   * @example
+   * ```typescript
+   * const experiment = await langwatch.experiments.init("ci-quality-check");
+   * await experiment.run(dataset, async ({ item, index }) => {
+   *   const response = await myLLM(item.input);
+   *   await experiment.evaluate("ragas/faithfulness", { index, data: { input: item.input, output: response } });
+   * });
+   * experiment.printSummary();
+   * ```
+   */
+  printSummary(exitOnFailure = true) {
+    var _a, _b;
+    const evaluators = /* @__PURE__ */ new Map();
+    let totalPassed = 0;
+    let totalFailed = 0;
+    let totalCost = 0;
+    for (const e of this.cumulativeEvaluations) {
+      const name = (_b = (_a = e.name) != null ? _a : e.evaluator) != null ? _b : "unknown";
+      if (!evaluators.has(name)) {
+        evaluators.set(name, { passed: 0, failed: 0, scoreSum: 0, scoreCount: 0 });
+      }
+      const stats = evaluators.get(name);
+      if (e.status === "error" || e.passed === false) {
+        stats.failed += 1;
+        totalFailed += 1;
+      } else if (e.passed === true) {
+        stats.passed += 1;
+        totalPassed += 1;
+      }
+      if (typeof e.score === "number") {
+        stats.scoreSum += e.score;
+        stats.scoreCount += 1;
+      }
+      if (typeof e.cost === "number") {
+        totalCost += e.cost;
+      }
+    }
+    const targets = /* @__PURE__ */ new Map();
+    for (const entry of this.cumulativeEntries) {
+      const tid = entry.target_id;
+      if (!tid) continue;
+      if (!targets.has(tid)) {
+        targets.set(tid, { passed: 0, failed: 0, latencySum: 0, latencyCount: 0, cost: 0 });
+      }
+      const stats = targets.get(tid);
+      if (typeof entry.duration === "number") {
+        stats.latencySum += entry.duration;
+        stats.latencyCount += 1;
+      }
+      if (typeof entry.cost === "number") {
+        stats.cost += entry.cost;
+      }
+    }
+    for (const e of this.cumulativeEvaluations) {
+      const tid = e.target_id;
+      if (!tid) continue;
+      if (!targets.has(tid)) {
+        targets.set(tid, { passed: 0, failed: 0, latencySum: 0, latencyCount: 0, cost: 0 });
+      }
+      const stats = targets.get(tid);
+      if (e.status === "error" || e.passed === false) stats.failed += 1;
+      else if (e.passed === true) stats.passed += 1;
+      if (typeof e.cost === "number") stats.cost += e.cost;
+    }
+    const total = totalPassed + totalFailed;
+    const passRate = total > 0 ? totalPassed / total * 100 : 0;
+    const duration = Math.max(0, Date.now() - this.createdAtMs);
+    let failedCells = 0;
+    for (const entry of this.cumulativeEntries) {
+      if (typeof entry.cost === "number") totalCost += entry.cost;
+      if (entry.error) failedCells += 1;
+    }
+    const hasFailures = totalFailed > 0 || failedCells > 0;
+    printSummary({
+      runId: this.runId,
+      status: hasFailures ? "failed" : "completed",
+      passed: totalPassed,
+      failed: totalFailed,
+      passRate,
+      duration,
+      runUrl: this.runUrl,
+      summary: {
+        runId: this.runId,
+        totalCells: this.cumulativeEntries.length || total,
+        completedCells: Math.max(0, (this.cumulativeEntries.length || total) - failedCells),
+        failedCells,
+        duration,
+        runUrl: this.runUrl,
+        totalPassed,
+        totalFailed,
+        passRate,
+        totalCost,
+        targets: Array.from(targets.entries()).map(([targetId, s]) => ({
+          targetId,
+          name: targetId,
+          passed: s.passed,
+          failed: s.failed,
+          avgLatency: s.latencyCount > 0 ? s.latencySum / s.latencyCount : 0,
+          totalCost: s.cost
+        })),
+        evaluators: Array.from(evaluators.entries()).map(([name, s]) => ({
+          evaluatorId: name,
+          name,
+          passed: s.passed,
+          failed: s.failed,
+          passRate: s.passed + s.failed > 0 ? s.passed / (s.passed + s.failed) * 100 : 0,
+          avgScore: s.scoreCount > 0 ? s.scoreSum / s.scoreCount : void 0
+        }))
+      }
+    });
+    if (exitOnFailure && hasFailures) {
+      process.exit(1);
+    }
+  }
   /**
    * Send current batch to the API
    */
@@ -1117,7 +1311,7 @@ var ExperimentsFacade = class {
       summary,
       printSummary: (exitOnFailure = true) => {
         var _a2;
-        this.printSummary({
+        printSummary({
           runId,
           status,
           passed: totalPassed,
@@ -1136,50 +1330,6 @@ var ExperimentsFacade = class {
     result[Symbol.for("nodejs.util.inspect.custom")] = () => result.toString();
     return result;
   }
-  /**
-   * Print a CI-friendly summary of the experiment results
-   */
-  printSummary(result) {
-    const { runId, status, passed, failed, passRate, duration, runUrl, summary } = result;
-    console.log("\n" + "\u2550".repeat(60));
-    console.log("  EXPERIMENT RESULTS");
-    console.log("\u2550".repeat(60));
-    console.log(`  Run ID:     ${runId}`);
-    console.log(`  Status:     ${status.toUpperCase()}`);
-    console.log(`  Duration:   ${(duration / 1e3).toFixed(1)}s`);
-    console.log("\u2500".repeat(60));
-    console.log(`  Passed:     ${passed}`);
-    console.log(`  Failed:     ${failed}`);
-    console.log(`  Pass Rate:  ${passRate.toFixed(1)}%`);
-    if (summary.targets && summary.targets.length > 0) {
-      console.log("\u2500".repeat(60));
-      console.log("  TARGETS:");
-      for (const target of summary.targets) {
-        console.log(`    ${target.name}: ${target.passed} passed, ${target.failed} failed`);
-        if (target.avgLatency) {
-          console.log(`      Avg latency: ${target.avgLatency.toFixed(0)}ms`);
-        }
-        if (target.totalCost) {
-          console.log(`      Total cost: $${target.totalCost.toFixed(4)}`);
-        }
-      }
-    }
-    if (summary.evaluators && summary.evaluators.length > 0) {
-      console.log("\u2500".repeat(60));
-      console.log("  EVALUATORS:");
-      for (const evaluator of summary.evaluators) {
-        console.log(
-          `    ${evaluator.name}: ${evaluator.passRate.toFixed(1)}% pass rate`
-        );
-        if (evaluator.avgScore !== void 0) {
-          console.log(`      Avg score: ${evaluator.avgScore.toFixed(2)}`);
-        }
-      }
-    }
-    console.log("\u2500".repeat(60));
-    console.log(`  View details: ${runUrl}`);
-    console.log("\u2550".repeat(60) + "\n");
-  }
   sleep(ms) {
     return new Promise((resolve) => setTimeout(resolve, ms));
   }