langwatch 0.1.7 → 0.3.0-prerelease.1

package/src/evaluation/__tests__/run-evaluation.test.ts

@@ -0,0 +1,171 @@
+// --- Mock setup (must be at the top for Vitest hoisting) ---
+const { mockStartActiveSpan } = vi.hoisted(() => ({
+  mockStartActiveSpan: vi.fn((name, fn) => fn({
+    setType: vi.fn(),
+    setInput: vi.fn(),
+    setMetrics: vi.fn(),
+    setStatus: vi.fn(),
+    setOutputEvaluation: vi.fn(),
+    recordException: vi.fn(),
+    end: vi.fn(),
+    spanContext: () => ({ traceId: 'trace', spanId: 'span' }),
+  })),
+}));
+
+vi.mock('../tracer', () => ({ tracer: { startActiveSpan: mockStartActiveSpan } }));
+
+const mockFetch = vi.fn();
+globalThis.fetch = mockFetch;
+
+vi.mock('../../client', () => ({
+  canAutomaticallyCaptureInput: () => true,
+  getApiKey: () => 'test-key',
+  getEndpoint: () => 'https://api',
+}));
+
+// --- Imports (must be after mocks for Vitest hoisting) ---
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import { runEvaluation } from '../run-evaluation';
+import { LangWatchApiError } from '../../internal/api/errors';
+
+const baseProcessed = {
+  status: 'processed',
+  passed: true,
+  score: 1,
+  details: 'ok',
+  label: 'label',
+  cost: { currency: 'USD', amount: 0.1 },
+};
+const baseSkipped = { status: 'skipped', details: 'skipped' };
+const baseError = { status: 'error', details: 'fail', error_type: 'EvalError', traceback: ['trace'] };
+
+const details = {
+  name: 'test',
+  data: { input: 'foo', output: 'bar' },
+  evaluator: 'test-eval',
+};
+
+describe('runEvaluation', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it('returns processed result', async () => {
+    mockFetch.mockResolvedValueOnce({
+      ok: true,
+      json: async () => ({ ...baseProcessed }),
+    });
+    const result = await runEvaluation(details as any);
+    expect(result.status).toBe('processed');
+    if (result.status === 'processed') {
+      expect(result.passed).toBe(true);
+      expect(result.score).toBe(1);
+      expect(result.details).toBe('ok');
+      expect(result.label).toBe('label');
+      expect(result.cost).toEqual({ currency: 'USD', amount: 0.1 });
+    } else {
+      throw new Error('Expected processed result');
+    }
+    expect(mockFetch).toHaveBeenCalledWith(
+      expect.stringContaining('/api/evaluations/test-eval/evaluate'),
+      expect.objectContaining({ method: 'POST' })
+    );
+  });
+
+  it('returns skipped result', async () => {
+    mockFetch.mockResolvedValueOnce({
+      ok: true,
+      json: async () => ({ ...baseSkipped }),
+    });
+    const result = await runEvaluation(details as any);
+    expect(result.status).toBe('skipped');
+    expect(result.details).toBe('skipped');
+  });
+
+  it('returns error result', async () => {
+    mockFetch.mockResolvedValueOnce({
+      ok: true,
+      json: async () => ({ ...baseError }),
+    });
+    const result = await runEvaluation(details as any);
+    expect(result.status).toBe('error');
+    if (result.status === 'error') {
+      expect(result.details).toBe('fail');
+      expect(result.error_type).toBe('EvalError');
+      expect(result.traceback).toEqual(['trace']);
+    } else {
+      throw new Error('Expected error result');
+    }
+  });
+
+  it('returns unknown status as error', async () => {
+    mockFetch.mockResolvedValueOnce({
+      ok: true,
+      json: async () => ({ status: 'weird' }),
+    });
+    const result = await runEvaluation(details as any);
+    expect(result.status).toBe('error');
+    if (result.status === 'error') {
+      expect(result.error_type).toBe('UnknownStatus');
+      expect(result.details).toContain('Unknown evaluation status');
+    } else {
+      throw new Error('Expected error result');
+    }
+  });
+
+  it('throws LangWatchApiError on non-ok response', async () => {
+    mockFetch.mockResolvedValueOnce({ ok: false, json: async () => ({}), status: 400, statusText: 'Bad', headers: { get: () => 'application/json' } });
+    await expect(runEvaluation(details as any)).rejects.toBeInstanceOf(LangWatchApiError);
+  });
+
+  it('propagates fetch errors', async () => {
+    mockFetch.mockRejectedValueOnce(new Error('network fail'));
+    await expect(runEvaluation(details as any)).rejects.toThrow('network fail');
+  });
+
+  it('calls setInput if canAutomaticallyCaptureInput is true', async () => {
+    vi.resetModules();
+    vi.doMock('../../client', () => ({
+      canAutomaticallyCaptureInput: () => true,
+      getApiKey: () => 'test-key',
+      getEndpoint: () => 'https://api',
+    }));
+    const span = {
+      setType: vi.fn(),
+      setInput: vi.fn(),
+      setMetrics: vi.fn(),
+      setOutputEvaluation: vi.fn(),
+      recordException: vi.fn(),
+      end: vi.fn(),
+      spanContext: () => ({ traceId: 'trace', spanId: 'span' }),
+    };
+    mockStartActiveSpan.mockImplementationOnce((name, fn) => fn(span));
+    mockFetch.mockResolvedValueOnce({ ok: true, json: async () => ({ ...baseProcessed }) });
+    const { runEvaluation: runEval } = await import('../run-evaluation.js');
+    await runEval(details as any);
+    expect(span.setInput).toHaveBeenCalledWith(expect.objectContaining({ trace_id: 'trace' }));
+  });
+
+  it('does not call setInput if canAutomaticallyCaptureInput is false', async () => {
+    vi.resetModules();
+    vi.doMock('../../client', () => ({
+      canAutomaticallyCaptureInput: () => false,
+      getApiKey: () => 'test-key',
+      getEndpoint: () => 'https://api',
+    }));
+    const span = {
+      setType: vi.fn(),
+      setInput: vi.fn(),
+      setMetrics: vi.fn(),
+      setOutputEvaluation: vi.fn(),
+      recordException: vi.fn(),
+      end: vi.fn(),
+      spanContext: () => ({ traceId: 'trace', spanId: 'span' }),
+    };
+    mockStartActiveSpan.mockImplementationOnce((name, fn) => fn(span));
+    mockFetch.mockResolvedValueOnce({ ok: true, json: async () => ({ ...baseProcessed }) });
+    const { runEvaluation: runEval } = await import('../run-evaluation.js');
+    await runEval(details as any);
+    expect(span.setInput).not.toHaveBeenCalled();
+  });
+});

package/src/evaluation/index.ts

@@ -0,0 +1,2 @@
+export * from "./run-evaluation";
+export * from "./record-evaluation";

package/src/evaluation/record-evaluation.ts

@@ -0,0 +1,101 @@
+import { EvaluationRESTResult } from "../internal/generated/types/evaluations";
+import * as intSemconv from "../observability/semconv";
+import { Attributes, SpanStatusCode } from "@opentelemetry/api";
+import { generate } from "xksuid";
+import { tracer } from "./tracer";
+
+export interface RecordedEvaluationDetails {
+  evaluationId?: string;
+  name: string;
+  type?: string;
+  isGuardrail?: boolean;
+  status?: "processed" | "skipped" | "error";
+  passed?: boolean;
+  score?: number;
+  label?: string;
+  details?: string;
+  cost?: number | { currency: string; amount: number };
+  error?: Error;
+  timestamps?: {
+    startedAtUnixMs: number;
+    finishedAtUnixMs: number;
+  };
+}
+
+export function recordEvaluation(
+  details: RecordedEvaluationDetails,
+  attributes?: Attributes,
+) {
+  let result: EvaluationRESTResult;
+  const status = details.status || "processed";
+
+  if (status === "skipped") {
+    result = {
+      status: "skipped",
+      details: details.details,
+    };
+  } else if (status === "error") {
+    result = {
+      status: "error",
+      error_type: details.error?.name || "Unknown",
+      details: details.details || details.error?.message || "Unknown error",
+    };
+  } else {
+    result = {
+      status: "processed",
+      passed: details.passed,
+      score: details.score,
+      label: details.label,
+      details: details.details,
+    };
+    if (details.cost) {
+      (result as any).cost =
+        typeof details.cost === "number"
+          ? { currency: "USD", amount: details.cost }
+          : details.cost;
+    }
+  }
+
+  tracer.startActiveSpan("record evaluation", (span) => {
+    try {
+      span.setType(details.isGuardrail ? "guardrail" : "evaluation");
+      span.addEvent(intSemconv.ATTR_LANGWATCH_EVALUATION_CUSTOM, {
+        json_encoded_event: JSON.stringify({
+          evaluation_id: details.evaluationId ?? `eval_${generate()}`,
+          name: details.name,
+          type: details.type,
+          is_guardrail: details.isGuardrail,
+          status: result.status,
+          passed: details.passed,
+          score: details.score,
+          label: details.label,
+          details: details.details,
+          cost: details.cost,
+          error: details.error,
+          timestamps: details.timestamps,
+        }),
+      });
+
+      span.setOutput(result);
+
+      if (attributes) {
+        span.setAttributes(attributes);
+      }
+      if (details.cost) {
+        span.setMetrics({
+          cost:
+            typeof details.cost === "number"
+              ? details.cost
+              : details.cost.amount,
+        });
+      }
+    } catch (error) {
+      span.recordException(error as Error);
+      span.setStatus({ code: SpanStatusCode.ERROR, message: (error as Error)?.message });
+    } finally {
+      span.end();
+    }
+
+    return;
+  });
+}

package/src/evaluation/run-evaluation.ts

@@ -0,0 +1,133 @@
+import { LangWatchApiError } from "../internal/api/errors";
+import { canAutomaticallyCaptureInput, getApiKey, getEndpoint } from "../client";
+import { Conversation } from "../internal/generated/types/evaluations";
+import {
+  Evaluators,
+  EvaluatorTypes,
+  SingleEvaluationResult,
+} from "../internal/generated/types/evaluators.generated";
+import { RAGChunk } from "../internal/generated/types/tracer";
+import { tracer } from "./tracer";
+import { EvaluationResultModel } from "./types";
+import { SpanStatusCode } from "@opentelemetry/api";
+
+export interface BasicEvaluationData {
+  input?: string;
+  output?: string;
+  expected_output?: unknown;
+  contexts?: RAGChunk[] | string[];
+  expected_contexts?: RAGChunk[] | string[];
+  conversation?: Conversation;
+}
+
+export interface EvaluationDetailsBase {
+  name?: string;
+  data: BasicEvaluationData | Record<string, unknown>;
+  contexts?: RAGChunk[] | string[];
+  conversation?: Conversation;
+  asGuardrail?: boolean;
+}
+
+export interface SavedEvaluationDetails extends EvaluationDetailsBase {
+  slug: string;
+  settings?: Record<string, unknown>;
+}
+
+export interface LangEvalsEvaluationDetails<T extends EvaluatorTypes>
+  extends EvaluationDetailsBase {
+  evaluator: T;
+  settings?: Evaluators[T]["settings"];
+}
+
+export type EvaluationDetails =
+  | SavedEvaluationDetails
+  | LangEvalsEvaluationDetails<EvaluatorTypes>;
+
+export async function runEvaluation(
+  details: EvaluationDetails,
+): Promise<SingleEvaluationResult> {
+  return await tracer.startActiveSpan("run evaluation", async (span) => {
+    span.setType(details.asGuardrail ? "guardrail" : "evaluation");
+
+    try {
+      const evaluatorId =
+        "slug" in details ? details.slug : details.evaluator;
+      const request = {
+        trace_id: span.spanContext().traceId,
+        span_id: span.spanContext().spanId,
+        data: details.data,
+        name: details.name,
+        settings: details.settings,
+        as_guardrail: details.asGuardrail,
+      };
+
+      if (canAutomaticallyCaptureInput()) {
+        span.setInput(request);
+      }
+
+      const url = new URL(
+        `/api/evaluations/${evaluatorId}/evaluate`,
+        getEndpoint(),
+      );
+
+      const response = await fetch(url.toString(), {
+        method: "POST",
+        headers: {
+          "X-Auth-Token": getApiKey(),
+          "Content-Type": "application/json",
+        },
+        body: JSON.stringify(request),
+      });
+      if (!response.ok) {
+        const err = new LangWatchApiError("Unable to run evaluation", response);
+        await err.safeParseBody(response);
+
+        throw err;
+      }
+
+      const result: EvaluationResultModel = await response.json();
+
+      span.setMetrics({
+        cost: result.cost?.amount,
+      });
+
+      span.setOutputEvaluation(details.asGuardrail ?? false, result);
+
+      if (result.status === "processed") {
+        return {
+          status: "processed",
+          passed: result.passed,
+          score: result.score,
+          details: result.details,
+          label: result.label,
+          cost: result.cost,
+        } as SingleEvaluationResult;
+      } else if (result.status === "skipped") {
+        return {
+          status: "skipped",
+          details: result.details,
+        } as SingleEvaluationResult;
+      } else if (result.status === "error") {
+        return {
+          status: "error",
+          error_type: (result as any).error_type || "Unknown",
+          details: result.details || "Unknown error",
+          traceback: (result as any).traceback || [],
+        } as SingleEvaluationResult;
+      } else {
+        return {
+          status: "error",
+          error_type: "UnknownStatus",
+          details: `Unknown evaluation status: ${result.status}`,
+          traceback: [],
+        } as SingleEvaluationResult;
+      }
+    } catch (error) {
+      span.recordException(error as Error);
+      span.setStatus({ code: SpanStatusCode.ERROR, message: (error as Error)?.message });
+      throw error;
+    } finally {
+      span.end();
+    }
+  });
+}

package/src/evaluation/tracer.ts

@@ -0,0 +1,3 @@
+import { getLangWatchTracer } from "../observability/trace";
+
+export const tracer = getLangWatchTracer("langwatch.evaluation");

package/src/evaluation/types.ts

@@ -0,0 +1,23 @@
+export class EvaluationError extends Error {
+  readonly httpStatus: number;
+  readonly body: unknown;
+
+  constructor(message: string, httpStatus: number, body: unknown) {
+    super(message);
+    this.name = "EvaluationError";
+    this.httpStatus = httpStatus;
+    this.body = body;
+  }
+}
+
+export interface EvaluationResultModel {
+  status: "processed" | "skipped" | "error";
+  passed?: boolean;
+  score?: number;
+  details?: string;
+  label?: string;
+  cost?: {
+    currency: string;
+    amount: number;
+  };
+}