langwatch 0.3.0-prerelease.1 → 0.3.0-prerelease.2

package/src/evaluation/record-evaluation.ts

@@ -1,101 +0,0 @@
-import { EvaluationRESTResult } from "../internal/generated/types/evaluations";
-import * as intSemconv from "../observability/semconv";
-import { Attributes, SpanStatusCode } from "@opentelemetry/api";
-import { generate } from "xksuid";
-import { tracer } from "./tracer";
-
-export interface RecordedEvaluationDetails {
-  evaluationId?: string;
-  name: string;
-  type?: string;
-  isGuardrail?: boolean;
-  status?: "processed" | "skipped" | "error";
-  passed?: boolean;
-  score?: number;
-  label?: string;
-  details?: string;
-  cost?: number | { currency: string; amount: number };
-  error?: Error;
-  timestamps?: {
-    startedAtUnixMs: number;
-    finishedAtUnixMs: number;
-  };
-}
-
-export function recordEvaluation(
-  details: RecordedEvaluationDetails,
-  attributes?: Attributes,
-) {
-  let result: EvaluationRESTResult;
-  const status = details.status || "processed";
-
-  if (status === "skipped") {
-    result = {
-      status: "skipped",
-      details: details.details,
-    };
-  } else if (status === "error") {
-    result = {
-      status: "error",
-      error_type: details.error?.name || "Unknown",
-      details: details.details || details.error?.message || "Unknown error",
-    };
-  } else {
-    result = {
-      status: "processed",
-      passed: details.passed,
-      score: details.score,
-      label: details.label,
-      details: details.details,
-    };
-    if (details.cost) {
-      (result as any).cost =
-        typeof details.cost === "number"
-          ? { currency: "USD", amount: details.cost }
-          : details.cost;
-    }
-  }
-
-  tracer.startActiveSpan("record evaluation", (span) => {
-    try {
-      span.setType(details.isGuardrail ? "guardrail" : "evaluation");
-      span.addEvent(intSemconv.ATTR_LANGWATCH_EVALUATION_CUSTOM, {
-        json_encoded_event: JSON.stringify({
-          evaluation_id: details.evaluationId ?? `eval_${generate()}`,
-          name: details.name,
-          type: details.type,
-          is_guardrail: details.isGuardrail,
-          status: result.status,
-          passed: details.passed,
-          score: details.score,
-          label: details.label,
-          details: details.details,
-          cost: details.cost,
-          error: details.error,
-          timestamps: details.timestamps,
-        }),
-      });
-
-      span.setOutput(result);
-
-      if (attributes) {
-        span.setAttributes(attributes);
-      }
-      if (details.cost) {
-        span.setMetrics({
-          cost:
-            typeof details.cost === "number"
-              ? details.cost
-              : details.cost.amount,
-        });
-      }
-    } catch (error) {
-      span.recordException(error as Error);
-      span.setStatus({ code: SpanStatusCode.ERROR, message: (error as Error)?.message });
-    } finally {
-      span.end();
-    }
-
-    return;
-  });
-}

package/src/evaluation/run-evaluation.ts

@@ -1,133 +0,0 @@
-import { LangWatchApiError } from "../internal/api/errors";
-import { canAutomaticallyCaptureInput, getApiKey, getEndpoint } from "../client";
-import { Conversation } from "../internal/generated/types/evaluations";
-import {
-  Evaluators,
-  EvaluatorTypes,
-  SingleEvaluationResult,
-} from "../internal/generated/types/evaluators.generated";
-import { RAGChunk } from "../internal/generated/types/tracer";
-import { tracer } from "./tracer";
-import { EvaluationResultModel } from "./types";
-import { SpanStatusCode } from "@opentelemetry/api";
-
-export interface BasicEvaluationData {
-  input?: string;
-  output?: string;
-  expected_output?: unknown;
-  contexts?: RAGChunk[] | string[];
-  expected_contexts?: RAGChunk[] | string[];
-  conversation?: Conversation;
-}
-
-export interface EvaluationDetailsBase {
-  name?: string;
-  data: BasicEvaluationData | Record<string, unknown>;
-  contexts?: RAGChunk[] | string[];
-  conversation?: Conversation;
-  asGuardrail?: boolean;
-}
-
-export interface SavedEvaluationDetails extends EvaluationDetailsBase {
-  slug: string;
-  settings?: Record<string, unknown>;
-}
-
-export interface LangEvalsEvaluationDetails<T extends EvaluatorTypes>
-  extends EvaluationDetailsBase {
-  evaluator: T;
-  settings?: Evaluators[T]["settings"];
-}
-
-export type EvaluationDetails =
-  | SavedEvaluationDetails
-  | LangEvalsEvaluationDetails<EvaluatorTypes>;
-
-export async function runEvaluation(
-  details: EvaluationDetails,
-): Promise<SingleEvaluationResult> {
-  return await tracer.startActiveSpan("run evaluation", async (span) => {
-    span.setType(details.asGuardrail ? "guardrail" : "evaluation");
-
-    try {
-      const evaluatorId =
-        "slug" in details ? details.slug : details.evaluator;
-      const request = {
-        trace_id: span.spanContext().traceId,
-        span_id: span.spanContext().spanId,
-        data: details.data,
-        name: details.name,
-        settings: details.settings,
-        as_guardrail: details.asGuardrail,
-      };
-
-      if (canAutomaticallyCaptureInput()) {
-        span.setInput(request);
-      }
-
-      const url = new URL(
-        `/api/evaluations/${evaluatorId}/evaluate`,
-        getEndpoint(),
-      );
-
-      const response = await fetch(url.toString(), {
-        method: "POST",
-        headers: {
-          "X-Auth-Token": getApiKey(),
-          "Content-Type": "application/json",
-        },
-        body: JSON.stringify(request),
-      });
-      if (!response.ok) {
-        const err = new LangWatchApiError("Unable to run evaluation", response);
-        await err.safeParseBody(response);
-
-        throw err;
-      }
-
-      const result: EvaluationResultModel = await response.json();
-
-      span.setMetrics({
-        cost: result.cost?.amount,
-      });
-
-      span.setOutputEvaluation(details.asGuardrail ?? false, result);
-
-      if (result.status === "processed") {
-        return {
-          status: "processed",
-          passed: result.passed,
-          score: result.score,
-          details: result.details,
-          label: result.label,
-          cost: result.cost,
-        } as SingleEvaluationResult;
-      } else if (result.status === "skipped") {
-        return {
-          status: "skipped",
-          details: result.details,
-        } as SingleEvaluationResult;
-      } else if (result.status === "error") {
-        return {
-          status: "error",
-          error_type: (result as any).error_type || "Unknown",
-          details: result.details || "Unknown error",
-          traceback: (result as any).traceback || [],
-        } as SingleEvaluationResult;
-      } else {
-        return {
-          status: "error",
-          error_type: "UnknownStatus",
-          details: `Unknown evaluation status: ${result.status}`,
-          traceback: [],
-        } as SingleEvaluationResult;
-      }
-    } catch (error) {
-      span.recordException(error as Error);
-      span.setStatus({ code: SpanStatusCode.ERROR, message: (error as Error)?.message });
-      throw error;
-    } finally {
-      span.end();
-    }
-  });
-}

package/src/evaluation/tracer.ts

@@ -1,3 +0,0 @@
-import { getLangWatchTracer } from "../observability/trace";
-
-export const tracer = getLangWatchTracer("langwatch.evaluation");

package/src/evaluation/types.ts

@@ -1,23 +0,0 @@
-export class EvaluationError extends Error {
-  readonly httpStatus: number;
-  readonly body: unknown;
-
-  constructor(message: string, httpStatus: number, body: unknown) {
-    super(message);
-    this.name = "EvaluationError";
-    this.httpStatus = httpStatus;
-    this.body = body;
-  }
-}
-
-export interface EvaluationResultModel {
-  status: "processed" | "skipped" | "error";
-  passed?: boolean;
-  score?: number;
-  details?: string;
-  label?: string;
-  cost?: {
-    currency: string;
-    amount: number;
-  };
-}

package/src/index.ts

@@ -1,13 +0,0 @@
-export { getLangWatchTracer, type LangWatchSpan } from "./observability";
-export {
-  FilterableBatchSpanProcessor,
-  type SpanProcessingExcludeRule,
-} from "./observability/processors";
-export { LangWatchExporter } from "./observability/exporters";
-
-export { recordEvaluation, runEvaluation } from "./evaluation";
-
-export {
-  getPrompt,
-  getPromptVersion,
-} from "./prompt";

package/src/internal/api/__tests__/errors.test.ts

@@ -1,98 +0,0 @@
-import { describe, it, expect, vi } from 'vitest';
-import { LangWatchApiError } from '../errors';
-
-describe('LangWatchApiError', () => {
-  it('creates error with message and response', () => {
-    const mockResponse = {
-      status: 400,
-      statusText: 'Bad Request'
-    } as Response;
-
-    const error = new LangWatchApiError('Test error message', mockResponse);
-    expect(error.message).toBe('Test error message');
-    expect(error.name).toBe('Error');
-    expect(error.httpStatus).toBe(400);
-    expect(error.httpStatusText).toBe('Bad Request');
-  });
-
-  it('inherits from Error', () => {
-    const mockResponse = { status: 500, statusText: 'Internal Server Error' } as Response;
-    const error = new LangWatchApiError('Test error', mockResponse);
-    expect(error).toBeInstanceOf(Error);
-    expect(error).toBeInstanceOf(LangWatchApiError);
-  });
-
-  it('has stack trace', () => {
-    const mockResponse = { status: 404, statusText: 'Not Found' } as Response;
-    const error = new LangWatchApiError('Test error', mockResponse);
-    expect(error.stack).toBeDefined();
-    expect(typeof error.stack).toBe('string');
-  });
-
-  it('parses JSON response body', async () => {
-    const mockResponse = {
-      status: 400,
-      statusText: 'Bad Request',
-      headers: {
-        get: vi.fn().mockReturnValue('application/json')
-      },
-      json: vi.fn().mockResolvedValue({ error: 'Invalid API key' })
-    } as unknown as Response;
-
-    const error = new LangWatchApiError('Bad request', mockResponse);
-    await error.safeParseBody(mockResponse);
-
-    expect(error.body).toEqual({ error: 'Invalid API key' });
-    expect(error.apiError).toBe('Invalid API key');
-  });
-
-  it('parses text response body for non-JSON content', async () => {
-    const mockResponse = {
-      status: 500,
-      statusText: 'Internal Server Error',
-      headers: {
-        get: vi.fn().mockReturnValue('text/plain')
-      },
-      text: vi.fn().mockResolvedValue('Server error occurred')
-    } as unknown as Response;
-
-    const error = new LangWatchApiError('Server error', mockResponse);
-    await error.safeParseBody(mockResponse);
-
-    expect(error.body).toBe('Server error occurred');
-    expect(error.apiError).toBeUndefined();
-  });
-
-  it('handles parsing errors gracefully', async () => {
-    const mockResponse = {
-      status: 500,
-      statusText: 'Internal Server Error',
-      headers: {
-        get: vi.fn().mockReturnValue('application/json')
-      },
-      json: vi.fn().mockRejectedValue(new Error('Parse error'))
-    } as unknown as Response;
-
-    const error = new LangWatchApiError('Server error', mockResponse);
-    await error.safeParseBody(mockResponse);
-
-    expect(error.body).toBe(null);
-  });
-
-  it('handles JSON without error field', async () => {
-    const mockResponse = {
-      status: 400,
-      statusText: 'Bad Request',
-      headers: {
-        get: vi.fn().mockReturnValue('application/json')
-      },
-      json: vi.fn().mockResolvedValue({ message: 'Something went wrong' })
-    } as unknown as Response;
-
-    const error = new LangWatchApiError('Bad request', mockResponse);
-    await error.safeParseBody(mockResponse);
-
-    expect(error.body).toEqual({ message: 'Something went wrong' });
-    expect(error.apiError).toBeUndefined();
-  });
-});

package/src/internal/api/client.ts

@@ -1,30 +0,0 @@
-import openApiCreateClient from "openapi-fetch";
-import type { paths } from "../generated/openapi/api-client";
-import { z } from "zod";
-import { getApiKey, getEndpoint } from "../../client";
-
-// Define the client type explicitly to avoid naming issues
-export type LangwatchApiClient = ReturnType<typeof openApiCreateClient<paths>>;
-
-const configSchema = z.object({
-  apiKey: z.string().min(1, "API key is required"),
-  endpoint: z.string().url("Endpoint must be a valid URL"),
-});
-
-export function createLangWatchApiClient(apiKey?: string | undefined, endpoint?: string | undefined ): LangwatchApiClient {
-  // This will error if the config is invalid
-  const config = configSchema.parse({
-    apiKey: apiKey ?? getApiKey(),
-    endpoint: endpoint ?? getEndpoint(),
-  });
-
-  return openApiCreateClient<paths>({
-    baseUrl: config.endpoint,
-    headers: {
-      "X-Auth-Token": config.apiKey,
-      "Content-Type": "application/json",
-    },
-  });
-}
-
-

package/src/internal/api/errors.ts

@@ -1,32 +0,0 @@
-export class LangWatchApiError extends Error {
-  public readonly httpStatus: number;
-  public readonly httpStatusText: string;
-  public apiError: string | undefined;
-  public body: unknown;
-
-  constructor(message: string, response: Response) {
-    super(message);
-    this.httpStatus = response.status;
-    this.httpStatusText = response.statusText;
-  }
-
-  async safeParseBody(response: Response): Promise<void> {
-    try {
-      if (response.headers.get("Content-Type")?.includes("application/json")) {
-        const json = await response.json();
-
-        this.body = json;
-
-        if (json.error && typeof json.error === "string") {
-          this.apiError = json.error;
-        }
-
-        return;
-      }
-
-      this.body = await response.text();
-    } catch {
-      this.body = null;
-    }
-  }
-}

package/src/observability/__tests__/integration/base.test.ts

@@ -1,74 +0,0 @@
-import { beforeAll, describe, expect, it } from "vitest";
-import { setupLangWatch } from "../../../client-node";
-import { getLangWatchTracer } from "../../trace";
-
-const tracerName = "basic-observability.test";
-
-describe("basic observability tests around tracing", () => {
-  beforeAll(async () => {
-    await setupLangWatch();
-  });
-
-  it("traces should be sent", async () => {
-    const tracer = getLangWatchTracer(tracerName);
-    await tracer.withActiveSpan(
-      "basic trace",
-      async () => { },
-    );
-  });
-
-  it("traces should be sent with complex arguments", async () => {
-    const tracer = getLangWatchTracer(tracerName);
-    await tracer.withActiveSpan(
-      "complex argument trace",
-      { attributes: { foo: "bar" }, root: true },
-      async (span) => {
-        span.setAttributes({
-          bar: "bas",
-        });
-        span.addEvent("test event", {
-          foo: "bar",
-        });
-      },
-    );
-  });
-
-  it("traces exceptions", async () => {
-    const tracer = getLangWatchTracer(tracerName);
-
-    await expect(
-      tracer.withActiveSpan(
-        "trace exception",
-        async () => {
-          throw new Error("this is meant to error");
-        },
-      )
-    ).rejects.toThrow("this is meant to error");
-  });
-
-  it("traces handle complex nesting", async () => {
-    const tracer = getLangWatchTracer(tracerName);
-    await tracer.withActiveSpan(
-      "complex nesting trace",
-      async () => {
-        await tracer.withActiveSpan(
-          "nested trace alpha",
-          async () => { },
-        );
-        await tracer.withActiveSpan(
-          "nested trace beta",
-          async () => { },
-        );
-        await tracer.withActiveSpan(
-          "nested trace gamma",
-          async () => {
-            await tracer.withActiveSpan(
-              "nested trace gamma child",
-              async () => { },
-            );
-          },
-        );
-      },
-    );
-  });
-});

package/src/observability/__tests__/integration/browser-setup-ordering.test.ts

@@ -1,60 +0,0 @@
-import { describe, it, expect, vi } from "vitest";
-import { setupLangWatch as setupBrowser } from "../../../client-browser";
-import { getLangWatchTracer } from "../../trace";
-
-// Mock window object for Node.js test environment
-const mockWindow = {
-  addEventListener: vi.fn(),
-  removeEventListener: vi.fn(),
-};
-global.window = mockWindow as any;
-
-describe("Browser SDK setup ordering", () => {
-  it("should work when LangWatch browser setup is called multiple times", async () => {
-    // First setup
-    await setupBrowser({
-      apiKey: "test-key-1",
-      endpoint: "http://localhost:9999",
-      skipOpenTelemetrySetup: false,
-    });
-
-    // Second setup should work (browser doesn't have the same restriction as node)
-    await setupBrowser({
-      apiKey: "test-key-2",
-      endpoint: "http://localhost:9999",
-      skipOpenTelemetrySetup: false,
-    });
-
-    // Test that LangWatch still works
-    const tracer = getLangWatchTracer("browser-multiple-test");
-    await tracer.withActiveSpan("test span", async () => {
-      expect(true).toBe(true);
-    });
-  });
-
-  it("should work when LangWatch is set up with skipOpenTelemetrySetup=true", async () => {
-    // Setup LangWatch with automatic setup disabled
-    await setupBrowser({
-      apiKey: "test-key",
-      endpoint: "http://localhost:9999",
-      skipOpenTelemetrySetup: true,
-    });
-
-    // Test that LangWatch still works
-    const tracer = getLangWatchTracer("browser-disabled-test");
-    await tracer.withActiveSpan("test span", async () => {
-      expect(true).toBe(true);
-    });
-  });
-
-  it("should handle window event listeners correctly", async () => {
-    await setupBrowser({
-      apiKey: "test-key",
-      endpoint: "http://localhost:9999",
-      skipOpenTelemetrySetup: false,
-    });
-
-    // Check that window event listeners were added
-    expect(mockWindow.addEventListener).toHaveBeenCalled();
-  });
-});

package/src/observability/__tests__/integration/complex-nested-spans.test.ts

@@ -1,29 +0,0 @@
-import { describe, it, expect } from "vitest";
-import { registerOTel } from '@vercel/otel';
-import { setupLangWatch } from "../../../client-node";
-import { getLangWatchTracer } from "../../trace";
-
-describe("Complex nested spans with Vercel AI", () => {
-  it("should work with complex nested spans when Vercel AI is set up first", async () => {
-    registerOTel({ serviceName: 'complex-test' });
-    await setupLangWatch({
-      apiKey: "test-key",
-      endpoint: "http://localhost:9999",
-      skipOpenTelemetrySetup: false,
-    });
-
-    const tracer = getLangWatchTracer("complex-otel-test");
-
-    await tracer.withActiveSpan("root span", async () => {
-      await tracer.withActiveSpan("child span 1", async () => {
-        await tracer.withActiveSpan("grandchild span", async () => {
-          expect(true).toBe(true);
-        });
-      });
-
-      await tracer.withActiveSpan("child span 2", async () => {
-        expect(true).toBe(true);
-      });
-    });
-  });
-});

package/src/observability/__tests__/integration/error-handling.test.ts

@@ -1,24 +0,0 @@
-import { describe, it, expect } from "vitest";
-import { registerOTel } from '@vercel/otel';
-import { setupLangWatch } from "../../../client-node";
-import { getLangWatchTracer } from "../../trace";
-
-describe("Error handling with Vercel AI", () => {
-  it("should handle errors gracefully when both are set up", async () => {
-    registerOTel({ serviceName: 'error-test' });
-    await setupLangWatch({
-      apiKey: "test-key",
-      endpoint: "http://localhost:9999",
-      skipOpenTelemetrySetup: false,
-    });
-
-    const tracer = getLangWatchTracer("error-otel-test");
-
-    // Test that exceptions in spans are properly handled
-    await expect(
-      tracer.withActiveSpan("error span", async () => {
-        throw new Error("Test error");
-      })
-    ).rejects.toThrow("Test error");
-  });
-});

package/src/observability/__tests__/integration/langwatch-disabled-otel.test.ts

@@ -1,24 +0,0 @@
-import { describe, it, expect } from "vitest";
-import { registerOTel } from '@vercel/otel';
-import { setupLangWatch } from "../../../client-node";
-import { getLangWatchTracer } from "../../trace";
-
-describe("LangWatch with skipOpenTelemetrySetup=true", () => {
-  it("should work when LangWatch is set up with skipOpenTelemetrySetup=true", async () => {
-    // Setup Vercel AI first
-    registerOTel({ serviceName: 'vercel-first' });
-
-    // Then setup LangWatch with automatic setup disabled
-    await setupLangWatch({
-      apiKey: "test-key",
-      endpoint: "http://localhost:9999",
-      skipOpenTelemetrySetup: true,
-    });
-
-    // Test that LangWatch still works
-    const tracer = getLangWatchTracer("disabled-otel-test");
-    await tracer.withActiveSpan("test span", async () => {
-      expect(true).toBe(true);
-    });
-  });
-});

package/src/observability/__tests__/integration/langwatch-first-then-vercel.test.ts

@@ -1,24 +0,0 @@
-import { describe, it, expect } from "vitest";
-import { registerOTel } from '@vercel/otel';
-import { setupLangWatch } from "../../../client-node";
-import { getLangWatchTracer } from "../../trace";
-
-describe("LangWatch setup first, then Vercel AI", () => {
-  it("should work when LangWatch is set up first, then Vercel AI", async () => {
-    // Setup LangWatch first
-    await setupLangWatch({
-      apiKey: "test-key",
-      endpoint: "http://localhost:9999",
-      skipOpenTelemetrySetup: false,
-    });
-
-    // Then setup Vercel AI
-    registerOTel({ serviceName: 'langwatch-first' });
-
-    // Test that LangWatch still works
-    const tracer = getLangWatchTracer("langwatch-first-test");
-    await tracer.withActiveSpan("test span", async () => {
-      expect(true).toBe(true);
-    });
-  });
-});