langwatch 0.2.0 → 0.3.0-prerelease.1

package/src/__tests__/client.test.ts

@@ -0,0 +1,71 @@
+import { describe, it, expect, beforeEach, afterAll, vi } from "vitest";
+
+describe("client config", () => {
+  let client: any;
+
+  beforeEach(async () => {
+    // Mock environment variables before importing the client module
+    vi.stubEnv('LANGWATCH_API_KEY', undefined);
+    vi.stubEnv('LANGWATCH_ENDPOINT', undefined);
+
+    // Reset modules to ensure fresh import after env stubbing
+    vi.resetModules();
+
+    // Dynamically import the client module
+    client = await import("../client.js");
+
+    // Reset config before each test
+    client.setConfig({
+      apiKey: "",
+      endpoint: void 0,
+      disableOpenTelemetryAutomaticSetup: false,
+      disableAutomaticInputCapture: false,
+      disableAutomaticOutputCapture: false,
+    });
+  });
+
+  afterAll(() => {
+    // Restore original environment
+    vi.unstubAllEnvs();
+  });
+
+  it("should use default values if nothing is set", () => {
+    expect(client.getApiKey()).toBe("");
+    expect(client.getEndpoint()).toBe("https://app.langwatch.ai");
+    expect(client.canAutomaticallyCaptureInput()).toBe(true);
+    expect(client.canAutomaticallyCaptureOutput()).toBe(true);
+  });
+
+  it("should use env vars if set", () => {
+    process.env.LANGWATCH_API_KEY = "env-key";
+    process.env.LANGWATCH_ENDPOINT = "https://env.endpoint";
+
+    client.setConfig({});
+
+    expect(client.getApiKey()).toBe("env-key");
+    expect(client.getEndpoint()).toBe("https://env.endpoint");
+  });
+
+  it("should update config with setConfig", () => {
+    client.setConfig({
+      apiKey: "test-key",
+      endpoint: "https://test.endpoint",
+      disableAutomaticInputCapture: true,
+      disableAutomaticOutputCapture: true,
+    });
+    expect(client.getApiKey()).toBe("test-key");
+    expect(client.getEndpoint()).toBe("https://test.endpoint");
+    expect(client.canAutomaticallyCaptureInput()).toBe(false);
+    expect(client.canAutomaticallyCaptureOutput()).toBe(false);
+  });
+
+  it("should not override values if undefined is passed", () => {
+    client.setConfig({
+      apiKey: "first-key",
+      endpoint: "https://first.endpoint",
+    });
+    client.setConfig({ apiKey: undefined, endpoint: undefined });
+    expect(client.getApiKey()).toBe("first-key");
+    expect(client.getEndpoint()).toBe("https://first.endpoint");
+  });
+});

package/src/__tests__/integration/client-browser.test.ts

@@ -0,0 +1,46 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+import { setupLangWatch as setupBrowser } from '../../client-browser';
+
+// Mock window object for Node.js test environment
+const mockWindow = {
+  addEventListener: vi.fn(),
+  removeEventListener: vi.fn(),
+};
+global.window = mockWindow as any;
+
+describe('client-browser integration', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    mockWindow.addEventListener.mockClear();
+  });
+
+  afterEach(() => {
+    // Clean up global state by clearing the module cache
+    vi.resetModules();
+  });
+
+  it('should start and shut down the WebTracerProvider without error', async () => {
+    // First setup should complete without error
+    await expect(setupBrowser({
+      apiKey: 'integration-key',
+      endpoint: 'http://localhost:9999',
+      skipOpenTelemetrySetup: false,
+    })).resolves.not.toThrow();
+
+    // Second setup should also complete without error (triggers shutdown of previous)
+    await expect(setupBrowser({
+      apiKey: 'integration-key2',
+      endpoint: 'http://localhost:9999',
+      skipOpenTelemetrySetup: false,
+    })).resolves.not.toThrow();
+  });
+
+  it('should not set up WebTracerProvider if skipOpenTelemetrySetup is true', async () => {
+    // Should complete without error and without setting up WebTracerProvider
+    await expect(setupBrowser({
+      apiKey: 'integration-key',
+      endpoint: 'http://localhost:9999',
+      skipOpenTelemetrySetup: true,
+    })).resolves.not.toThrow();
+  });
+});

package/src/__tests__/integration/client-node.test.ts

@@ -0,0 +1,46 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+
+describe('client-node integration', () => {
+  let clientNode: any;
+
+  beforeEach(async () => {
+    vi.clearAllMocks();
+    vi.resetModules();
+    // Import fresh modules for each test to get clean state
+    clientNode = await import('../../client-node.js');
+  });
+
+  afterEach(() => {
+    // Clean up global state by clearing the module cache
+    vi.resetModules();
+  });
+
+  it('should start and shut down the NodeSDK without error', async () => {
+    // First setup should complete without error
+    await expect(clientNode.setupLangWatch({
+      apiKey: 'integration-key',
+      endpoint: 'http://localhost:9999',
+      disableOpenTelemetryAutomaticSetup: false,
+    })).resolves.not.toThrow();
+
+    // Reset module to clear setupCalled state for second call
+    vi.resetModules();
+    clientNode = await import('../../client-node.js');
+
+    // Second setup should also complete without error (triggers shutdown of previous)
+    await expect(clientNode.setupLangWatch({
+      apiKey: 'integration-key2',
+      endpoint: 'http://localhost:9999',
+      disableOpenTelemetryAutomaticSetup: false,
+    })).resolves.not.toThrow();
+  });
+
+  it('should not start NodeSDK if disableOpenTelemetryAutomaticSetup is true', async () => {
+    // Should complete without error and without setting up NodeSDK
+    await expect(clientNode.setupLangWatch({
+      apiKey: 'integration-key',
+      endpoint: 'http://localhost:9999',
+      disableOpenTelemetryAutomaticSetup: true,
+    })).resolves.not.toThrow();
+  });
+});

package/src/client-browser.ts

@@ -0,0 +1,70 @@
+import { setConfig, SetupOptions, getApiKey, getEndpoint } from "./client";
+import { SpanProcessor, WebTracerProvider } from '@opentelemetry/sdk-trace-web';
+import { ZoneContextManager } from '@opentelemetry/context-zone';
+import { W3CTraceContextPropagator } from '@opentelemetry/core';
+import { version } from "../package.json";
+import { resourceFromAttributes } from "@opentelemetry/resources";
+import * as intSemconv from "./observability/semconv";
+import { FilterableBatchSpanProcessor } from "./observability/processors";
+import { LangWatchExporter } from "./observability/exporters";
+import { addSpanProcessorToExistingTracerProvider, isOtelInitialized, mergeResourceIntoExistingTracerProvider } from "./client-shared";
+
+let managedSpanProcessors: SpanProcessor[] = [];
+let provider: WebTracerProvider | null = null;
+let browserSetupCalled: boolean = false;
+
+export async function setupLangWatch(options: SetupOptions = {}) {
+  if (browserSetupCalled) {
+    throw new Error("LangWatch setup has already been called in this process. Setup can only be called once, if you need to modify OpenTelemetry setup then use the OpenTelemetry API directly.");
+  }
+
+  setConfig(options);
+
+  if (options.skipOpenTelemetrySetup) return;
+
+  const endpointURL = new URL("/api/otel/v1/traces", getEndpoint());
+  const langwatchSpanProcessor = new FilterableBatchSpanProcessor(
+    new LangWatchExporter(getApiKey(), endpointURL.toString()),
+    options.otelSpanProcessingExcludeRules ?? [],
+  );
+
+  const langwatchResource = resourceFromAttributes({
+    ...options.baseAttributes,
+    [intSemconv.ATTR_LANGWATCH_SDK_LANGUAGE]: "typescript-browser",
+    [intSemconv.ATTR_LANGWATCH_SDK_VERSION]: version,
+    [intSemconv.ATTR_LANGWATCH_SDK_NAME]: "langwatch-observability-sdk",
+  });
+
+  if (isOtelInitialized()) {
+    mergeResourceIntoExistingTracerProvider(langwatchResource);
+    addSpanProcessorToExistingTracerProvider(langwatchSpanProcessor);
+    for (const spanProcessor of options.otelSpanProcessors ?? []) {
+      addSpanProcessorToExistingTracerProvider(spanProcessor);
+    }
+
+    managedSpanProcessors = [langwatchSpanProcessor];
+  } else {
+    provider = new WebTracerProvider({
+      resource: resourceFromAttributes({
+        [intSemconv.ATTR_LANGWATCH_SDK_LANGUAGE]: "typescript-browser",
+        [intSemconv.ATTR_LANGWATCH_SDK_VERSION]: version,
+        [intSemconv.ATTR_LANGWATCH_SDK_NAME]: "langwatch-observability-sdk",
+      }),
+      spanProcessors: [langwatchSpanProcessor, ...(options.otelSpanProcessors ?? [])],
+    });
+
+    provider.register({
+      contextManager: new ZoneContextManager(),
+      propagator: new W3CTraceContextPropagator(),
+    });
+  }
+
+  // This is not guaranteed to be called, but it's a good nice to have.
+  window.addEventListener("beforeunload", async () => {
+    if (provider) {
+      await provider.shutdown();
+    } else {
+      await Promise.all(managedSpanProcessors.map(p => p.shutdown()));
+    }
+  });
+}

package/src/client-node.ts

@@ -0,0 +1,82 @@
+import { SpanProcessor } from "@opentelemetry/sdk-trace-base";
+import { getApiKey, getEndpoint, setConfig, SetupOptions } from "./client";
+import { NodeSDK } from '@opentelemetry/sdk-node';
+import { resourceFromAttributes } from "@opentelemetry/resources";
+import { AsyncLocalStorageContextManager } from "@opentelemetry/context-async-hooks";
+import { W3CTraceContextPropagator } from "@opentelemetry/core";
+import { version } from "../package.json";
+import * as intSemconv from "./observability/semconv";
+import { addSpanProcessorToExistingTracerProvider, isOtelInitialized, mergeResourceIntoExistingTracerProvider } from "./client-shared";
+import { FilterableBatchSpanProcessor } from "./observability";
+import { LangWatchExporter } from "./observability/exporters";
+
+let managedSpanProcessors: SpanProcessor[] = [];
+let nodeSetupCalled: boolean = false;
+let sdk: NodeSDK | null = null;
+
+export async function setupLangWatch(options: SetupOptions = {}) {
+  if (nodeSetupCalled) {
+    throw new Error("LangWatch setup has already been called in this process. Setup can only be called once, if you need to modify OpenTelemetry setup then use the OpenTelemetry API directly.");
+  }
+
+  setConfig(options);
+  nodeSetupCalled = true;
+
+  if (options.skipOpenTelemetrySetup) return;
+
+  const endpointURL = new URL("/api/otel/v1/traces", getEndpoint());
+  const langwatchSpanProcessor = new FilterableBatchSpanProcessor(
+    new LangWatchExporter(getApiKey(), endpointURL.toString()),
+    options.otelSpanProcessingExcludeRules ?? [],
+  );
+
+  const langwatchResource = resourceFromAttributes({
+    ...options.baseAttributes,
+    [intSemconv.ATTR_LANGWATCH_SDK_LANGUAGE]: "typescript-node",
+    [intSemconv.ATTR_LANGWATCH_SDK_VERSION]: version,
+    [intSemconv.ATTR_LANGWATCH_SDK_NAME]: "langwatch-observability-sdk",
+  });
+
+  if (isOtelInitialized()) {
+    mergeResourceIntoExistingTracerProvider(langwatchResource);
+    addSpanProcessorToExistingTracerProvider(langwatchSpanProcessor);
+    for (const spanProcessor of options.otelSpanProcessors ?? []) {
+      addSpanProcessorToExistingTracerProvider(spanProcessor);
+    }
+
+    managedSpanProcessors = [langwatchSpanProcessor];
+  } else {
+    sdk = new NodeSDK({
+      resource: langwatchResource,
+      spanProcessors: [langwatchSpanProcessor, ...(options.otelSpanProcessors ?? [])],
+      contextManager: new AsyncLocalStorageContextManager(),
+      textMapPropagator: new W3CTraceContextPropagator(),
+    });
+
+    sdk.start();
+  }
+
+  // If we detect interrupt, termination, or test beforeExit signals, then we attempt
+  // to shutdown.
+  // - If an SDK exists, then we just attempt to shutdown the SDK.
+  // - If no SDK exists, then we attempt to shutdown ONLY the SpanProcessors that are
+  //   managed by this LangWatch SDK.
+  ["SIGINT", "SIGTERM", "beforeExit"].forEach((signal) => {
+    process.on(signal as any, async () => {
+      try {
+        if (sdk) {
+          await sdk.shutdown();
+        } else {
+          await Promise.all(managedSpanProcessors.map(p => p.shutdown()));
+        }
+      } catch (error) {
+        // eslint-disable-next-line no-console
+        console.error("Error shutting down OpenTelemetry SDK:", error);
+      }
+
+      if (signal !== "beforeExit") {
+        process.exit();
+      }
+    });
+  });
+}

package/src/client-shared.ts

@@ -0,0 +1,72 @@
+import {
+  trace,
+  ProxyTracerProvider,
+} from "@opentelemetry/api";
+import { Resource } from "@opentelemetry/resources";
+import { SpanProcessor } from "@opentelemetry/sdk-trace-base";
+
+/**
+ * Gets the actual tracer provider, handling the proxy delegate pattern.
+ *
+ * @returns The actual tracer provider or undefined if not available
+ */
+function getActualTracerProvider(): any {
+  const potentiallyProxyTracerProvider = trace.getTracerProvider() as unknown;
+
+  // Attempt to get the delegate if it's a ProxyTracerProvider
+  const delegate = (potentiallyProxyTracerProvider as ProxyTracerProvider | undefined)?.getDelegate?.();
+
+  // Return the delegate if available, otherwise return the original provider
+  return delegate ?? (potentiallyProxyTracerProvider as any);
+}
+
+/**
+ * Checks if the OpenTelemetry SDK has been initialized anywhere in the process.
+ *
+ * @returns true if the OpenTelemetry SDK has been initialized, false otherwise.
+ */
+export function isOtelInitialized() {
+  const provider = getActualTracerProvider();
+
+  // Check if the provider has the addSpanProcessor method, which indicates SDK initialization
+  return provider && typeof provider.addSpanProcessor === "function";
+}
+
+/**
+ * Merges a resource into the existing tracer provider.
+ *
+ * @param resource - The resource to merge into the existing tracer provider.
+ */
+export function mergeResourceIntoExistingTracerProvider(resource: Resource) {
+  if (!isOtelInitialized()) {
+    throw new Error("OpenTelemetry SDK is not initialized, cannot merge resource into existing tracer provider.");
+  }
+
+  const provider = getActualTracerProvider();
+
+  if (!provider?.resource) {
+    throw new Error("OpenTelemetry SDK is not initialized, provider does not have a resource.");
+  }
+  if (typeof resource !== "object") {
+    throw new Error("OpenTelemetry SDK is not initialized, provider resource is not an object.");
+  }
+  if (typeof provider.resource.merge !== "function") {
+    throw new Error("OpenTelemetry SDK is not initialized, provider resource does not have a merge method.");
+  }
+
+  provider.resource = provider.resource.merge(resource);
+}
+
+export function addSpanProcessorToExistingTracerProvider(spanProcessor: SpanProcessor) {
+  if (!isOtelInitialized()) {
+    throw new Error("OpenTelemetry SDK is not initialized, cannot add span processor to existing tracer provider.");
+  }
+
+  const provider = getActualTracerProvider();
+
+  if (!provider?.addSpanProcessor) {
+    throw new Error("OpenTelemetry SDK is not initialized, provider does not have a addSpanProcessor method.");
+  }
+
+  provider.addSpanProcessor(spanProcessor);
+}

package/src/client.ts

@@ -0,0 +1,119 @@
+import { SpanProcessor } from "@opentelemetry/sdk-trace-base";
+import { SpanProcessingExcludeRule } from "./observability";
+import { Attributes } from "@opentelemetry/api";
+
+export interface SetupOptions {
+  /**
+   * The API key to use for the LangWatch API.
+   */
+  apiKey?: string;
+
+  /**
+   * The endpoint to use for the LangWatch API.
+   */
+  endpoint?: string;
+
+  /**
+   * The span processors to use for the OpenTelemetry SDK.
+   *
+   * If provided, these will be added to the OpenTelemetry SDK after the LangWatch SDK has
+   * been initialized.
+   */
+  otelSpanProcessors?: SpanProcessor[];
+
+  /**
+   * The span processing exclude rules to use for the OpenTelemetry SDK.
+   *
+   * If provided, these will be added to the OpenTelemetry SDK after the LangWatch SDK has
+   * been initialized.
+   *
+   * If you are using the `otelSpanProcessors` option, then these will be ignored.
+   */
+  otelSpanProcessingExcludeRules?: SpanProcessingExcludeRule[];
+
+  /**
+   * Whether to skip the automatic setup of the OpenTelemetry SDK. If this is set, then
+   * the LangWatch SDK will not attempt to setup the OpenTelemetry SDK. You will need to
+   * setup the OpenTelemetry yourself, and ensure that a SpanProcessor is added to the
+   * OpenTelemetry SDK that will send traces to the LangWatch API.
+   */
+  skipOpenTelemetrySetup?: boolean;
+
+  /**
+   * Whether to disable the automatic capture of input.
+   */
+  disableAutomaticInputCapture?: boolean;
+
+  /**
+   * Whether to disable the automatic capture of output.
+   */
+  disableAutomaticOutputCapture?: boolean;
+
+  /**
+   * The base attributes to use for the OpenTelemetry SDK.
+   */
+  baseAttributes?: Attributes;
+}
+
+interface InternalConfig {
+  apiKey: string;
+  endpoint: string;
+  setupCalled: boolean;
+  skipOpenTelemetrySetup: boolean;
+  disableAutomaticInputCapture: boolean;
+  disableAutomaticOutputCapture: boolean;
+
+  baseAttributes: Attributes;
+}
+
+const config: InternalConfig = {
+  apiKey: process.env.LANGWATCH_API_KEY ?? "",
+  endpoint: process.env.LANGWATCH_ENDPOINT ?? "https://app.langwatch.ai",
+  setupCalled: false,
+  skipOpenTelemetrySetup: false,
+  disableAutomaticInputCapture: false,
+  disableAutomaticOutputCapture: false,
+  baseAttributes: {},
+};
+
+export function setConfig(options: SetupOptions) {
+  config.setupCalled = true;
+
+  config.apiKey = options.apiKey !== void 0
+    ? options.apiKey
+    : (process.env.LANGWATCH_API_KEY ?? config.apiKey);
+
+  config.endpoint = options.endpoint !== void 0
+    ? options.endpoint
+    : (process.env.LANGWATCH_ENDPOINT ?? config.endpoint);
+
+  if (config.apiKey === "") {
+    console.warn("[langwatch setup] No API key provided. Please set the LANGWATCH_API_KEY environment variable or pass it to the setup function. The SDK will perform no operations.");
+  }
+
+  config.skipOpenTelemetrySetup = options.skipOpenTelemetrySetup ?? config.skipOpenTelemetrySetup;
+  config.disableAutomaticInputCapture = options.disableAutomaticInputCapture ?? config.disableAutomaticInputCapture;
+  config.disableAutomaticOutputCapture = options.disableAutomaticOutputCapture ?? config.disableAutomaticOutputCapture;
+
+  config.baseAttributes = options.baseAttributes ?? config.baseAttributes;
+}
+
+export function getApiKey(): string {
+  return config.apiKey;
+}
+
+export function getEndpoint(): string {
+  return config.endpoint;
+}
+
+export function canAutomaticallyCaptureInput(): boolean {
+  return !config.disableAutomaticInputCapture;
+}
+
+export function canAutomaticallyCaptureOutput(): boolean {
+  return !config.disableAutomaticOutputCapture;
+}
+
+export function isSetupCalled(): boolean {
+  return config.setupCalled;
+}

package/src/evaluation/__tests__/record-evaluation.test.ts

@@ -0,0 +1,112 @@
+// --- Mock setup (must be at the top for Vitest hoisting) ---
+const { mockStartActiveSpan } = vi.hoisted(() => ({
+  mockStartActiveSpan: vi.fn((name, fn) => fn({
+    setType: vi.fn(),
+    addEvent: vi.fn(),
+    setOutput: vi.fn(),
+    setAttributes: vi.fn(),
+    setMetrics: vi.fn(),
+    setStatus: vi.fn(),
+    recordException: vi.fn(),
+    end: vi.fn(),
+  })),
+}));
+
+vi.mock('../tracer', () => ({ tracer: { startActiveSpan: mockStartActiveSpan } }));
+vi.mock('../../observability/semconv', () => ({
+  ATTR_LANGWATCH_EVALUATION_CUSTOM: 'custom_event',
+}));
+
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import { recordEvaluation } from '../record-evaluation';
+
+const baseDetails: import('../record-evaluation').RecordedEvaluationDetails = {
+  evaluationId: 'eval1',
+  name: 'test',
+  type: 'custom',
+  isGuardrail: false,
+  status: 'processed',
+  passed: true,
+  score: 1,
+  label: 'label',
+  details: 'ok',
+  cost: { currency: 'USD', amount: 0.1 },
+  error: undefined,
+  timestamps: { startedAtUnixMs: 1, finishedAtUnixMs: 2 },
+};
+
+describe('recordEvaluation', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it('records processed evaluation', () => {
+    const span = {
+      setType: vi.fn(),
+      addEvent: vi.fn(),
+      setOutput: vi.fn(),
+      setAttributes: vi.fn(),
+      setMetrics: vi.fn(),
+      setOutputEvaluation: vi.fn(),
+      recordException: vi.fn(),
+      end: vi.fn(),
+    };
+
+    mockStartActiveSpan.mockImplementationOnce((name, fn) => fn(span));
+    recordEvaluation({ ...baseDetails });
+
+    expect(mockStartActiveSpan).toHaveBeenCalledWith(
+      'record evaluation',
+      expect.any(Function)
+    );
+    expect(span.setType).toHaveBeenCalledWith('evaluation');
+    expect(span.addEvent).toHaveBeenCalledWith('custom_event', expect.objectContaining({
+      json_encoded_event: expect.stringContaining('"name":"test"')
+    }));
+    expect(span.setOutput).toHaveBeenCalledWith(expect.objectContaining({
+      status: 'processed',
+      passed: true,
+      score: 1,
+      label: 'label',
+      details: 'ok',
+      cost: { currency: 'USD', amount: 0.1 },
+    }));
+    expect(span.end).toHaveBeenCalled();
+  });
+
+  it('records skipped evaluation', () => {
+    recordEvaluation({ ...baseDetails, status: 'skipped', details: 'skipped' });
+    expect(mockStartActiveSpan).toHaveBeenCalled();
+  });
+
+  it('records error evaluation', () => {
+    recordEvaluation({ ...baseDetails, status: 'error', error: new Error('fail'), details: 'fail' });
+    expect(mockStartActiveSpan).toHaveBeenCalled();
+  });
+
+  it('sets cost metric if cost is present', () => {
+    recordEvaluation({ ...baseDetails, cost: { currency: 'USD', amount: 42 } });
+    // No assertion needed, just ensure no error
+  });
+
+  it('sets attributes if provided', () => {
+    const attrs = { foo: 'bar' };
+    recordEvaluation({ ...baseDetails }, attrs);
+    // No assertion needed, just ensure no error
+  });
+
+  it('handles error in span', () => {
+    const errorSpan = {
+      setType: vi.fn(() => { throw new Error('fail in span'); }),
+      addEvent: vi.fn(),
+      setOutput: vi.fn(),
+      setAttributes: vi.fn(),
+      setMetrics: vi.fn(),
+      recordException: vi.fn(),
+      end: vi.fn(),
+    };
+    expect(() => {
+      mockStartActiveSpan.mock.calls[0]?.[1]?.(errorSpan);
+    }).not.toThrow();
+  });
+});