langwatch 0.2.0 → 0.3.0-prerelease.1

package/src/observability/trace.ts

@@ -0,0 +1,301 @@
+import {
+  trace as otelTrace,
+  Tracer,
+  Span,
+  SpanOptions,
+  Context,
+  SpanStatusCode,
+} from "@opentelemetry/api";
+import { LangWatchSpan, createLangWatchSpan } from "./span";
+
+/**
+ * LangWatch OpenTelemetry Tracing Extensions
+ *
+ * This module provides wrappers and helpers for OpenTelemetry Tracer and Span objects,
+ * adding ergonomic methods for LLM/GenAI observability and structured tracing.
+ *
+ * @module trace
+ */
+export interface LangWatchTracer extends Tracer {
+  /**
+   * Starts a new {@link LangWatchSpan}. Start the span without setting it on context.
+   *
+   * This method does NOT modify the current Context.
+   *
+   * @param name The name of the span
+   * @param [options] SpanOptions used for span creation
+   * @param [context] Context to use to extract parent
+   * @returns LangWatchSpan The newly created span
+   *
+   * @example
+   *     const span = tracer.startSpan('op');
+   *     span.setAttribute('key', 'value');
+   *     span.end();
+   */
+  startSpan(
+    name: string,
+    options?: SpanOptions,
+    context?: Context,
+  ): LangWatchSpan;
+
+  /**
+   * Starts a new {@link LangWatchSpan} and calls the given function passing it the
+   * created span as first argument.
+   * Additionally the new span gets set in context and this context is activated
+   * for the duration of the function call.
+   *
+   * @param name The name of the span
+   * @param [options] SpanOptions used for span creation
+   * @param [context] Context to use to extract parent
+   * @param fn function called in the context of the span and receives the newly created span as an argument
+   * @returns return value of fn
+   *
+   * @example
+   *     const result = tracer.startActiveSpan('op', span => {
+   *       try {
+   *         // do some work
+   *         span.setStatus({code: SpanStatusCode.OK});
+   *         return something;
+   *       } catch (err) {
+   *         span.setStatus({
+   *           code: SpanStatusCode.ERROR,
+   *           message: err.message,
+   *         });
+   *         throw err;
+   *       } finally {
+   *         span.end();
+   *       }
+   *     });
+   *
+   * @example
+   *     const span = tracer.startActiveSpan('op', span => {
+   *       try {
+   *         do some work
+   *         return span;
+   *       } catch (err) {
+   *         span.setStatus({
+   *           code: SpanStatusCode.ERROR,
+   *           message: err.message,
+   *         });
+   *         throw err;
+   *       }
+   *     });
+   *     do some more work
+   *     span.end();
+   */
+  startActiveSpan<F extends (span: LangWatchSpan) => unknown>(
+    name: string,
+    fn: F,
+  ): ReturnType<F>;
+  startActiveSpan<F extends (span: LangWatchSpan) => unknown>(
+    name: string,
+    options: SpanOptions,
+    fn: F,
+  ): ReturnType<F>;
+  startActiveSpan<F extends (span: LangWatchSpan) => unknown>(
+    name: string,
+    options: SpanOptions,
+    context: Context,
+    fn: F,
+  ): ReturnType<F>;
+
+  /**
+   * Starts a new {@link LangWatchSpan}, runs the provided async function, and automatically handles
+   * error recording, status setting, and span ending. This is a safer and more ergonomic alternative
+   * to manually using try/catch/finally blocks with startActiveSpan.
+   *
+   * Overloads:
+   * - withActiveSpan(name, fn)
+   * - withActiveSpan(name, options, fn)
+   * - withActiveSpan(name, options, context, fn)
+   *
+   * @param name The name of the span
+   * @param options Optional SpanOptions for span creation
+   * @param context Optional Context to use to extract parent
+   * @param fn   The async function to execute within the span context. Receives the span as its first argument.
+   * @returns The return value of the provided function
+   *
+   * @example
+   *   await tracer.withActiveSpan('my-operation', async (span) => {
+   *     // ... your code ...
+   *   });
+   *
+   *   await tracer.withActiveSpan('my-operation', { attributes: { foo: 'bar' } }, async (span) => {
+   *     // ... your code ...
+   *   });
+   *
+   *   await tracer.withActiveSpan('my-operation', { attributes: { foo: 'bar' } }, myContext, async (span) => {
+   *     // ... your code ...
+   *   });
+   */
+  withActiveSpan<T>(
+    name: string,
+    fn: (span: LangWatchSpan) => Promise<T> | T
+  ): Promise<T>;
+  withActiveSpan<T>(
+    name: string,
+    options: SpanOptions,
+    fn: (span: LangWatchSpan) => Promise<T> | T
+  ): Promise<T>;
+  withActiveSpan<T>(
+    name: string,
+    options: SpanOptions,
+    context: Context,
+    fn: (span: LangWatchSpan) => Promise<T> | T
+  ): Promise<T>;
+}
+
+/**
+ * Extension of OpenTelemetry's Tracer with LangWatch-specific helpers.
+ *
+ * This interface provides methods for starting spans and active spans that return LangWatchSpan objects,
+ * which include ergonomic helpers for LLM/GenAI tracing.
+ *
+ * @example
+ * import { getLangWatchTracer } from 'langwatch';
+ * const tracer = getLangWatchTracer('my-service');
+ * const span = tracer.startSpan('llm-call');
+ * span.setType('llm').setInput('Prompt').setOutput('Completion');
+ * span.end();
+ *
+ * tracer.startActiveSpan('llm-call', (span) => {
+ *   span.setType('llm');
+ *   // ...
+ *   span.end();
+ * });
+ */
+export function getLangWatchTracer(name: string, version?: string): LangWatchTracer {
+  const tracer = otelTrace.getTracer(name, version);
+
+  // Create a proxy for the tracer that intercepts the calls to startActiveSpan and
+  // startSpan, and wraps the span object with our custom LangWatchSpan.
+  const handler: ProxyHandler<LangWatchTracer> = {
+    get(target, prop, _receiver) {
+      switch (prop) {
+        case "startActiveSpan": {
+          const startActiveSpan: StartActiveSpanOverloads = (
+            ...args: [
+              string,
+              SpanOptions?,
+              Context?,
+              ((span: Span) => unknown)?,
+            ]
+          ) => {
+            // Find the span callback function (usually the last argument!)
+            const fnIndex = args.findIndex((arg) => typeof arg === "function");
+            if (fnIndex === -1) {
+              throw new Error(
+                "startActiveSpan requires a function as the last argument",
+              );
+            }
+
+            // A type assertion is safe here due to the check above, but still sad 😥
+            const userFn = args[fnIndex] as (
+              span: Span,
+              ...rest: unknown[]
+            ) => unknown;
+
+            // Replace the function with one that wraps the span first
+            const spanWrapFunc = (...fnArgs: unknown[]) => {
+              const [span, ...rest] = fnArgs;
+              return userFn(createLangWatchSpan(span as Span), ...rest);
+            };
+
+            const newArgs = [...args];
+            newArgs[fnIndex] = spanWrapFunc;
+
+            // TypeScript can't infer the overload, but this is safe
+            return (
+              target.startActiveSpan as unknown as (
+                ...args: unknown[]
+              ) => unknown
+            )(...newArgs);
+          };
+          return startActiveSpan;
+        }
+
+        case "startSpan": {
+          return function (
+            ...args: Parameters<Tracer["startSpan"]>
+          ): ReturnType<Tracer["startSpan"]> {
+            const span = target.startSpan(...args);
+            return createLangWatchSpan(span);
+          };
+        }
+
+        case "withActiveSpan": {
+          /**
+           * Implementation of withActiveSpan: supports all overloads like startActiveSpan.
+           * Uses startActiveSpan to ensure context propagation for nested spans.
+           */
+          return async function withActiveSpan(...args: any[]): Promise<any> {
+            // Find the function argument (should be the last argument)
+            const fnIndex = args.findIndex((arg) => typeof arg === "function");
+            if (fnIndex === -1) {
+              throw new Error("withActiveSpan requires a function as the last argument");
+            }
+            const userFn = args[fnIndex] as (span: LangWatchSpan) => Promise<any> | any;
+            // The preceding arguments are: name, options?, context?
+            const name = args[0];
+            const options = args.length > 2 ? args[1] : undefined;
+            const context = args.length > 3 ? args[2] : undefined;
+
+            return await new Promise((resolve, reject) => {
+              // Use startActiveSpan to ensure context propagation
+              const cb = async (span: Span) => {
+                const wrappedSpan = createLangWatchSpan(span);
+                try {
+                  resolve(await userFn(wrappedSpan));
+                } catch (err: any) {
+                  wrappedSpan.setStatus({
+                    code: SpanStatusCode.ERROR,
+                    message: err && err.message ? err.message : String(err),
+                  });
+                  wrappedSpan.recordException(err);
+                  reject(err);
+                } finally {
+                  wrappedSpan.end();
+                }
+              };
+              // Call the correct overload of startActiveSpan
+              if (context !== undefined) {
+                target.startActiveSpan(name, options, context, cb);
+              } else if (options !== undefined) {
+                target.startActiveSpan(name, options, cb);
+              } else {
+                target.startActiveSpan(name, cb);
+              }
+            });
+          };
+        }
+
+        default: {
+          const value = target[prop as keyof Tracer];
+          return typeof value === "function" ? value.bind(target) : value;
+        }
+      }
+    },
+  };
+
+  return new Proxy(tracer, handler) as LangWatchTracer;
+}
+
+/**
+ * Helper type for the function overloads of startActiveSpan.
+ *
+ * This matches OpenTelemetry's Tracer interface and is used internally for type safety.
+ */
+type StartActiveSpanOverloads = {
+  <F extends (span: Span) => unknown>(name: string, fn: F): ReturnType<F>;
+  <F extends (span: Span) => unknown>(
+    name: string,
+    options: SpanOptions,
+    fn: F,
+  ): ReturnType<F>;
+  <F extends (span: Span) => unknown>(
+    name: string,
+    options: SpanOptions,
+    context: Context,
+    fn: F,
+  ): ReturnType<F>;
+};

package/src/prompt/__tests__/prompt.test.ts

@@ -0,0 +1,139 @@
+import { describe, it, expect, beforeAll, afterEach, vi } from "vitest";
+import { PromptService } from "../service";
+import { Prompt, PromptCompilationError } from "../prompt";
+import type { LangwatchApiClient } from "../../internal/api/client";
+
+// Mock the client with proper Vitest mock methods
+const mockClient = {
+  GET: vi.fn(),
+  POST: vi.fn(),
+  PUT: vi.fn(),
+  DELETE: vi.fn(),
+} as unknown as LangwatchApiClient & {
+  GET: ReturnType<typeof vi.fn>;
+  POST: ReturnType<typeof vi.fn>;
+  PUT: ReturnType<typeof vi.fn>;
+  DELETE: ReturnType<typeof vi.fn>;
+};
+
+// Mock the createLangWatchApiClient function
+vi.mock("../../internal/api/client", () => ({
+  createLangWatchApiClient: vi.fn(() => mockClient),
+}));
+
+describe("Prompt", () => {
+  let promptService: PromptService;
+
+  beforeAll(async () => {
+    promptService = new PromptService({ client: mockClient });
+  });
+
+  afterEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it("should fetch and compile a prompt", async () => {
+    // Mock the API response
+    const mockPromptData = {
+      id: "prompt_123",
+      name: "Test Prompt",
+      prompt: "Hello {{user_name}}, how is the {{topic}} today?",
+      messages: [
+        {
+          role: "user",
+          content: "Tell me about {{topic}}",
+        },
+      ],
+      model: "gpt-4",
+      version: 1,
+      updatedAt: "2024-01-01T00:00:00Z",
+    };
+
+    mockClient.GET.mockResolvedValueOnce({
+      data: mockPromptData,
+      error: null,
+    });
+
+    const prompt = await promptService.get("prompt_123");
+
+    expect(prompt.id).toBe("prompt_123");
+    expect(prompt.name).toBe("Test Prompt");
+
+    // Test template compilation
+    const compiled = prompt.compile({
+      user_name: "Alice",
+      topic: "weather",
+    });
+
+    expect(compiled.prompt).toContain("Alice");
+    expect(JSON.stringify(compiled.messages)).toContain("weather");
+  });
+
+  it("should handle missing template variables gracefully", async () => {
+    // Mock the API response
+    const mockPromptData = {
+      id: "prompt_123",
+      name: "Test Prompt",
+      prompt: "Hello {{user_name}}, how is the {{topic}} today?",
+      messages: [
+        {
+          role: "user",
+          content: "Tell me about {{topic}}",
+        },
+      ],
+      model: "gpt-4",
+      version: 1,
+      updatedAt: "2024-01-01T00:00:00Z",
+    };
+
+    mockClient.GET.mockResolvedValueOnce({
+      data: mockPromptData,
+      error: null,
+    });
+
+    const prompt = await promptService.get("prompt_123");
+
+    // Lenient compilation should not throw and should replace missing variables with empty strings
+    const compiled = prompt.compile({ user_name: "Alice", topic: "weather" });
+    expect(compiled).toBeInstanceOf(Prompt);
+    expect(compiled.prompt).toBe("Hello Alice, how is the weather today?");
+    expect(compiled.messages[0]?.content).toBe("Tell me about weather");
+  });
+
+  it("should throw on strict compilation with missing variables", async () => {
+    // Mock the API response
+    const mockPromptData = {
+      id: "prompt_123",
+      name: "Test Prompt",
+      prompt: "Hello {{user_name}}, how is the {{topic}} today?",
+      messages: [
+        {
+          role: "user",
+          content: "Tell me about {{topic}}",
+        },
+      ],
+      model: "gpt-4",
+      version: 1,
+      updatedAt: "2024-01-01T00:00:00Z",
+    };
+
+    mockClient.GET.mockResolvedValueOnce({
+      data: mockPromptData,
+      error: null,
+    });
+
+    const prompt = await promptService.get("prompt_123");
+
+    expect(() => {
+      prompt.compileStrict({ });
+    }).toThrow(PromptCompilationError);
+  });
+
+  it.todo("should create a prompt");
+  it.todo("should update a prompt");
+  it.todo("should delete a prompt");
+  it.todo("should create a prompt version");
+  it.todo("should get a prompt version");
+  it.todo("should list prompt versions");
+  it.todo("should delete a prompt version");
+});

package/src/prompt/get-prompt-version.ts

@@ -0,0 +1,49 @@
+import { PromptService } from "./service";
+import { CompiledPrompt, Prompt, TemplateVariables } from "./prompt";
+import * as intSemconv from "../observability/semconv";
+import { tracer } from "./tracer";
+import { canAutomaticallyCaptureInput, canAutomaticallyCaptureOutput } from "../client";
+
+export async function getPromptVersion(id: string, versionId: string, variables: TemplateVariables): Promise<CompiledPrompt>;
+export async function getPromptVersion(id: string, versionId: string): Promise<Prompt>;
+
+export async function getPromptVersion(id: string, versionId: string, variables?: TemplateVariables): Promise<Prompt | CompiledPrompt> {
+  return tracer.withActiveSpan("retrieve prompt version", async (span) => {
+    span.setType("prompt");
+    span.setAttribute(intSemconv.ATTR_LANGWATCH_PROMPT_ID, id);
+
+    const service = PromptService.getInstance();
+    const prompt = await service.getVersions(id);
+    const promptVersion = prompt[versionId];
+
+    if (!promptVersion) {
+      throw new Error(`Prompt version ${versionId} not found for prompt ${id}`);
+    }
+
+    if (canAutomaticallyCaptureOutput()) {
+      span.setOutput(prompt);
+    }
+
+    span.setAttributes({
+      [intSemconv.ATTR_LANGWATCH_PROMPT_ID]: id,
+      [intSemconv.ATTR_LANGWATCH_PROMPT_VERSION_ID]: promptVersion.id,
+      [intSemconv.ATTR_LANGWATCH_PROMPT_VERSION_NUMBER]: promptVersion.version,
+    });
+
+    if (variables) {
+      if (canAutomaticallyCaptureInput()) {
+        span.setAttribute(
+          intSemconv.ATTR_LANGWATCH_PROMPT_VARIABLES,
+          JSON.stringify({
+            type: "json",
+            value: variables,
+          }),
+        );
+      }
+
+      return promptVersion.compile(variables);
+    }
+
+    return promptVersion;
+  });
+}

package/src/prompt/get-prompt.ts

@@ -0,0 +1,44 @@
+import { PromptService } from "./service";
+import { CompiledPrompt, Prompt, TemplateVariables } from "./prompt";
+import * as intSemconv from "../observability/semconv";
+import { tracer } from "./tracer";
+import { canAutomaticallyCaptureInput, canAutomaticallyCaptureOutput } from "../client";
+
+export async function getPrompt(id: string, variables: TemplateVariables): Promise<CompiledPrompt>;
+export async function getPrompt(id: string): Promise<Prompt>;
+
+export async function getPrompt(id: string, variables?: TemplateVariables): Promise<Prompt | CompiledPrompt> {
+  return tracer.withActiveSpan("retrieve prompt", async (span) => {
+    span.setType("prompt");
+    span.setAttribute(intSemconv.ATTR_LANGWATCH_PROMPT_ID, id);
+
+    const service = PromptService.getInstance();
+    const prompt = await service.get(id);
+
+    if (canAutomaticallyCaptureOutput()) {
+      span.setOutput(prompt);
+    }
+
+    span.setAttributes({
+      [intSemconv.ATTR_LANGWATCH_PROMPT_ID]: id,
+      [intSemconv.ATTR_LANGWATCH_PROMPT_VERSION_ID]: prompt.versionId,
+      [intSemconv.ATTR_LANGWATCH_PROMPT_VERSION_NUMBER]: prompt.version,
+    });
+
+    if (variables) {
+      if (canAutomaticallyCaptureInput()) {
+        span.setAttribute(
+          intSemconv.ATTR_LANGWATCH_PROMPT_VARIABLES,
+          JSON.stringify({
+            type: "json",
+            value: variables,
+          }),
+        );
+      }
+
+      return prompt.compile(variables);
+    }
+
+    return prompt;
+  });
+}

package/src/prompt/index.ts

@@ -0,0 +1,3 @@
+export { getPrompt } from "./get-prompt";
+export { getPromptVersion } from "./get-prompt-version";
+export { CompiledPrompt, Prompt, PromptCompilationError, type TemplateVariables } from "./prompt";

package/src/prompt/prompt.ts

@@ -0,0 +1,133 @@
+import { Liquid } from "liquidjs";
+import type { paths } from "../internal/generated/openapi/api-client";
+
+// Extract the prompt response type from OpenAPI schema
+export type PromptResponse = NonNullable<
+  paths["/api/prompts/{id}"]["get"]["responses"]["200"]["content"]["application/json"]
+>;
+
+// Type for template variables - supporting common data types
+export type TemplateVariables = Record<
+  string,
+  string | number | boolean | object | null
+>;
+
+/**
+ * Error class for template compilation issues
+ */
+export class PromptCompilationError extends Error {
+  constructor(
+    message: string,
+    public readonly template: string,
+    public readonly originalError?: any
+  ) {
+    super(message);
+    this.name = "PromptCompilationError";
+  }
+}
+
+// Global Liquid instance - shared across all prompts for efficiency
+const liquid = new Liquid({
+  strictFilters: true,
+});
+
+export class Prompt implements PromptResponse {
+  public readonly id!: string;
+  public readonly name!: string;
+  public readonly updatedAt!: string;
+  public readonly version!: number;
+  public readonly versionId!: string;
+  public readonly versionCreatedAt!: string;
+  public readonly model!: string;
+  public readonly prompt!: string;
+  public readonly messages!: PromptResponse["messages"];
+  public readonly response_format!: PromptResponse["response_format"];
+
+  constructor(promptData: PromptResponse) {
+    this.id = promptData.id;
+    this.name = promptData.name;
+    this.updatedAt = promptData.updatedAt;
+    this.version = promptData.version;
+    this.versionId = promptData.versionId;
+    this.versionCreatedAt = promptData.versionCreatedAt;
+    this.model = promptData.model;
+    this.prompt = promptData.prompt;
+    this.messages = promptData.messages;
+    this.response_format = promptData.response_format;
+  }
+
+  /**
+   * Get the raw prompt data from the API
+   */
+  get raw(): PromptResponse {
+    return this;
+  }
+
+  /**
+   * Compile the prompt template with provided variables (lenient - missing variables become empty)
+   * @param variables - Object containing variable values for template compilation
+   * @returns CompiledPrompt instance with compiled content
+   */
+  private _compile(variables: TemplateVariables, strict: boolean): CompiledPrompt {
+    try {
+      // Compile main prompt
+      const compiledPrompt = this.prompt
+        ? liquid.parseAndRenderSync(this.prompt, variables, {
+          strictVariables: strict,
+        })
+        : "";
+
+      // Compile messages
+      const compiledMessages = (this.messages || []).map((message) => ({
+        ...message,
+        content: message.content
+          ? liquid.parseAndRenderSync(message.content, variables, {
+            strictVariables: strict,
+          })
+          : message.content,
+      }));
+
+      // Create new prompt data with compiled content
+      const compiledData: PromptResponse = {
+        ...this,
+        prompt: compiledPrompt,
+        messages: compiledMessages,
+      };
+
+      return new CompiledPrompt(compiledData, this);
+    } catch (error) {
+      const templateStr = this.prompt || JSON.stringify(this.messages);
+      throw new PromptCompilationError(
+        `Failed to compile prompt template: ${error instanceof Error ? error.message : "Unknown error"
+        }`,
+        templateStr,
+        error
+      );
+    }
+  }
+
+  compile(variables: TemplateVariables = {}): CompiledPrompt {
+    return this._compile(variables, false);
+  }
+
+  /**
+   * Compile with validation - throws error if required variables are missing
+   * @param variables - Template variables
+   * @returns CompiledPrompt instance with compiled content
+   */
+  compileStrict(variables: TemplateVariables): CompiledPrompt {
+    return this._compile(variables, true);
+  }
+}
+
+/**
+ * Represents a compiled prompt that extends Prompt with reference to the original template
+ */
+export class CompiledPrompt extends Prompt {
+  constructor(
+    compiledData: PromptResponse,
+    public readonly original: Prompt
+  ) {
+    super(compiledData);
+  }
+}