langwatch 0.0.2 → 0.1.0

package/src/index.test.ts

@@ -1,4 +1,4 @@
-import { OpenAI } from "openai";
+import { AzureOpenAI, OpenAI } from "openai";
 import { LangWatch, convertFromVercelAIMessages } from "./index";
 import {
   describe,
@@ -10,6 +10,7 @@ import {
 } from "vitest";
 import { openai } from "@ai-sdk/openai";
 import { generateText, type CoreMessage } from "ai";
+import "dotenv/config";
 
 describe("LangWatch tracer", () => {
   let mockFetch: SpyInstanceFn;
@@ -52,14 +53,12 @@ describe("LangWatch tracer", () => {
       },
     });
     span.end({
-      outputs: [
-        {
-          type: "json",
-          value: {
-            weather: "sunny",
-          },
+      output: {
+        type: "json",
+        value: {
+          weather: "sunny",
         },
-      ],
+      },
     });
 
     expect(trace.metadata).toEqual({
@@ -99,17 +98,15 @@ describe("LangWatch tracer", () => {
       },
     });
     llmSpan.end({
-      outputs: [
-        {
-          type: "chat_messages",
-          value: [
-            {
-              role: "assistant",
-              content: "It's cloudy in Tokyo.",
-            },
-          ],
-        },
-      ],
+      output: {
+        type: "chat_messages",
+        value: [
+          {
+            role: "assistant",
+            content: "It's cloudy in Tokyo.",
+          },
+        ],
+      },
     });
 
     ragSpan.end();
@@ -140,6 +137,47 @@ describe("LangWatch tracer", () => {
     expect(requestBody.spans.length).toBe(3);
   });
 
+  it("captures exceptions", async () => {
+    const langwatch = new LangWatch({
+      apiKey: "test",
+      endpoint: "http://localhost.test",
+    });
+    const trace = langwatch.getTrace();
+    trace.update({
+      metadata: { threadId: "123", userId: "123", labels: ["foo"] },
+    });
+    trace.update({ metadata: { userId: "456", labels: ["bar"] } });
+
+    const span = trace.startSpan({
+      name: "weather_function",
+      input: {
+        type: "json",
+        value: {
+          city: "Tokyo",
+        },
+      },
+    });
+
+    try {
+      throw new Error("unexpected error");
+    } catch (error) {
+      span.end({
+        error: error,
+      });
+    }
+
+    await trace.sendSpans();
+
+    expect(mockFetch).toHaveBeenCalled();
+    const firstCall: any = mockFetch.mock.calls[0];
+    const requestBody = JSON.parse(firstCall[1].body);
+    expect(requestBody.spans[0].error).toEqual({
+      has_error: true,
+      message: "unexpected error",
+      stacktrace: expect.any(Array),
+    });
+  });
+
   it.skip("captures openai llm call", async () => {
     const langwatch = new LangWatch({
       apiKey: "test",
@@ -175,10 +213,79 @@ describe("LangWatch tracer", () => {
     });
 
     span.end({
-      outputs: chatCompletion.choices.map((choice) => ({
+      output: {
         type: "chat_messages",
-        value: [choice.message],
-      })),
+        value: [chatCompletion.choices[0]!.message],
+      },
+      metrics: {
+        promptTokens: chatCompletion.usage?.prompt_tokens,
+        completionTokens: chatCompletion.usage?.completion_tokens,
+      },
+    });
+
+    await trace.sendSpans();
+
+    expect(mockFetch).toHaveBeenCalled();
+    expect(mockFetch).toHaveBeenCalledWith(
+      "http://localhost.test/api/collector",
+      expect.objectContaining({
+        method: "POST",
+        headers: {
+          "X-Auth-Token": "test",
+          "Content-Type": "application/json",
+        },
+        body: expect.any(String),
+      })
+    );
+  });
+
+  it.skip("captures azure openai llm call", async () => {
+    const langwatch = new LangWatch({
+      apiKey: "test",
+      endpoint: "http://localhost.test",
+    });
+    const trace = langwatch.getTrace();
+
+    // Model to be used and messages that will be sent to the LLM
+    const model = "gpt-4-turbo-2024-04-09";
+    const messages: OpenAI.Chat.ChatCompletionMessageParam[] = [
+      { role: "system", content: "You are a helpful assistant." },
+      {
+        role: "user",
+        content: "Write a tweet-size vegetarian lasagna recipe for 4 people.",
+      },
+    ];
+
+    // Capture the llm call with a span
+    const span = trace.startLLMSpan({
+      name: "llm",
+      model: model,
+      input: {
+        type: "chat_messages",
+        value: messages,
+      },
+    });
+
+    // Continue with the LLM call normally
+    const openai = new AzureOpenAI({
+      apiKey: process.env.AZURE_OPENAI_API_KEY,
+      apiVersion: "2024-02-01",
+      endpoint: process.env.AZURE_OPENAI_ENDPOINT,
+    });
+    const chatCompletion = await openai.chat.completions.create({
+      messages: messages,
+      model: model,
+    });
+
+    span.end({
+      output: {
+        type: "chat_messages",
+        value: [chatCompletion.choices[0]!.message],
+      },
+      metrics: {
+        promptTokens: chatCompletion.usage?.prompt_tokens,
+        completionTokens: chatCompletion.usage?.completion_tokens,
+      },
     });
 
     await trace.sendSpans();
@@ -229,12 +336,14 @@ describe("LangWatch tracer", () => {
     });
 
     span.end({
-      outputs: [
-        {
-          type: "chat_messages",
-          value: convertFromVercelAIMessages(response.responseMessages),
-        },
-      ],
+      output: {
+        type: "chat_messages",
+        value: convertFromVercelAIMessages(response.responseMessages),
+      },
+      metrics: {
+        promptTokens: response.usage?.promptTokens,
+        completionTokens: response.usage?.completionTokens,
+      },
     });
 
     await trace.sendSpans();

package/src/index.ts

@@ -24,7 +24,7 @@ import {
   type RAGSpan,
   type SpanInputOutput,
 } from "./types";
-import { convertFromVercelAIMessages } from "./utils";
+import { captureError, convertFromVercelAIMessages } from "./utils";
 
 export type {
   BaseSpan,
@@ -39,7 +39,7 @@ export type {
   SpanInputOutput,
 };
 
-export { convertFromVercelAIMessages };
+export { convertFromVercelAIMessages, captureError };
 
 export class LangWatch extends EventEmitter {
   apiKey: string | undefined;
@@ -55,9 +55,10 @@ export class LangWatch extends EventEmitter {
     super();
     const apiKey_ = apiKey ?? process.env.LANGWATCH_API_KEY;
     if (!apiKey_) {
-      console.warn(
-        "[LangWatch] ⚠️ LangWatch API key is not set, please set the LANGWATCH_API_KEY environment variable or pass it in the constructor. Traces will not be captured."
+      const error = new Error(
+        "LangWatch API key is not set, please set the LANGWATCH_API_KEY environment variable or pass it in the constructor. Traces will not be captured."
       );
+      this.emit("error", error);
     }
     this.apiKey = apiKey_;
     this.endpoint = endpoint;
@@ -99,10 +100,9 @@ export class LangWatch extends EventEmitter {
 
     if (!this.apiKey) {
       const error = new Error(
-        "[LangWatch] ⚠️ LangWatch API key is not set, LLMs traces will not be sent, go to https://langwatch.ai to set it up"
+        "LangWatch API key is not set, LLMs traces will not be sent, go to https://langwatch.ai to set it up"
       );
       this.emit("error", error);
-      console.warn(error.message);
       return;
     }
 
@@ -117,15 +117,14 @@ export class LangWatch extends EventEmitter {
 
     if (response.status === 429) {
       const error = new Error(
-        "[LangWatch] ⚠️ Rate limit exceeded, dropping message from being sent to LangWatch. Please check your dashboard to upgrade your plan."
+        "Rate limit exceeded, dropping message from being sent to LangWatch. Please check your dashboard to upgrade your plan."
       );
       this.emit("error", error);
-      console.warn(error.message);
       return;
     }
     if (!response.ok) {
       const error = new Error(
-        `[LangWatch] ⚠️ Failed to send trace, status: ${response.status}`
+        `Failed to send trace, status: ${response.status}`
       );
       this.emit("error", error);
       throw error;
@@ -214,8 +213,6 @@ export class LangWatchTrace {
       if (error instanceof ZodError) {
         console.warn("[LangWatch] ⚠️ Failed to parse trace");
         console.warn(fromZodError(error).message);
-      } else {
-        console.warn(error);
       }
       this.client.emit("error", error);
     }
@@ -233,8 +230,8 @@ export class LangWatchSpan implements PendingBaseSpan {
   parentId?: string | null;
   type: SpanTypes;
   name?: string | null;
-  input: PendingBaseSpan["input"];
-  outputs: PendingBaseSpan["outputs"];
+  input?: PendingBaseSpan["input"];
+  output?: PendingBaseSpan["output"];
   error?: PendingBaseSpan["error"];
   timestamps: PendingBaseSpan["timestamps"];
   metrics: PendingBaseSpan["metrics"];
@@ -246,7 +243,7 @@ export class LangWatchSpan implements PendingBaseSpan {
     type,
     name,
     input,
-    outputs,
+    output,
     error,
     timestamps,
     metrics,
@@ -257,7 +254,7 @@ export class LangWatchSpan implements PendingBaseSpan {
     this.type = type ?? "span";
     this.name = name;
     this.input = input;
-    this.outputs = outputs ?? [];
+    this.output = output;
     this.error = error;
     this.timestamps = timestamps ?? {
       startedAt: Date.now(),
@@ -266,6 +263,14 @@ export class LangWatchSpan implements PendingBaseSpan {
   }
 
   update(params: Partial<Omit<PendingBaseSpan, "spanId" | "parentId">>) {
+    if (Object.isFrozen(this)) {
+      const error = new Error(
+        `Tried to update span ${this.spanId}, but the span is already finished, discarding update`
+      );
+      this.trace.client.emit("error", error);
+      return;
+    }
+
     if (params.type) {
       this.type = params.type;
     }
@@ -275,8 +280,8 @@ export class LangWatchSpan implements PendingBaseSpan {
     if ("input" in params) {
       this.input = params.input;
     }
-    if (params.outputs) {
-      this.outputs = params.outputs;
+    if ("output" in params) {
+      this.output = params.output;
     }
     if ("error" in params) {
       this.error = params.error;
@@ -322,6 +327,8 @@ export class LangWatchSpan implements PendingBaseSpan {
       this.update(params);
     }
 
+    Object.freeze(this);
+
     try {
       const finalSpan = spanSchema.parse(
         camelToSnakeCaseNested({
@@ -332,6 +339,7 @@ export class LangWatchSpan implements PendingBaseSpan {
             ...this.timestamps,
             finishedAt: this.timestamps.finishedAt,
           },
+          ...(this.error && { error: captureError(this.error) }),
         }) as ServerSpan
       );
       this.trace.onEnd(finalSpan);
@@ -339,8 +347,6 @@ export class LangWatchSpan implements PendingBaseSpan {
       if (error instanceof ZodError) {
         console.warn("[LangWatch] ⚠️ Failed to parse span");
         console.warn(fromZodError(error).message);
-      } else {
-        console.warn(error);
       }
       this.trace.client.emit("error", error);
     }

package/src/types.ts

@@ -31,7 +31,7 @@ export type ChatRichContent = ServerChatRichContent;
 ({}) as {
   type: "chat_messages";
   value: OpenAI.Chat.ChatCompletionMessageParam[];
-}[] satisfies BaseSpan["outputs"];
+} satisfies BaseSpan["output"];
 
 // Keep the input/output types signatures as snake case to match the official openai nodejs api
 export type SpanInputOutput =
@@ -41,9 +41,10 @@ export type SpanInputOutput =
   | (TypedValueChatMessages & { type: ChatMessage });
 
 export type ConvertServerSpan<T extends ServerBaseSpan> =
-  SnakeToCamelCaseNested<Omit<T, "input" | "outputs">> & {
+  SnakeToCamelCaseNested<Omit<T, "input" | "output" | "error">> & {
     input?: SpanInputOutput | null;
-    outputs: SpanInputOutput[];
+    output?: SpanInputOutput | null;
+    error?: T["error"] | NonNullable<unknown>;
   };
 
 export type PendingSpan<T extends BaseSpan> = Omit<

package/src/utils.ts

@@ -1,6 +1,7 @@
 import { convertUint8ArrayToBase64 } from "@ai-sdk/provider-utils";
 import { type ImagePart, type CoreMessage } from "ai";
 import { type ChatMessage } from "./types";
+import { type ErrorCapture } from "./server/types/tracer";
 
 const convertImageToUrl = (
   image: ImagePart["image"],
@@ -132,3 +133,47 @@ export function convertFromVercelAIMessages(
 
   return lwMessages;
 }
+
+export const captureError = (error: unknown): ErrorCapture => {
+  if (
+    error &&
+    typeof error === "object" &&
+    "has_error" in error &&
+    "message" in error &&
+    "stacktrace" in error
+  ) {
+    return error as ErrorCapture;
+  } else if (error instanceof Error) {
+    return {
+      has_error: true,
+      message: error.message,
+      stacktrace: error.stack ? error.stack.split("\n") : [],
+    };
+  } else if (typeof error === "object" && error !== null) {
+    const err = error as { message: unknown; stack: unknown };
+    const message =
+      typeof err.message === "string"
+        ? err.message
+        : "An unknown error occurred";
+    const stacktrace =
+      typeof err.stack === "string"
+        ? err.stack.split("\n")
+        : Array.isArray(err.stack) &&
+          err.stack.length > 0 &&
+          typeof err.stack[0] === "string"
+        ? err.stack
+        : ["No stack trace available"];
+    return {
+      has_error: true,
+      message,
+      stacktrace,
+    };
+  } else {
+    // Handle primitives and other types that are not an error object
+    return {
+      has_error: true,
+      message: String(error),
+      stacktrace: [],
+    };
+  }
+};