@langwatch/mcp-server 0.0.5 → 0.2.0

package/tests/fixtures/opentelemetry/openinference_openai_bot_input.py

@@ -0,0 +1,41 @@
+# This example uses the OpenTelemetry instrumentation for OpenAI from OpenInference: https://pypi.org/project/openinference-instrumentation-openai/
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+import chainlit as cl
+
+from openinference.instrumentation.openai import OpenAIInstrumentor
+from openinference.instrumentation import using_attributes
+from openai import OpenAI
+
+client = OpenAI()
+
+# Manual instrumentation setup would go here
+OpenAIInstrumentor().instrument()
+
+
+@cl.on_message
+async def main(message: cl.Message):
+    msg = cl.Message(
+        content="",
+    )
+
+    completion = client.chat.completions.create(
+        model="gpt-5",
+        messages=[
+            {
+                "role": "system",
+                "content": "You are a helpful assistant that only reply in short tweet-like responses, using lots of emojis.",
+            },
+            {"role": "user", "content": message.content},
+        ],
+        stream=True,
+    )
+
+    for part in completion:
+        if token := part.choices[0].delta.content or "":
+            await msg.stream_token(token)
+
+    await msg.update()

package/tests/fixtures/opentelemetry/openllmetry_openai_bot_expected.py

@@ -0,0 +1,44 @@
+# This example uses the OpenTelemetry instrumentation for OpenAI from OpenLLMetry: https://pypi.org/project/opentelemetry-instrumentation-openai/
+
+from dotenv import load_dotenv
+
+import langwatch
+
+load_dotenv()
+
+import chainlit as cl
+
+from opentelemetry.instrumentation.openai import OpenAIInstrumentor
+from openai import OpenAI
+
+
+client = OpenAI()
+
+langwatch.setup(
+    instrumentors=[OpenAIInstrumentor()],
+)
+
+
+@cl.on_message
+async def main(message: cl.Message):
+    msg = cl.Message(
+        content="",
+    )
+
+    completion = client.chat.completions.create(
+        model="gpt-5",
+        messages=[
+            {
+                "role": "system",
+                "content": "You are a helpful assistant that only reply in short tweet-like responses, using lots of emojis.",
+            },
+            {"role": "user", "content": message.content},
+        ],
+        stream=True,
+    )
+
+    for part in completion:
+        if token := part.choices[0].delta.content or "":
+            await msg.stream_token(token)
+
+    await msg.update()

package/tests/fixtures/opentelemetry/openllmetry_openai_bot_input.py

@@ -0,0 +1,40 @@
+# This example uses the OpenTelemetry instrumentation for OpenAI from OpenLLMetry
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+import chainlit as cl
+from openai import OpenAI
+
+# Manual instrumentation setup would go here
+from openllmetry.instrumentation.openai import OpenAIInstrumentor
+
+OpenAIInstrumentor().instrument()
+
+client = OpenAI()
+
+
+@cl.on_message
+async def main(message: cl.Message):
+    msg = cl.Message(
+        content="",
+    )
+
+    completion = client.chat.completions.create(
+        model="gpt-5",
+        messages=[
+            {
+                "role": "system",
+                "content": "You are a helpful assistant that only reply in short tweet-like responses, using lots of emojis.",
+            },
+            {"role": "user", "content": message.content},
+        ],
+        stream=True,
+    )
+
+    for part in completion:
+        if token := part.choices[0].delta.content or "":
+            await msg.stream_token(token)
+
+    await msg.update()

package/tests/fixtures/strands/strands_bot_expected.py

@@ -0,0 +1,84 @@
+import os
+from strands import Agent, tool
+from strands.models.litellm import LiteLLMModel
+import langwatch
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+import chainlit.config as cl_config
+import chainlit as cl
+
+cl_config.config.project.enable_telemetry = False
+
+# OpenTelemetry Setup Options:
+
+
+# Option 1: Use only the LangWatch SDK. This is the simplest option.
+langwatch.setup()  # The api key is set from the environment variable automatically
+
+
+# Option 2: Use StrandsTelemetry to handle complete OpenTelemetry setup
+# (Creates new tracer provider and sets it as global)
+# from strands.telemetry import StrandsTelemetry
+# strands_telemetry = StrandsTelemetry()
+# strands_telemetry.setup_otlp_exporter(
+#     endpoint=f"{os.environ.get('LANGWATCH_ENDPOINT', 'https://app.langwatch.ai')}/api/otel/v1/traces",
+#     headers={"Authorization": "Bearer " + os.environ["LANGWATCH_API_KEY"]},
+# )
+# As OTel is managed by StrandsTelemetry, we must skip setting it up in LangWatch
+# langwatch.setup(skip_open_telemetry_setup=True)
+
+
+
+@tool
+@langwatch.span(type="tool")
+def get_user_location() -> str:
+    """Get the user's location."""
+
+    # Implement user location lookup logic here
+    return "London, UK"
+
+
+class KiteAgent:
+    def __init__(self):
+        self.model = LiteLLMModel(
+            client_args={
+                "api_key": os.getenv("OPENAI_API_KEY"),
+            },
+            model_id="openai/gpt-5-mini",
+        )
+        self.agent = Agent(
+            name="kite-agent",
+            model=self.model,
+            system_prompt="Always use the get_user_location tool before answering any questions.",
+            tools=[get_user_location],
+            trace_attributes={
+                "custom.model_id": "openai/gpt-5-mini",
+                "custom.example.attribute": "swift",
+            },
+        )
+
+    def run(self, prompt: str):
+        return self.agent(prompt)
+
+
+@cl.on_message
+@langwatch.trace()
+async def main(message: cl.Message):
+    msg = cl.Message(
+        content="",
+    )
+
+    langwatch.get_current_trace().update(
+        metadata={
+            "custom.example.attribute2": "langwatch",
+        }
+    )
+
+    agent = KiteAgent()
+    response = agent.run(message.content)
+
+    await msg.stream_token(str(response))
+    await msg.update()

package/tests/fixtures/strands/strands_bot_input.py

@@ -0,0 +1,52 @@
+import os
+from strands import Agent, tool
+from strands.models.litellm import LiteLLMModel
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+import chainlit.config as cl_config
+import chainlit as cl
+
+cl_config.config.project.enable_telemetry = False
+
+
+@tool
+def get_user_location() -> str:
+    """Get the user's location."""
+
+    # Implement user location lookup logic here
+    return "London, UK"
+
+
+class KiteAgent:
+    def __init__(self):
+        self.model = LiteLLMModel(
+            client_args={
+                "api_key": os.getenv("OPENAI_API_KEY"),
+            },
+            model_id="openai/gpt-5-mini",
+        )
+        self.agent = Agent(
+            name="kite-agent",
+            model=self.model,
+            system_prompt="Always use the get_user_location tool before answering any questions.",
+            tools=[get_user_location],
+        )
+
+    def run(self, prompt: str):
+        return self.agent(prompt)
+
+
+@cl.on_message
+async def main(message: cl.Message):
+    msg = cl.Message(
+        content="",
+    )
+
+    agent = KiteAgent()
+    response = agent.run(message.content)
+
+    await msg.stream_token(str(response))
+    await msg.update()

package/tests/scenario-openai.test.ts

@@ -0,0 +1,158 @@
+import scenario, {
+  type AgentAdapter,
+  AgentRole,
+  ScenarioExecutionStateLike,
+} from "@langwatch/scenario";
+import fs from "fs";
+import { execSync } from "child_process";
+import { describe, it, expect } from "vitest";
+import dotenv from "dotenv";
+import os from "os";
+import path from "path";
+import * as pty from "node-pty";
+import chalk from "chalk";
+import { anthropic } from "@ai-sdk/anthropic";
+
+dotenv.config();
+
+const claudeCodeAgent = (workingDirectory: string): AgentAdapter => ({
+  role: AgentRole.AGENT,
+  call: async (state) => {
+    const formattedMessages = state.messages
+      .map((message) => `${message.role}: ${message.content}`)
+      .join("\n\n");
+
+    const mcpConfig = {
+      mcpServers: {
+        LangWatch: {
+          command: "node",
+          args: [
+            `${__dirname}/../dist/index.js`,
+            "--apiKey",
+            process.env.LANGWATCH_API_KEY!,
+          ],
+        },
+      },
+    };
+    fs.writeFileSync(
+      `${__dirname}/.mcp-config.json`,
+      JSON.stringify(mcpConfig)
+    );
+
+    return new Promise<string>((resolve, reject) => {
+      const args = [
+        "--output-format",
+        "stream-json",
+        "-p",
+        "--mcp-config",
+        `${__dirname}/.mcp-config.json`,
+        "--dangerously-skip-permissions",
+        "--verbose",
+        formattedMessages,
+      ];
+
+      console.log(chalk.blue("Starting claude in:"), workingDirectory);
+
+      const ptyProcess = pty.spawn(
+        `${__dirname}/../node_modules/.bin/claude`,
+        args,
+        {
+          name: "xterm-256color",
+          cols: 80,
+          rows: 30,
+          cwd: workingDirectory,
+          env: { ...process.env, FORCE_COLOR: "1" },
+        }
+      );
+
+      let output = "";
+
+      ptyProcess.onData((data) => {
+        console.log(chalk.cyan("Claude Code:"), data);
+        output += data;
+      });
+
+      ptyProcess.onExit(({ exitCode }) => {
+        if (exitCode === 0) {
+          console.log("output", output);
+          const messages: any = output
+            .split("\n")
+            .map((line) => {
+              try {
+                return JSON.parse(line.trim());
+              } catch (error) {
+                return null;
+              }
+            })
+            .filter((message) => message !== null && "message" in message)
+            .map((message) => message.message);
+          console.log("messages", JSON.stringify(messages, undefined, 2));
+
+          resolve(messages);
+        } else {
+          reject(new Error(`Command failed with exit code ${exitCode}`));
+        }
+      });
+    });
+  },
+});
+
+describe("OpenAI Implementation", () => {
+  it("implements LangWatch in an OpenAI bot project", async () => {
+    const tempFolder = fs.mkdtempSync(
+      path.join(os.tmpdir(), "langwatch-openai-bot-")
+    );
+    execSync(
+      `cp -r tests/fixtures/openai/openai_bot_function_call_input.py ${tempFolder}/main.py`
+    );
+
+    const result = await scenario.run({
+      name: "OpenAI bot project",
+      description: `Implementing code changes in an OpenAI bot project to add LangWatch instrumentation.`,
+      agents: [
+        claudeCodeAgent(tempFolder),
+        scenario.userSimulatorAgent(),
+        scenario.judgeAgent({
+          model: anthropic("claude-sonnet-4-20250514"),
+          criteria: [
+            "Agent should edit main.py file",
+            "Agent should use the langwatch MCP for checking the documentation",
+          ],
+        }),
+      ],
+      script: [
+        scenario.user(
+          "please instrument my code with langwatch, short and sweet, no need to test the changes"
+        ),
+        scenario.agent(),
+        () => {
+          const resultFile = fs.readFileSync(`${tempFolder}/main.py`, "utf8");
+
+          expect(resultFile).toContain("@langwatch.trace(");
+          expect(resultFile).toContain("autotrack_openai_calls(client)");
+          // TODO: expect(resultFile).toContain('@langwatch.span(type="tool")');
+        },
+        toolCallFix,
+        scenario.judge(),
+      ],
+    });
+
+    expect(result.success).toBe(true);
+  });
+});
+
+function toolCallFix(state: ScenarioExecutionStateLike) {
+  // Fix for anthropic tool use format, that is not supported by vercel ai for the judge
+  state.messages.forEach((message) => {
+    if (Array.isArray(message.content)) {
+      message.content.forEach((content, index) => {
+        if (content.type !== "text") {
+          (message.content as any)[index] = {
+            type: "text",
+            text: JSON.stringify(content),
+          };
+        }
+      });
+    }
+  });
+}

package/tsconfig.json

@@ -17,7 +17,6 @@
     "jsx": "preserve",
     "incremental": true,
     "noUncheckedIndexedAccess": true,
-    "baseUrl": ".",
     "tsBuildInfoFile": "./tsconfig.tsbuildinfo"
   },
   "include": [