@langwatch/mcp-server 2.0.0 → 2.2.0

package/src/__tests__/get-experiment-results.unit.test.ts

@@ -1,313 +0,0 @@
-import { describe, it, expect, vi, beforeEach } from "vitest";
-
-vi.mock("../langwatch-api.js", async (importOriginal) => {
-  const actual = (await importOriginal()) as Record<string, unknown>;
-  return {
-    ...actual,
-    makeRequest: vi.fn(),
-  };
-});
-
-import { LangWatchApiError, makeRequest } from "../langwatch-api.js";
-import { handleExperimentResults } from "../tools/get-experiment-results.js";
-
-const mockMakeRequest = vi.mocked(makeRequest);
-
-const sample = {
-  experimentId: "exp_1",
-  runId: "run_1",
-  projectId: "proj_1",
-  progress: 3,
-  total: 3,
-  dataset: [
-    { index: 0, entry: { input: "hello world" } },
-    { index: 1, entry: { input: "broken" }, error: "boom" },
-    { index: 2, entry: { input: "low" } },
-  ],
-  evaluations: [
-    { evaluator: "quality", index: 0, status: "processed", score: 0.9, passed: true },
-    { evaluator: "quality", index: 2, status: "processed", score: 0.2, passed: false, details: "off-topic" },
-    { evaluator: "safety", index: 0, status: "processed", score: 1.0, passed: true },
-  ],
-  timestamps: { createdAt: 0, updatedAt: 0, finishedAt: 1 },
-};
-
-beforeEach(() => {
-  vi.clearAllMocks();
-});
-
-describe("handleExperimentResults()", () => {
-  describe("given a completed run", () => {
-    describe("when called without filters", () => {
-      it("hits the v3 results endpoint with the run id", async () => {
-        mockMakeRequest.mockResolvedValueOnce(sample);
-        await handleExperimentResults({ runId: "run_1" });
-        expect(mockMakeRequest).toHaveBeenCalledWith(
-          "GET",
-          "/api/experiments/runs/run_1/results",
-        );
-      });
-
-      it("renders evaluator averages and per-row sections in markdown", async () => {
-        mockMakeRequest.mockResolvedValueOnce(sample);
-        const out = await handleExperimentResults({ runId: "run_1" });
-        expect(out).toContain("# Evaluation Results: run_1");
-        expect(out).toContain("## Evaluator Summary");
-        expect(out).toContain("quality");
-        expect(out).toContain("safety");
-        expect(out).toContain("Row #0");
-        expect(out).toContain("Row #1");
-        expect(out).toContain("Row #2");
-      });
-    });
-
-    describe("when filter is 'failed'", () => {
-      it("includes only failed/errored rows", async () => {
-        mockMakeRequest.mockResolvedValueOnce(sample);
-        const out = await handleExperimentResults({
-          runId: "run_1",
-          filter: "failed",
-        });
-        expect(out).toContain("Row #1");
-        expect(out).toContain("Row #2");
-        expect(out).not.toContain("Row #0");
-      });
-    });
-
-    describe("when an evaluator name is supplied", () => {
-      it("limits the report to that evaluator", async () => {
-        mockMakeRequest.mockResolvedValueOnce(sample);
-        const out = await handleExperimentResults({
-          runId: "run_1",
-          evaluator: "quality",
-        });
-        expect(out).toContain("quality");
-        // The per-row sections should not list the safety evaluator bullet
-        expect(out).not.toMatch(/\*\*safety\*\*/);
-      });
-    });
-
-    describe("when there are more rows than the cap", () => {
-      it("truncates and notes the cap", async () => {
-        const big = {
-          ...sample,
-          dataset: Array.from({ length: 80 }, (_, i) => ({
-            index: i,
-            entry: { input: `row ${i}` },
-          })),
-          evaluations: [],
-        };
-        mockMakeRequest.mockResolvedValueOnce(big);
-        const out = await handleExperimentResults({ runId: "run_1" });
-        expect(out).toContain("Output truncated to 50 rows of 80");
-      });
-    });
-
-    describe("when a caller asks for more than the row cap", () => {
-      it("still caps output at 50 rows", async () => {
-        const big = {
-          ...sample,
-          dataset: Array.from({ length: 80 }, (_, i) => ({
-            index: i,
-            entry: { input: `row ${i}` },
-          })),
-          evaluations: [],
-        };
-        mockMakeRequest.mockResolvedValueOnce(big);
-        const out = await handleExperimentResults({
-          runId: "run_1",
-          limit: 5000,
-        });
-        expect(out).toContain("Output truncated to 50 rows of 80");
-      });
-    });
-
-    describe("when only some rows have evaluations", () => {
-      it("explains rows with no evaluation coverage", async () => {
-        mockMakeRequest.mockResolvedValueOnce({
-          ...sample,
-          dataset: [
-            { index: 0, entry: { input: "covered" } },
-            { index: 1, entry: { input: "uncovered" } },
-          ],
-          evaluations: [
-            {
-              evaluator: "quality",
-              index: 0,
-              status: "processed",
-              score: 0.9,
-              passed: true,
-            },
-          ],
-        });
-
-        const out = await handleExperimentResults({ runId: "run_1" });
-
-        expect(out).toContain("uncovered");
-        expect(out).toContain("No evaluations recorded for this row");
-      });
-    });
-
-    describe("when multiple targets share the same dataset index", () => {
-      it("keeps evaluator rows target-scoped", async () => {
-        mockMakeRequest.mockResolvedValueOnce({
-          ...sample,
-          dataset: [
-            { index: 0, targetId: "a", entry: { input: "target a" } },
-            { index: 0, targetId: "b", entry: { input: "target b" } },
-          ],
-          evaluations: [
-            {
-              evaluator: "quality",
-              index: 0,
-              targetId: "a",
-              status: "processed",
-              score: 0.9,
-              passed: true,
-            },
-            {
-              evaluator: "quality",
-              index: 0,
-              targetId: "b",
-              status: "processed",
-              score: 0.1,
-              passed: false,
-              details: "target-b-only",
-            },
-          ],
-        });
-
-        const out = await handleExperimentResults({
-          runId: "run_1",
-          filter: "failed",
-        });
-
-        expect(out).toContain("target b");
-        expect(out).toContain("target-b-only");
-        expect(out).not.toContain("target a");
-      });
-    });
-  });
-
-  describe("given the run is missing", () => {
-    describe("when the API throws a 404-shaped error", () => {
-      it("returns a graceful 'not found' markdown", async () => {
-        mockMakeRequest.mockRejectedValueOnce(
-          new LangWatchApiError("missing", 404, "Run not found"),
-        );
-        const out = await handleExperimentResults({ runId: "missing" });
-        expect(out).toContain("not found");
-        expect(out).toContain("experimentSlug");
-      });
-    });
-  });
-
-  describe("given the run is still in progress", () => {
-    describe("when results are requested", () => {
-      it("serves the partial rows logged so far with an in-progress note", async () => {
-        mockMakeRequest.mockResolvedValueOnce({
-          ...sample,
-          progress: 2,
-          total: 3,
-          timestamps: {
-            createdAt: Date.now(),
-            updatedAt: Date.now(),
-            finishedAt: null,
-            stoppedAt: null,
-          },
-        });
-
-        const out = await handleExperimentResults({ runId: "unfinished" });
-
-        expect(out).toContain("**Status**: running");
-        expect(out).toContain("still in progress");
-        expect(out).toContain("Row #0");
-        expect(out).toContain("Progress**: 2/3");
-      });
-    });
-  });
-
-  describe("given the run was interrupted before finishing", () => {
-    describe("when results are requested", () => {
-      it("serves the partial rows with an interrupted note", async () => {
-        mockMakeRequest.mockResolvedValueOnce({
-          ...sample,
-          timestamps: {
-            createdAt: Date.now() - 60 * 60 * 1000,
-            updatedAt: Date.now() - 30 * 60 * 1000,
-            finishedAt: null,
-            stoppedAt: null,
-          },
-        });
-
-        const out = await handleExperimentResults({ runId: "interrupted" });
-
-        expect(out).toContain("**Status**: interrupted");
-        expect(out).toContain("interrupted");
-        expect(out).toContain("Row #0");
-      });
-    });
-  });
-
-  describe("given a non-terminal run with zero rows", () => {
-    describe("when the run is still running", () => {
-      it("says the run is in progress", async () => {
-        mockMakeRequest.mockResolvedValueOnce({
-          ...sample,
-          dataset: [],
-          evaluations: [],
-          timestamps: {
-            createdAt: Date.now(),
-            updatedAt: Date.now(),
-            finishedAt: null,
-            stoppedAt: null,
-          },
-        });
-        const out = await handleExperimentResults({ runId: "empty-running" });
-        expect(out).toContain("still in progress");
-        expect(out).not.toContain("interrupted");
-      });
-    });
-
-    describe("when the run was interrupted", () => {
-      it("says no rows were recorded before it was interrupted, not to wait", async () => {
-        mockMakeRequest.mockResolvedValueOnce({
-          ...sample,
-          dataset: [],
-          evaluations: [],
-          timestamps: {
-            createdAt: Date.now() - 60 * 60 * 1000,
-            updatedAt: Date.now() - 30 * 60 * 1000,
-            finishedAt: null,
-            stoppedAt: null,
-          },
-        });
-        const out = await handleExperimentResults({ runId: "empty-interrupted" });
-        expect(out).toContain("interrupted");
-        expect(out).not.toContain("still in progress");
-        expect(out).not.toContain("Call again shortly");
-      });
-    });
-  });
-
-  describe("given no run state but rows exist (SDK / >24h run)", () => {
-    describe("when results are requested with a stopped run", () => {
-      it("renders results for a stopped run", async () => {
-        mockMakeRequest.mockResolvedValueOnce({
-          ...sample,
-          timestamps: {
-            createdAt: 0,
-            updatedAt: 10,
-            finishedAt: null,
-            stoppedAt: 20,
-          },
-        });
-
-        const out = await handleExperimentResults({ runId: "stopped" });
-
-        expect(out).toContain("**Status**: stopped");
-        expect(out).toContain("Row #0");
-      });
-    });
-  });
-});

package/src/__tests__/http-security-origin.unit.test.ts

@@ -1,97 +0,0 @@
-import { describe, expect, it } from "vitest";
-
-import {
-  DEFAULT_BIND_HOST,
-  isLoopbackHost,
-  isOriginAllowed,
-  parseAllowedOrigins,
-} from "../http-security.js";
-
-describe("bind host", () => {
-  it("defaults to loopback", () => {
-    expect(DEFAULT_BIND_HOST).toBe("127.0.0.1");
-    expect(isLoopbackHost(DEFAULT_BIND_HOST)).toBe(true);
-  });
-
-  it.each(["127.0.0.1", "localhost", "::1", "[::1]", " LOCALHOST "])(
-    "recognises %s as loopback",
-    (host) => {
-      expect(isLoopbackHost(host)).toBe(true);
-    }
-  );
-
-  it.each(["0.0.0.0", "::", "192.168.1.10", "10.0.0.1", "example.com"])(
-    "does not treat %s as loopback",
-    (host) => {
-      expect(isLoopbackHost(host)).toBe(false);
-    }
-  );
-});
-
-describe("parseAllowedOrigins", () => {
-  it("returns an empty list when nothing is configured", () => {
-    expect(parseAllowedOrigins(undefined)).toEqual([]);
-    expect(parseAllowedOrigins("")).toEqual([]);
-  });
-
-  it("splits, trims, and normalizes entries", () => {
-    expect(
-      parseAllowedOrigins(" https://Example.com/ , https://app.test:8443 ")
-    ).toEqual(["https://example.com", "https://app.test:8443"]);
-  });
-
-  it("drops entries that are not usable origins", () => {
-    expect(parseAllowedOrigins("not-an-origin, https://ok.test")).toEqual([
-      "https://ok.test",
-    ]);
-  });
-
-  it("deduplicates equivalent origins", () => {
-    expect(parseAllowedOrigins("https://a.test,https://a.test/")).toEqual([
-      "https://a.test",
-    ]);
-  });
-});
-
-describe("isOriginAllowed", () => {
-  it.each([
-    "http://localhost:5173",
-    "http://127.0.0.1:3000",
-    "http://[::1]:3000",
-    "https://localhost",
-  ])("always allows the loopback origin %s", (origin) => {
-    expect(isOriginAllowed({ origin, allowedOrigins: [] })).toBe(true);
-  });
-
-  it("rejects an unlisted remote origin", () => {
-    expect(isOriginAllowed({ origin: "https://evil.test", allowedOrigins: [] })).toBe(false);
-  });
-
-  it("allows a configured remote origin", () => {
-    expect(isOriginAllowed({ origin: "https://app.test", allowedOrigins: ["https://app.test"] })).toBe(
-      true
-    );
-  });
-
-  it("matches on scheme, host and port together", () => {
-    const allowlist = ["https://app.test"];
-    expect(isOriginAllowed({ origin: "http://app.test", allowedOrigins: allowlist })).toBe(false);
-    expect(isOriginAllowed({ origin: "https://app.test:8443", allowedOrigins: allowlist })).toBe(false);
-    expect(isOriginAllowed({ origin: "https://other.test", allowedOrigins: allowlist })).toBe(false);
-  });
-
-  it("does not let a hostname that merely ends in localhost through", () => {
-    expect(isOriginAllowed({ origin: "https://notlocalhost", allowedOrigins: [] })).toBe(false);
-    expect(isOriginAllowed({ origin: "https://evil-localhost.test", allowedOrigins: [] })).toBe(false);
-  });
-
-  it("rejects the opaque null origin", () => {
-    expect(isOriginAllowed({ origin: "null", allowedOrigins: [] })).toBe(false);
-  });
-
-  it("treats a rebound attacker hostname as unlisted", () => {
-    // DNS rebinding points an attacker hostname at loopback, but the browser
-    // still sends the attacker hostname as the origin.
-    expect(isOriginAllowed({ origin: "http://rebind.attacker.test", allowedOrigins: [] })).toBe(false);
-  });
-});

package/src/__tests__/http-security-sessions.unit.test.ts

@@ -1,209 +0,0 @@
-import { describe, expect, it, vi } from "vitest";
-
-import { admitOAuthToken, createSessionStore } from "../http-security.js";
-
-describe("createSessionStore", () => {
-  function makeStore(maxAgeMs = 60_000) {
-    const closed: string[] = [];
-    const store = createSessionStore<{ id: string }>({
-      maxAgeMs,
-      closeTransport: (transport) => closed.push(transport.id),
-    });
-    return { store, closed };
-  }
-
-  it("stores and returns sessions bound to their key", () => {
-    const { store } = makeStore();
-    store.add({ sessionId: "s1", transport: { id: "s1" }, apiKey: "sk-a" });
-
-    expect(store.get("s1")?.apiKey).toBe("sk-a");
-    expect(store.size).toBe(1);
-  });
-
-  it("counts live sessions per key without holding the raw key", () => {
-    const { store } = makeStore();
-    store.add({ sessionId: "s1", transport: { id: "s1" }, apiKey: "sk-a" });
-    store.add({ sessionId: "s2", transport: { id: "s2" }, apiKey: "sk-a" });
-    store.add({ sessionId: "s3", transport: { id: "s3" }, apiKey: "sk-b" });
-
-    expect(store.countForKey("sk-a")).toBe(2);
-    expect(store.countForKey("sk-b")).toBe(1);
-    expect(store.countForKey("sk-unknown")).toBe(0);
-  });
-
-  it("decrements the per-key count when a session is removed", () => {
-    const { store } = makeStore();
-    store.add({ sessionId: "s1", transport: { id: "s1" }, apiKey: "sk-a" });
-    store.add({ sessionId: "s2", transport: { id: "s2" }, apiKey: "sk-a" });
-
-    store.remove("s1");
-    expect(store.countForKey("sk-a")).toBe(1);
-
-    store.remove("s2");
-    expect(store.countForKey("sk-a")).toBe(0);
-  });
-
-  it("ignores removal of a session it does not hold", () => {
-    const { store } = makeStore();
-    store.add({ sessionId: "s1", transport: { id: "s1" }, apiKey: "sk-a" });
-
-    store.remove("never-existed");
-    store.remove("never-existed");
-
-    expect(store.countForKey("sk-a")).toBe(1);
-  });
-
-  it("closes and forgets sessions that have gone idle", () => {
-    vi.useFakeTimers();
-    try {
-      const { store, closed } = makeStore(1_000);
-      store.add({ sessionId: "idle", transport: { id: "idle" }, apiKey: "sk-a" });
-      store.add({ sessionId: "busy", transport: { id: "busy" }, apiKey: "sk-a" });
-
-      vi.advanceTimersByTime(900);
-      store.touch("busy");
-      vi.advanceTimersByTime(900);
-      store.sweep();
-
-      expect(closed).toEqual(["idle"]);
-      expect(store.get("idle")).toBeUndefined();
-      expect(store.get("busy")).toBeDefined();
-      expect(store.countForKey("sk-a")).toBe(1);
-    } finally {
-      vi.useRealTimers();
-    }
-  });
-
-  it("closes everything on shutdown", () => {
-    const { store, closed } = makeStore();
-    store.add({ sessionId: "s1", transport: { id: "s1" }, apiKey: "sk-a" });
-    store.add({ sessionId: "s2", transport: { id: "s2" }, apiKey: "sk-b" });
-
-    store.closeAll();
-
-    expect(closed.sort()).toEqual(["s1", "s2"]);
-    expect(store.size).toBe(0);
-    expect(store.countForKey("sk-a")).toBe(0);
-  });
-
-  it("does not let a prototype-shaped session id reach the prototype chain", () => {
-    const { store } = makeStore();
-
-    expect(store.get("__proto__")).toBeUndefined();
-    expect(store.get("constructor")).toBeUndefined();
-
-    store.add({ sessionId: "__proto__", transport: { id: "weird" }, apiKey: "sk-a" });
-    expect(store.get("__proto__")?.transport.id).toBe("weird");
-    expect(store.size).toBe(1);
-  });
-});
-
-describe("session reservations", () => {
-  function makeStore() {
-    return createSessionStore<{ id: string }>({
-      maxAgeMs: 60_000,
-      closeTransport: () => undefined,
-    });
-  }
-
-  it("counts a reserved slot before any session exists", () => {
-    const store = makeStore();
-    store.reserve("sk-a");
-
-    // The point of the reservation: the cap sees the slot while the session is
-    // still being set up.
-    expect(store.countForKey("sk-a")).toBe(1);
-    expect(store.size).toBe(0);
-  });
-
-  it("hands the reserved slot to the session rather than counting twice", () => {
-    const store = makeStore();
-    store.reserve("sk-a").commit({ sessionId: "s1", transport: { id: "s1" } });
-
-    expect(store.countForKey("sk-a")).toBe(1);
-    expect(store.get("s1")?.apiKey).toBe("sk-a");
-  });
-
-  it("returns the slot when the session never came up", () => {
-    const store = makeStore();
-    const reservation = store.reserve("sk-a");
-    reservation.release();
-
-    expect(store.countForKey("sk-a")).toBe(0);
-  });
-
-  it("ignores a release once the slot was committed", () => {
-    const store = makeStore();
-    const reservation = store.reserve("sk-a");
-    reservation.commit({ sessionId: "s1", transport: { id: "s1" } });
-    reservation.release();
-
-    expect(store.countForKey("sk-a")).toBe(1);
-    expect(store.get("s1")).toBeDefined();
-  });
-
-  it("frees the slot when the committed session is removed", () => {
-    const store = makeStore();
-    store.reserve("sk-a").commit({ sessionId: "s1", transport: { id: "s1" } });
-    store.remove("s1");
-
-    expect(store.countForKey("sk-a")).toBe(0);
-  });
-});
-
-describe("admitOAuthToken", () => {
-  function entry(apiKey: string, expiresAt: number) {
-    return { apiKey, expiresAt };
-  }
-
-  it("evicts the oldest token for a key that is at the cap", () => {
-    const tokens = new Map([
-      ["t1", entry("sk-a", Date.now() + 1_000)],
-      ["t2", entry("sk-a", Date.now() + 2_000)],
-      ["t3", entry("sk-a", Date.now() + 3_000)],
-    ]);
-
-    admitOAuthToken({ apiKey: "sk-a", tokens, maxPerKey: 3 });
-
-    // Room for the caller's new token, and the longest-standing one went.
-    expect(tokens.has("t1")).toBe(false);
-    expect([...tokens.keys()]).toEqual(["t2", "t3"]);
-  });
-
-  it("leaves other keys alone", () => {
-    const tokens = new Map([
-      ["t1", entry("sk-a", Date.now() + 1_000)],
-      ["t2", entry("sk-b", Date.now() + 1_000)],
-      ["t3", entry("sk-b", Date.now() + 2_000)],
-    ]);
-
-    admitOAuthToken({ apiKey: "sk-a", tokens, maxPerKey: 1 });
-
-    expect(tokens.has("t1")).toBe(false);
-    expect(tokens.has("t2")).toBe(true);
-    expect(tokens.has("t3")).toBe(true);
-  });
-
-  it("keeps a key from growing the map without bound", () => {
-    const tokens = new Map<string, { apiKey: string; expiresAt: number }>();
-
-    for (let i = 0; i < 50; i += 1) {
-      admitOAuthToken({ apiKey: "sk-a", tokens, maxPerKey: 10 });
-      tokens.set(`t${i}`, entry("sk-a", Date.now() + 60_000 + i));
-    }
-
-    expect(tokens.size).toBe(10);
-  });
-
-  it("drops expired tokens for any key on the way past", () => {
-    const tokens = new Map([
-      ["stale", entry("sk-b", Date.now() - 1)],
-      ["live", entry("sk-b", Date.now() + 60_000)],
-    ]);
-
-    admitOAuthToken({ apiKey: "sk-a", tokens, maxPerKey: 10 });
-
-    expect(tokens.has("stale")).toBe(false);
-    expect(tokens.has("live")).toBe(true);
-  });
-});