@langwatch/mcp-server 0.9.0 → 0.10.1

package/src/__tests__/experiment-status.unit.test.ts

@@ -0,0 +1,194 @@
+import { describe, it, expect, vi, beforeEach } from "vitest";
+
+vi.mock("../langwatch-api.js", async (importOriginal) => {
+  const actual = (await importOriginal()) as Record<string, unknown>;
+  return {
+    ...actual,
+    makeRequest: vi.fn(),
+  };
+});
+
+import { LangWatchApiError, makeRequest } from "../langwatch-api.js";
+import { handleExperimentStatus } from "../tools/run-experiment.js";
+import {
+  deriveRunStatus,
+  isTerminalStatus,
+} from "../tools/experiment-run-status.js";
+
+const mockMakeRequest = vi.mocked(makeRequest);
+
+beforeEach(() => {
+  vi.clearAllMocks();
+});
+
+describe("deriveRunStatus()", () => {
+  describe("when stoppedAt is set", () => {
+    it("is stopped", () => {
+      expect(deriveRunStatus({ stoppedAt: 5 })).toBe("stopped");
+    });
+  });
+
+  describe("when finishedAt is set", () => {
+    it("is completed", () => {
+      expect(deriveRunStatus({ finishedAt: 5 })).toBe("completed");
+    });
+  });
+
+  describe("when no terminal marker and updates are recent", () => {
+    it("is running", () => {
+      const now = 1_000_000;
+      expect(deriveRunStatus({ updatedAt: now - 1000 }, now)).toBe("running");
+    });
+  });
+
+  describe("when no terminal marker and no recent updates", () => {
+    it("is interrupted", () => {
+      const now = 1_000_000;
+      expect(deriveRunStatus({ updatedAt: now - 6 * 60 * 1000 }, now)).toBe(
+        "interrupted",
+      );
+    });
+  });
+
+  describe("terminal classification", () => {
+    it("treats completed and stopped as terminal", () => {
+      expect(isTerminalStatus("completed")).toBe(true);
+      expect(isTerminalStatus("stopped")).toBe(true);
+      expect(isTerminalStatus("running")).toBe(false);
+      expect(isTerminalStatus("interrupted")).toBe(false);
+    });
+  });
+});
+
+describe("handleExperimentStatus()", () => {
+  describe("given the run state exists in Redis", () => {
+    it("reports the status straight from the status endpoint", async () => {
+      mockMakeRequest.mockResolvedValueOnce({
+        runId: "run_1",
+        status: "completed",
+        progress: 3,
+        total: 3,
+      });
+
+      const out = await handleExperimentStatus({ runId: "run_1" });
+
+      expect(mockMakeRequest).toHaveBeenCalledWith(
+        "GET",
+        "/api/experiments/runs/run_1",
+      );
+      expect(out).toContain("**Status**: completed");
+      expect(out).toContain("3/3 cells");
+    });
+  });
+
+  describe("given an SDK-logged run with no Redis state", () => {
+    it("falls back to deriving status from the results endpoint", async () => {
+      mockMakeRequest
+        .mockRejectedValueOnce(
+          new LangWatchApiError("missing", 404, "Run not found"),
+        )
+        .mockResolvedValueOnce({
+          progress: 5,
+          total: 5,
+          dataset: [1, 2, 3, 4, 5],
+          timestamps: {
+            createdAt: 1,
+            updatedAt: 2,
+            finishedAt: 3,
+            stoppedAt: null,
+          },
+        });
+
+      const out = await handleExperimentStatus({
+        runId: "sdk_run",
+        experimentSlug: "doc-qa",
+      });
+
+      expect(mockMakeRequest).toHaveBeenNthCalledWith(
+        1,
+        "GET",
+        "/api/experiments/runs/sdk_run",
+      );
+      expect(mockMakeRequest).toHaveBeenNthCalledWith(
+        2,
+        "GET",
+        "/api/experiments/runs/sdk_run/results?experimentSlug=doc-qa",
+      );
+      expect(out).toContain("**Status**: completed");
+      expect(out).toContain("5/5 cells");
+      expect(out).toContain("platform_experiment_results");
+    });
+
+    it("reports interrupted for a stale unfinished SDK run", async () => {
+      mockMakeRequest
+        .mockRejectedValueOnce(
+          new LangWatchApiError("missing", 404, "Run not found"),
+        )
+        .mockResolvedValueOnce({
+          progress: 2,
+          total: 5,
+          dataset: [1, 2],
+          timestamps: {
+            createdAt: Date.now() - 60 * 60 * 1000,
+            updatedAt: Date.now() - 30 * 60 * 1000,
+            finishedAt: null,
+            stoppedAt: null,
+          },
+        });
+
+      const out = await handleExperimentStatus({
+        runId: "sdk_run",
+        experimentSlug: "doc-qa",
+      });
+
+      expect(out).toContain("**Status**: interrupted");
+      expect(out).toContain("2/5 cells");
+    });
+  });
+
+  describe("given the run cannot be found anywhere", () => {
+    it("returns actionable guidance instead of a raw 404", async () => {
+      mockMakeRequest
+        .mockRejectedValueOnce(
+          new LangWatchApiError("missing", 404, "Run not found"),
+        )
+        .mockRejectedValueOnce(
+          new LangWatchApiError("missing", 404, "Run not found"),
+        );
+
+      const out = await handleExperimentStatus({
+        runId: "nope",
+        experimentSlug: "doc-qa",
+      });
+
+      expect(out).toContain("not found");
+      expect(out).toContain("experimentSlug");
+      expect(out).toContain("platform_experiment_results");
+    });
+
+    it("rethrows non-404 errors from the status endpoint", async () => {
+      mockMakeRequest.mockRejectedValueOnce(
+        new LangWatchApiError("boom", 500, "Internal error"),
+      );
+
+      await expect(
+        handleExperimentStatus({ runId: "nope" }),
+      ).rejects.toThrow();
+    });
+
+    it("propagates a real (non-404) error from the results fallback instead of faking not-found", async () => {
+      mockMakeRequest
+        .mockRejectedValueOnce(
+          new LangWatchApiError("missing", 404, "Run not found"),
+        )
+        .mockRejectedValueOnce(
+          new LangWatchApiError("boom", 500, "Internal error"),
+        );
+
+      // The fallback's 500 ("boom") must surface, not the original 404 ("missing").
+      await expect(
+        handleExperimentStatus({ runId: "sdk_run", experimentSlug: "doc-qa" }),
+      ).rejects.toThrow(/boom/);
+    });
+  });
+});

package/src/__tests__/get-experiment-results.unit.test.ts

@@ -197,25 +197,116 @@ describe("handleExperimentResults()", () => {
         );
         const out = await handleExperimentResults({ runId: "missing" });
         expect(out).toContain("not found");
-        expect(out).toContain("platform_experiment_status");
+        expect(out).toContain("experimentSlug");
       });
     });
   });
 
   describe("given the run is still in progress", () => {
     describe("when results are requested", () => {
-      it("returns guidance to check status instead of partial results", async () => {
+      it("serves the partial rows logged so far with an in-progress note", async () => {
         mockMakeRequest.mockResolvedValueOnce({
           ...sample,
           progress: 2,
           total: 3,
-          timestamps: { createdAt: 0, updatedAt: 0, finishedAt: null },
+          timestamps: {
+            createdAt: Date.now(),
+            updatedAt: Date.now(),
+            finishedAt: null,
+            stoppedAt: null,
+          },
         });
 
         const out = await handleExperimentResults({ runId: "unfinished" });
 
-        expect(out).toContain("results are not yet available");
-        expect(out).toContain("platform_experiment_status");
+        expect(out).toContain("**Status**: running");
+        expect(out).toContain("still in progress");
+        expect(out).toContain("Row #0");
+        expect(out).toContain("Progress**: 2/3");
+      });
+    });
+  });
+
+  describe("given the run was interrupted before finishing", () => {
+    describe("when results are requested", () => {
+      it("serves the partial rows with an interrupted note", async () => {
+        mockMakeRequest.mockResolvedValueOnce({
+          ...sample,
+          timestamps: {
+            createdAt: Date.now() - 60 * 60 * 1000,
+            updatedAt: Date.now() - 30 * 60 * 1000,
+            finishedAt: null,
+            stoppedAt: null,
+          },
+        });
+
+        const out = await handleExperimentResults({ runId: "interrupted" });
+
+        expect(out).toContain("**Status**: interrupted");
+        expect(out).toContain("interrupted");
+        expect(out).toContain("Row #0");
+      });
+    });
+  });
+
+  describe("given a non-terminal run with zero rows", () => {
+    describe("when the run is still running", () => {
+      it("says the run is in progress", async () => {
+        mockMakeRequest.mockResolvedValueOnce({
+          ...sample,
+          dataset: [],
+          evaluations: [],
+          timestamps: {
+            createdAt: Date.now(),
+            updatedAt: Date.now(),
+            finishedAt: null,
+            stoppedAt: null,
+          },
+        });
+        const out = await handleExperimentResults({ runId: "empty-running" });
+        expect(out).toContain("still in progress");
+        expect(out).not.toContain("interrupted");
+      });
+    });
+
+    describe("when the run was interrupted", () => {
+      it("says no rows were recorded before it was interrupted, not to wait", async () => {
+        mockMakeRequest.mockResolvedValueOnce({
+          ...sample,
+          dataset: [],
+          evaluations: [],
+          timestamps: {
+            createdAt: Date.now() - 60 * 60 * 1000,
+            updatedAt: Date.now() - 30 * 60 * 1000,
+            finishedAt: null,
+            stoppedAt: null,
+          },
+        });
+        const out = await handleExperimentResults({ runId: "empty-interrupted" });
+        expect(out).toContain("interrupted");
+        expect(out).not.toContain("still in progress");
+        expect(out).not.toContain("Call again shortly");
+      });
+    });
+  });
+
+  describe("given no run state but rows exist (SDK / >24h run)", () => {
+    describe("when results are requested with a stopped run", () => {
+      it("renders results for a stopped run", async () => {
+        mockMakeRequest.mockResolvedValueOnce({
+          ...sample,
+          timestamps: {
+            createdAt: 0,
+            updatedAt: 10,
+            finishedAt: null,
+            stoppedAt: 20,
+          },
+        });
+
+        const out = await handleExperimentResults({ runId: "stopped" });
+
+        expect(out).toContain("**Status**: stopped");
+        expect(out).toContain("Row #0");
       });
     });
   });

package/src/create-mcp-server.d.ts

@@ -1,6 +1,8 @@
 // Minimal type declaration — avoids pulling mcp-server's full dependency tree
 // into the langwatch app's typecheck scope. The actual return type is McpServer,
-// but we only expose the connect() method that handler.ts uses.
+// but we only expose the connect() and tool() methods that handler.ts +
+// in-app tool registrations (e.g. registerGovernanceMcpTools) use.
 export declare function createMcpServer(): {
   connect(transport: unknown): Promise<void>;
+  tool(...args: unknown[]): unknown;
 };

package/src/create-mcp-server.ts

@@ -1563,9 +1563,15 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
 
   server.tool(
     "platform_experiment_status",
-    "Check the status of an experiment run. Returns progress and summary when completed.",
+    "Check the status of an experiment run (running, completed, stopped, or interrupted) plus its progress. Works for both platform-run and SDK-logged experiments; pass experimentSlug for SDK runs or runs older than 24h.",
     {
       runId: z.string().describe("The run ID returned from platform_run_experiment"),
+      experimentSlug: z
+        .string()
+        .optional()
+        .describe(
+          "Experiment slug — required for SDK-logged runs and runs older than 24h (the Redis run-state they would otherwise be read from is absent/expired). Use platform_experiment_list_runs to find the slug.",
+        ),
     },
     withToolLogging("platform_experiment_status", async (params) => {
       requireApiKey();
@@ -1626,7 +1632,7 @@ NOTE: Scenarios can be created two ways. Determine which approach the user needs
 
   server.tool(
     "platform_experiment_results",
-    "Fetch per-row results for a completed experiment run so you can debug evaluator scores and missed rows. Returns a markdown report: per-evaluator averages plus row-by-row scores and failure details. Output is capped at 50 rows to protect the agent's context window — narrow with `filter: 'failed'` or `evaluator` to see what matters.",
+    "Fetch per-row results for an experiment run so you can debug evaluator scores and missed rows. Serves partial results for runs that are still running or were interrupted (it does not require a completed run), with the run status noted in the output. Returns a markdown report: per-evaluator averages plus row-by-row scores and failure details. Output is capped at 50 rows to protect the agent's context window — narrow with `filter: 'failed'` or `evaluator` to see what matters.",
     {
       runId: z
         .string()

package/src/http-server.ts

@@ -271,8 +271,13 @@ export async function startHttpServer({
   const sseSessions: Record<string, SseSessionState> = {};
 
   app.get("/sse", async (req: Request, res: Response) => {
+    // SSE/EventSource clients cannot set custom Authorization headers, so
+    // the apiKey fallback via ?apiKey= is the standard pattern for SSE
+    // auth across the MCP/CLI ecosystem. The header form is preferred and
+    // checked first; this fallback is documented at the client level.
     const apiKey =
       extractBearerToken(req) ||
+      // lgtm[js/sensitive-get-query]
       (req.query["apiKey"] as string | undefined) ||
       null;
 

package/src/tools/experiment-run-status.ts

@@ -0,0 +1,36 @@
+// Experiment runs are stored as a timestamp-driven state in ClickHouse: a run
+// is "completed" once finishedAt is set, "stopped" once stoppedAt is set, and
+// otherwise still "running". A run that stopped emitting updates a while ago
+// without ever finishing is reported as "interrupted": the SDK process likely
+// died before sending finished_at/stopped_at.
+
+export const INTERRUPTED_THRESHOLD_MS = 5 * 60 * 1000;
+
+export type RunStatus = "completed" | "stopped" | "running" | "interrupted";
+
+export interface RunTimestamps {
+  createdAt?: number | null;
+  updatedAt?: number | null;
+  finishedAt?: number | null;
+  stoppedAt?: number | null;
+}
+
+export const deriveRunStatus = (
+  timestamps: RunTimestamps,
+  now: number = Date.now(),
+): RunStatus => {
+  if (timestamps.stoppedAt != null) return "stopped";
+  if (timestamps.finishedAt != null) return "completed";
+  if (
+    timestamps.updatedAt != null &&
+    now - timestamps.updatedAt > INTERRUPTED_THRESHOLD_MS
+  ) {
+    return "interrupted";
+  }
+  return "running";
+};
+
+// Whether the run is in a terminal state. Non-terminal runs (running /
+// interrupted) can still be queried; their results are partial.
+export const isTerminalStatus = (status: RunStatus): boolean =>
+  status === "completed" || status === "stopped";

package/src/tools/get-experiment-results.ts

@@ -1,4 +1,5 @@
 import { LangWatchApiError, makeRequest } from "../langwatch-api.js";
+import { deriveRunStatus, isTerminalStatus } from "./experiment-run-status.js";
 
 interface DatasetEntry {
   index: number;
@@ -110,26 +111,34 @@ export async function handleExperimentResults(params: {
         "",
         "**Status**: not found",
         "",
-        `Could not load results for run \`${params.runId}\`. The run may still be in progress, or the id may be incorrect.`,
+        `Could not load results for run \`${params.runId}\`. The run id may be incorrect, or its run state may have expired (Redis keeps it for 24h).`,
         "",
-        "> Try \`platform_experiment_status\` first to confirm the run id and that it is completed.",
+        "> Pass `experimentSlug` to load results for runs older than 24h. Discover the slug with `platform_experiment_list`, then `platform_experiment_list_runs` for the run ids.",
       ].join("\n");
     }
     throw error;
   }
 
-  if (!results || (results.timestamps.finishedAt == null && results.timestamps.stoppedAt == null)) {
+  if (!results) {
     return [
       `# Evaluation Results: ${params.runId}`,
       "",
-      "**Status**: results are not yet available",
+      "**Status**: not found",
       "",
-      `Results for run \`${params.runId}\` are not ready yet.`,
+      `Could not load results for run \`${params.runId}\`.`,
       "",
-      "> Try `platform_experiment_status` first to confirm the run id and that it is completed.",
+      "> Pass `experimentSlug` if the run is older than 24h. Discover the slug with `platform_experiment_list`, then `platform_experiment_list_runs` to confirm the run id.",
     ].join("\n");
   }
 
+  // Partial results are served even while the run is still in progress.
+  // Rows land in ClickHouse incrementally, so a "running" or "interrupted"
+  // run can still expose every row recorded so far. A run only fails to set
+  // finished_at/stopped_at when the SDK process dies before flushing, but the
+  // rows it did log are still useful, so we never gate on a terminal status.
+  const runStatus = deriveRunStatus(results.timestamps);
+  const partial = !isTerminalStatus(runStatus);
+
   // Group evaluations by target-scoped row key, applying evaluator filter.
   const evaluationsByRow = new Map<string, EvaluationItem[]>();
   for (const evaluation of results.evaluations) {
@@ -196,6 +205,12 @@ export async function handleExperimentResults(params: {
   lines.push(`# Evaluation Results: ${results.runId}`);
   lines.push("");
   lines.push(`**Experiment**: ${results.experimentId}`);
+  lines.push(`**Status**: ${runStatus}`);
+  if (typeof results.total === "number" && results.total > 0) {
+    lines.push(
+      `**Progress**: ${results.progress ?? results.dataset.length}/${results.total} rows`,
+    );
+  }
   lines.push(`**Total rows**: ${results.dataset.length}`);
   lines.push(`**Total evaluations**: ${results.evaluations.length}`);
   if (filter === "failed") {
@@ -204,6 +219,14 @@ export async function handleExperimentResults(params: {
   if (evaluatorFilter) {
     lines.push(`**Evaluator filter**: ${evaluatorFilter}`);
   }
+  if (partial) {
+    lines.push("");
+    lines.push(
+      runStatus === "interrupted"
+        ? "> These are partial results. The run never sent a finished/stopped marker and has had no updates recently, so it likely was interrupted before completing. The rows below are everything recorded so far."
+        : "> These are partial results. The run is still in progress, so more rows may appear on a later call.",
+    );
+  }
   lines.push("");
 
   if (evaluatorAverages.size > 0) {
@@ -222,7 +245,17 @@ export async function handleExperimentResults(params: {
   }
 
   if (rows.length === 0) {
-    lines.push("_No rows matched the filter._");
+    if (filter === "failed") {
+      lines.push("_No rows matched the filter._");
+    } else if (runStatus === "running") {
+      lines.push(
+        "_No rows recorded yet. The run is still in progress; call again shortly._",
+      );
+    } else if (runStatus === "interrupted") {
+      lines.push("_No rows were recorded before the run was interrupted._");
+    } else {
+      lines.push("_No rows recorded for this run._");
+    }
     return lines.join("\n");
   }
 

package/src/tools/run-experiment.ts

@@ -1,4 +1,5 @@
-import { makeRequest } from "../langwatch-api.js";
+import { LangWatchApiError, makeRequest } from "../langwatch-api.js";
+import { deriveRunStatus } from "./experiment-run-status.js";
 
 interface EvaluationRunResponse {
   runId: string;
@@ -45,13 +46,107 @@ export async function handleRunExperiment(params: {
   return lines.join("\n");
 }
 
+interface ResultsForStatus {
+  progress?: number | null;
+  total?: number | null;
+  dataset?: unknown[];
+  timestamps: {
+    createdAt?: number | null;
+    updatedAt?: number | null;
+    finishedAt?: number | null;
+    stoppedAt?: number | null;
+  };
+}
+
+// SDK-logged runs (langwatch.experiment + evaluation.log) never populate the
+// Redis run-state that GET /runs/:runId reads, so that endpoint 404s for them.
+// Their data lives only in ClickHouse, reachable through the results endpoint,
+// so we derive the status from there as a fallback. experimentSlug is required
+// because runId is not unique across experiments once the Redis state expires.
+async function statusFromResults(params: {
+  runId: string;
+  experimentSlug?: string;
+}): Promise<string | null> {
+  const search = new URLSearchParams();
+  if (params.experimentSlug) search.set("experimentSlug", params.experimentSlug);
+  const qs = search.toString() ? `?${search.toString()}` : "";
+
+  let results: ResultsForStatus;
+  try {
+    results = (await makeRequest(
+      "GET",
+      `/api/experiments/runs/${encodeURIComponent(params.runId)}/results${qs}`,
+    )) as ResultsForStatus;
+  } catch (error) {
+    // Only a genuine "no such run" is a fallback miss (-> guidance). Real
+    // 5xx / auth / network errors must propagate, not be masked as not-found.
+    const code = error instanceof LangWatchApiError ? error.status : undefined;
+    const message = error instanceof Error ? error.message : String(error);
+    if (code === 404 || (code === undefined && /404|not found/i.test(message))) {
+      return null;
+    }
+    throw error;
+  }
+
+  const status = deriveRunStatus(results.timestamps);
+  const total = results.total ?? results.dataset?.length ?? 0;
+  const progress = results.progress ?? results.dataset?.length ?? 0;
+
+  const lines: string[] = [];
+  lines.push(`# Evaluation Run ${params.runId}\n`);
+  lines.push(`**Status**: ${status}`);
+  lines.push(`**Progress**: ${progress}/${total} cells`);
+  if (results.timestamps.createdAt) {
+    lines.push(
+      `**Started**: ${new Date(results.timestamps.createdAt).toISOString()}`,
+    );
+  }
+  if (results.timestamps.finishedAt) {
+    lines.push(
+      `**Finished**: ${new Date(results.timestamps.finishedAt).toISOString()}`,
+    );
+  }
+  if (results.timestamps.stoppedAt) {
+    lines.push(
+      `**Stopped**: ${new Date(results.timestamps.stoppedAt).toISOString()}`,
+    );
+  }
+  lines.push("");
+  lines.push(
+    "> Use `platform_experiment_results` to fetch the per-row scores (partial results are available even while running).",
+  );
+  return lines.join("\n");
+}
+
 export async function handleExperimentStatus(params: {
   runId: string;
+  experimentSlug?: string;
 }): Promise<string> {
-  const status = (await makeRequest(
-    "GET",
-    `/api/experiments/runs/${encodeURIComponent(params.runId)}`,
-  )) as EvaluationStatusResponse;
+  let status: EvaluationStatusResponse;
+  try {
+    status = (await makeRequest(
+      "GET",
+      `/api/experiments/runs/${encodeURIComponent(params.runId)}`,
+    )) as EvaluationStatusResponse;
+  } catch (error) {
+    const code =
+      error instanceof LangWatchApiError ? error.status : undefined;
+    const message = error instanceof Error ? error.message : String(error);
+    if (code === 404 || /404|not found/i.test(message)) {
+      const fallback = await statusFromResults(params);
+      if (fallback) return fallback;
+      return [
+        `# Evaluation Run ${params.runId}`,
+        "",
+        "**Status**: not found",
+        "",
+        `Could not find run \`${params.runId}\`. SDK-logged runs and runs older than 24h are not in the live run-state and must be resolved by experiment slug.`,
+        "",
+        "> Pass `experimentSlug`: discover it with `platform_experiment_list`, then use `platform_experiment_list_runs` for the run ids. Or fetch the rows directly with `platform_experiment_results`.",
+      ].join("\n");
+    }
+    throw error;
+  }
 
   const lines: string[] = [];
   lines.push(`# Evaluation Run ${status.runId}\n`);