@peerlm/mcp 0.1.0

package/dist/index.d.ts

@@ -0,0 +1 @@
+#!/usr/bin/env node

package/dist/index.js

@@ -0,0 +1,679 @@
+#!/usr/bin/env node
+
+// src/index.ts
+import { Server } from "@modelcontextprotocol/sdk/server/index.js";
+import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
+import {
+  CallToolRequestSchema,
+  ListToolsRequestSchema
+} from "@modelcontextprotocol/sdk/types.js";
+
+// src/client.ts
+var PeerLMError = class extends Error {
+  constructor(status, message) {
+    super(message);
+    this.status = status;
+    this.name = "PeerLMError";
+  }
+};
+var PeerLMClient = class {
+  baseUrl;
+  apiKey;
+  constructor(apiKey2, baseUrl2) {
+    this.apiKey = apiKey2;
+    this.baseUrl = baseUrl2.replace(/\/$/, "");
+  }
+  async request(method, path, body) {
+    const url = `${this.baseUrl}/api/v1${path}`;
+    const headers = {
+      "X-API-Key": this.apiKey,
+      "Content-Type": "application/json"
+    };
+    let res;
+    try {
+      res = await fetch(url, {
+        method,
+        headers,
+        body: body ? JSON.stringify(body) : void 0
+      });
+    } catch (err) {
+      throw new PeerLMError(
+        0,
+        `PeerLM API unreachable at ${this.baseUrl}. Check your PEERLM_BASE_URL and network connection.`
+      );
+    }
+    const json = await res.json();
+    if (!res.ok) {
+      const message = json?.error || `HTTP ${res.status}`;
+      throw new PeerLMError(res.status, message);
+    }
+    return json.data;
+  }
+  async listSuites() {
+    return this.request("GET", "/suites");
+  }
+  async getSuite(suiteId) {
+    return this.request("GET", `/suites/${suiteId}`);
+  }
+  async listModels(params) {
+    const query = new URLSearchParams();
+    if (params?.provider) query.set("provider", params.provider);
+    if (params?.tier) query.set("tier", params.tier);
+    const qs = query.toString();
+    return this.request("GET", `/models${qs ? `?${qs}` : ""}`);
+  }
+  async getUsage() {
+    return this.request("GET", "/usage");
+  }
+  async createSystemPrompt(data) {
+    return this.request("POST", "/system-prompts", data);
+  }
+  async createTestPrompt(data) {
+    return this.request("POST", "/test-prompts", data);
+  }
+  async createSuite(data) {
+    return this.request("POST", "/suites", data);
+  }
+  async runEval(suiteId) {
+    return this.request("POST", "/runs", { suite_id: suiteId });
+  }
+  async getResults(runId) {
+    return this.request("GET", `/runs/${runId}`);
+  }
+};
+
+// src/formatters.ts
+function formatSuiteList(suites) {
+  if (suites.length === 0) return "No evaluation suites found.";
+  const lines = suites.map(
+    (s) => `- **${s.name}** (v${s.version})
+  ID: ${s.id}${s.description ? `
+  ${s.description}` : ""}`
+  );
+  return `Found ${suites.length} suite(s):
+
+${lines.join("\n\n")}`;
+}
+function formatSuiteDetail(suite) {
+  const config = suite.configuration;
+  const lines = [
+    `**${suite.name}** (v${suite.version})`,
+    `ID: ${suite.id}`
+  ];
+  if (suite.description) lines.push(`Description: ${suite.description}`);
+  if (config) {
+    lines.push("");
+    lines.push(`Generator Models: ${config.generatorModels.length} model(s)`);
+    config.generatorModels.forEach((id) => lines.push(`  - ${id}`));
+    lines.push(`Evaluator Models: ${config.evaluatorModels.length} model(s)`);
+    config.evaluatorModels.forEach((id) => lines.push(`  - ${id}`));
+    lines.push(`System Prompts: ${config.systemPrompts.length}`);
+    lines.push(`Test Prompts: ${config.testPrompts.length}`);
+    lines.push(`Samples per Prompt: ${config.responsesPerTopicPerPersona || 1}`);
+    lines.push(`Deterministic Mode: ${config.deterministicMode ? "Yes" : "No"}`);
+    if (config.criteria?.length) {
+      lines.push("");
+      lines.push("Criteria:");
+      config.criteria.forEach(
+        (c) => lines.push(`  - ${c.label} (weight: ${c.weight}): ${c.description}`)
+      );
+    }
+    if (config.thresholds) {
+      lines.push("");
+      lines.push("Thresholds:");
+      if (config.thresholds.min_overall_score != null)
+        lines.push(`  Min Overall Score: ${config.thresholds.min_overall_score}`);
+      if (config.thresholds.max_latency_ms != null)
+        lines.push(`  Max Latency: ${config.thresholds.max_latency_ms}ms`);
+      if (config.thresholds.min_criteria_scores) {
+        for (const [k, v] of Object.entries(config.thresholds.min_criteria_scores)) {
+          lines.push(`  Min ${k}: ${v}`);
+        }
+      }
+    }
+  }
+  return lines.join("\n");
+}
+function formatModelList(models) {
+  if (models.length === 0) return "No models found matching your filters.";
+  const byProvider = /* @__PURE__ */ new Map();
+  for (const m of models) {
+    const list = byProvider.get(m.provider) || [];
+    list.push(m);
+    byProvider.set(m.provider, list);
+  }
+  const sections = [];
+  for (const [provider, provModels] of byProvider) {
+    const lines = provModels.map((m) => {
+      const pricing = m.pricingPrompt && m.pricingCompletion ? ` | $${m.pricingPrompt}/$${m.pricingCompletion} per 1M tokens` : "";
+      return `  - **${m.name}** [${m.tier}]${pricing}
+    ID: ${m.id}`;
+    });
+    sections.push(`**${provider}**
+${lines.join("\n")}`);
+  }
+  return `${models.length} model(s) available:
+
+${sections.join("\n\n")}`;
+}
+function formatRunStatus(run) {
+  const lines = [
+    `**${run.name || "Evaluation Run"}**`,
+    `Run ID: ${run.id}`,
+    `Status: ${run.status}`
+  ];
+  if (run.status === "queued" || run.status === "running") {
+    if (run.progress) {
+      const p = run.progress;
+      lines.push(`Phase: ${p.phase}`);
+      if (p.phase === "generating") {
+        const pct = p.totalResponses > 0 ? Math.round(p.completedResponses / p.totalResponses * 100) : 0;
+        lines.push(
+          `Progress: ${p.completedResponses}/${p.totalResponses} responses (${pct}%)`
+        );
+        if (p.cacheHits > 0) lines.push(`Cache Hits: ${p.cacheHits}`);
+      } else if (p.phase === "evaluating") {
+        const pct = p.totalEvaluations > 0 ? Math.round(
+          p.completedEvaluations / p.totalEvaluations * 100
+        ) : 0;
+        lines.push(
+          `Progress: ${p.completedEvaluations}/${p.totalEvaluations} evaluations (${pct}%)`
+        );
+      } else if (p.phase === "aggregating") {
+        lines.push("Aggregating results...");
+      }
+    }
+    if (run.evalCreditsEstimated) {
+      lines.push(`Estimated Credits: ${run.evalCreditsEstimated}`);
+    }
+    return lines.join("\n");
+  }
+  if (run.status === "failed") {
+    lines.push(`Error: ${run.errorMessage || "Unknown error"}`);
+    return lines.join("\n");
+  }
+  if (run.passed != null) {
+    lines.push(`Result: ${run.passed ? "PASSED" : "FAILED"}`);
+  }
+  if (run.evalCreditsConsumed != null) {
+    lines.push(`Credits Used: ${run.evalCreditsConsumed}`);
+  }
+  if (run.resultsSummary?.leaderboard?.length) {
+    lines.push("");
+    lines.push(formatLeaderboard(run.resultsSummary.leaderboard));
+  }
+  if (run.resultsSummary?.decisionMeta) {
+    const dm = run.resultsSummary.decisionMeta;
+    if (dm.recommendation) {
+      lines.push("");
+      lines.push(`Recommendation: ${dm.recommendation}`);
+    }
+    if (dm.insights?.length) {
+      lines.push("Insights:");
+      dm.insights.forEach((i) => lines.push(`  - ${i}`));
+    }
+  }
+  if (run.thresholdViolations) {
+    lines.push("");
+    lines.push(
+      `Threshold Violations: ${JSON.stringify(run.thresholdViolations)}`
+    );
+  }
+  return lines.join("\n");
+}
+function formatLeaderboard(entries) {
+  const header = "Rank | Model | Score | Latency | Cost";
+  const separator = "-----|-------|-------|---------|-----";
+  const rows = entries.map((e) => {
+    const latency = e.avgLatencyMs != null ? `${Math.round(e.avgLatencyMs)}ms` : "-";
+    const cost = e.totalCostUsd != null ? `$${e.totalCostUsd.toFixed(4)}` : "-";
+    return `#${e.rank} | ${e.modelName} | ${e.overallScore.toFixed(2)} | ${latency} | ${cost}`;
+  });
+  return `Leaderboard:
+${header}
+${separator}
+${rows.join("\n")}`;
+}
+function formatUsage(usage) {
+  const lines = [
+    `Plan: ${usage.plan}`,
+    `Credits: ${usage.creditsRemaining} remaining / ${usage.creditsIncluded} included`,
+    `Credits Used: ${usage.creditsUsed}`
+  ];
+  if (usage.overageEnabled) lines.push("Overage Billing: Enabled");
+  if (usage.paygEnabled) lines.push("Pay-as-you-go: Enabled");
+  return lines.join("\n");
+}
+
+// src/tools/shared.ts
+function handleToolError(err) {
+  if (err instanceof PeerLMError) {
+    let message2 = err.message;
+    if (err.status === 401) {
+      message2 = "Invalid or expired API key. Check your PEERLM_API_KEY. API access requires a Pro or Enterprise plan.";
+    } else if (err.status === 402) {
+      message2 = "Insufficient credits. Visit PeerLM to add credits or upgrade your plan.";
+    } else if (err.status === 403) {
+      message2 = err.message || "This action requires a read-write API key.";
+    }
+    return {
+      content: [{ type: "text", text: message2 }],
+      isError: true
+    };
+  }
+  const message = err instanceof Error ? err.message : "An unexpected error occurred.";
+  return {
+    content: [{ type: "text", text: message }],
+    isError: true
+  };
+}
+
+// src/tools/list-suites.ts
+var listSuitesDescription = "List all evaluation suites in your PeerLM workspace. Returns suite names, IDs, versions, and descriptions.";
+async function listSuitesHandler(client2) {
+  try {
+    const suites = await client2.listSuites();
+    return { content: [{ type: "text", text: formatSuiteList(suites) }] };
+  } catch (err) {
+    return handleToolError(err);
+  }
+}
+
+// src/tools/get-suite.ts
+var getSuiteSchema = {
+  suite_id: {
+    type: "string",
+    description: "The ID of the evaluation suite to retrieve"
+  }
+};
+var getSuiteDescription = "Get full details of an evaluation suite including generator/evaluator models, prompts, criteria, and thresholds.";
+async function getSuiteHandler(client2, args) {
+  try {
+    const suite = await client2.getSuite(args.suite_id);
+    return { content: [{ type: "text", text: formatSuiteDetail(suite) }] };
+  } catch (err) {
+    return handleToolError(err);
+  }
+}
+
+// src/tools/list-models.ts
+var listModelsSchema = {
+  provider: {
+    type: "string",
+    description: "Filter by provider (e.g. openai, anthropic, google, mistral)"
+  },
+  tier: {
+    type: "string",
+    description: "Filter by tier (standard, advanced, premium, frontier)"
+  }
+};
+var listModelsDescription = "List available LLM models in PeerLM with their IDs, providers, tiers, and pricing. Use these IDs when creating suites.";
+async function listModelsHandler(client2, args) {
+  try {
+    const models = await client2.listModels(args);
+    return { content: [{ type: "text", text: formatModelList(models) }] };
+  } catch (err) {
+    return handleToolError(err);
+  }
+}
+
+// src/tools/get-usage.ts
+var getUsageDescription = "Get your current PeerLM plan, credit balance, and usage information.";
+async function getUsageHandler(client2) {
+  try {
+    const usage = await client2.getUsage();
+    return { content: [{ type: "text", text: formatUsage(usage) }] };
+  } catch (err) {
+    return handleToolError(err);
+  }
+}
+
+// src/tools/create-system-prompt.ts
+var createSystemPromptSchema = {
+  name: {
+    type: "string",
+    description: "A descriptive name for the system prompt"
+  },
+  system_prompt: {
+    type: "string",
+    description: "The full system prompt text that defines the role/context given to models during evaluation"
+  },
+  description: {
+    type: "string",
+    description: "Optional description of what this system prompt tests"
+  },
+  tags: {
+    type: "array",
+    items: { type: "string" },
+    description: "Optional tags for organization"
+  }
+};
+var createSystemPromptDescription = "Create a system prompt (persona) in PeerLM's library. System prompts define the role/context given to models during evaluation. Returns the prompt ID needed for creating suites.";
+async function createSystemPromptHandler(client2, args) {
+  try {
+    const result = await client2.createSystemPrompt(args);
+    return {
+      content: [
+        {
+          type: "text",
+          text: `System prompt created successfully.
+
+ID: ${result.id}
+Name: ${result.name}${result.description ? `
+Description: ${result.description}` : ""}`
+        }
+      ]
+    };
+  } catch (err) {
+    return handleToolError(err);
+  }
+}
+
+// src/tools/create-test-prompt.ts
+var createTestPromptSchema = {
+  title: {
+    type: "string",
+    description: "A descriptive title for the test prompt"
+  },
+  prompt: {
+    type: "string",
+    description: "The actual question or task that models will be evaluated on"
+  },
+  description: {
+    type: "string",
+    description: "Optional description of what this prompt tests"
+  },
+  tags: {
+    type: "array",
+    items: { type: "string" },
+    description: "Optional tags for organization"
+  }
+};
+var createTestPromptDescription = "Create a test prompt (task) in PeerLM's library. Test prompts are the actual questions/tasks models will be evaluated on. Returns the prompt ID needed for creating suites.";
+async function createTestPromptHandler(client2, args) {
+  try {
+    const result = await client2.createTestPrompt(args);
+    return {
+      content: [
+        {
+          type: "text",
+          text: `Test prompt created successfully.
+
+ID: ${result.id}
+Title: ${result.title}${result.description ? `
+Description: ${result.description}` : ""}`
+        }
+      ]
+    };
+  } catch (err) {
+    return handleToolError(err);
+  }
+}
+
+// src/tools/create-suite.ts
+var createSuiteSchema = {
+  name: {
+    type: "string",
+    description: "Name for the evaluation suite"
+  },
+  description: {
+    type: "string",
+    description: "Optional description of the evaluation purpose"
+  },
+  generator_models: {
+    type: "array",
+    items: { type: "string" },
+    description: "Model IDs to compare (minimum 2). Use list_models to find IDs."
+  },
+  evaluator_models: {
+    type: "array",
+    items: { type: "string" },
+    description: "Model IDs to use as evaluators (minimum 1). Use list_models to find IDs."
+  },
+  system_prompt_ids: {
+    type: "array",
+    items: { type: "string" },
+    description: "IDs of system prompts to use. Create them first with create_system_prompt."
+  },
+  test_prompt_ids: {
+    type: "array",
+    items: { type: "string" },
+    description: "IDs of test prompts to use. Create them first with create_test_prompt."
+  },
+  criteria: {
+    type: "array",
+    items: {
+      type: "object",
+      properties: {
+        label: { type: "string", description: "Criterion name (e.g. Accuracy)" },
+        description: { type: "string", description: "What the evaluator should assess" },
+        weight: { type: "number", description: "Relative weight (default 1)" }
+      },
+      required: ["label", "description", "weight"]
+    },
+    description: "Evaluation criteria with labels, descriptions, and weights"
+  },
+  deterministic_mode: {
+    type: "boolean",
+    description: "If true, sets temperature=0 and fixed seed where supported (default false)"
+  },
+  samples_per_prompt: {
+    type: "number",
+    description: "Number of response samples per model per prompt (default 1)"
+  },
+  evaluation_method: {
+    type: "string",
+    enum: ["rubric", "comparative"],
+    description: 'Evaluation method (default "rubric")'
+  }
+};
+var createSuiteDescription = "Create an evaluation suite that defines which models to compare, which prompts to test, and which criteria to score on. Use list_models to find model IDs, and create_system_prompt/create_test_prompt to create prompts first.";
+async function createSuiteHandler(client2, args) {
+  try {
+    const result = await client2.createSuite(args);
+    return {
+      content: [
+        {
+          type: "text",
+          text: `Suite created successfully.
+
+ID: ${result.id}
+Name: ${result.name}
+Version: ${result.version}
+
+You can now run an evaluation with run_eval using this suite ID.`
+        }
+      ]
+    };
+  } catch (err) {
+    return handleToolError(err);
+  }
+}
+
+// src/tools/run-eval.ts
+var runEvalSchema = {
+  suite_id: {
+    type: "string",
+    description: "The ID of the evaluation suite to run"
+  }
+};
+var runEvalDescription = "Trigger an evaluation run for a suite. The evaluation runs asynchronously \u2014 use get_results to check progress and retrieve results.";
+async function runEvalHandler(client2, args) {
+  try {
+    const run = await client2.runEval(args.suite_id);
+    const lines = [
+      "Evaluation run started.",
+      "",
+      `Run ID: ${run.id}`,
+      `Status: ${run.status}`
+    ];
+    if (run.evalCreditsEstimated) {
+      lines.push(`Estimated Credits: ${run.evalCreditsEstimated}`);
+    }
+    lines.push(
+      "",
+      "The evaluation is running asynchronously. Use get_results with this run ID to check progress and retrieve results."
+    );
+    return { content: [{ type: "text", text: lines.join("\n") }] };
+  } catch (err) {
+    return handleToolError(err);
+  }
+}
+
+// src/tools/get-results.ts
+var getResultsSchema = {
+  run_id: {
+    type: "string",
+    description: "The ID of the evaluation run to check"
+  }
+};
+var getResultsDescription = "Get the status and results of an evaluation run. Shows progress while running, and a full leaderboard with scores, latency, cost, and decision insights when completed.";
+async function getResultsHandler(client2, args) {
+  try {
+    const run = await client2.getResults(args.run_id);
+    return { content: [{ type: "text", text: formatRunStatus(run) }] };
+  } catch (err) {
+    return handleToolError(err);
+  }
+}
+
+// src/index.ts
+var apiKey = process.env.PEERLM_API_KEY;
+if (!apiKey) {
+  console.error(
+    'Error: PEERLM_API_KEY environment variable is required.\n\nSet it in your MCP client config:\n  "env": { "PEERLM_API_KEY": "plm_live_..." }\n\nGenerate an API key at https://app.peerlm.com/settings/api-keys'
+  );
+  process.exit(1);
+}
+var baseUrl = process.env.PEERLM_BASE_URL || "https://app.peerlm.com";
+var client = new PeerLMClient(apiKey, baseUrl);
+var server = new Server(
+  { name: "peerlm", version: "0.1.0" },
+  { capabilities: { tools: {} } }
+);
+server.setRequestHandler(ListToolsRequestSchema, async () => ({
+  tools: [
+    {
+      name: "list_suites",
+      description: listSuitesDescription,
+      inputSchema: { type: "object", properties: {} }
+    },
+    {
+      name: "get_suite",
+      description: getSuiteDescription,
+      inputSchema: {
+        type: "object",
+        properties: getSuiteSchema,
+        required: ["suite_id"]
+      }
+    },
+    {
+      name: "list_models",
+      description: listModelsDescription,
+      inputSchema: {
+        type: "object",
+        properties: listModelsSchema
+      }
+    },
+    {
+      name: "get_usage",
+      description: getUsageDescription,
+      inputSchema: { type: "object", properties: {} }
+    },
+    {
+      name: "create_system_prompt",
+      description: createSystemPromptDescription,
+      inputSchema: {
+        type: "object",
+        properties: createSystemPromptSchema,
+        required: ["name", "system_prompt"]
+      }
+    },
+    {
+      name: "create_test_prompt",
+      description: createTestPromptDescription,
+      inputSchema: {
+        type: "object",
+        properties: createTestPromptSchema,
+        required: ["title", "prompt"]
+      }
+    },
+    {
+      name: "create_suite",
+      description: createSuiteDescription,
+      inputSchema: {
+        type: "object",
+        properties: createSuiteSchema,
+        required: [
+          "name",
+          "generator_models",
+          "evaluator_models",
+          "system_prompt_ids",
+          "test_prompt_ids",
+          "criteria"
+        ]
+      }
+    },
+    {
+      name: "run_eval",
+      description: runEvalDescription,
+      inputSchema: {
+        type: "object",
+        properties: runEvalSchema,
+        required: ["suite_id"]
+      }
+    },
+    {
+      name: "get_results",
+      description: getResultsDescription,
+      inputSchema: {
+        type: "object",
+        properties: getResultsSchema,
+        required: ["run_id"]
+      }
+    }
+  ]
+}));
+server.setRequestHandler(CallToolRequestSchema, async (request) => {
+  const { name, arguments: args } = request.params;
+  switch (name) {
+    case "list_suites":
+      return listSuitesHandler(client);
+    case "get_suite":
+      return getSuiteHandler(client, args);
+    case "list_models":
+      return listModelsHandler(client, args);
+    case "get_usage":
+      return getUsageHandler(client);
+    case "create_system_prompt":
+      return createSystemPromptHandler(
+        client,
+        args
+      );
+    case "create_test_prompt":
+      return createTestPromptHandler(
+        client,
+        args
+      );
+    case "create_suite":
+      return createSuiteHandler(client, args);
+    case "run_eval":
+      return runEvalHandler(client, args);
+    case "get_results":
+      return getResultsHandler(client, args);
+    default:
+      return {
+        content: [{ type: "text", text: `Unknown tool: ${name}` }],
+        isError: true
+      };
+  }
+});
+async function main() {
+  const transport = new StdioServerTransport();
+  await server.connect(transport);
+  console.error("PeerLM MCP server running on stdio");
+}
+main().catch((err) => {
+  console.error("Fatal error:", err);
+  process.exit(1);
+});

package/package.json

@@ -0,0 +1,26 @@
+{
+  "name": "@peerlm/mcp",
+  "version": "0.1.0",
+  "description": "MCP server for PeerLM — run evaluations from Claude Desktop, Cursor, or any MCP client",
+  "type": "module",
+  "bin": {
+    "peerlm-mcp": "dist/index.js"
+  },
+  "main": "./dist/index.js",
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build": "tsup src/index.ts --format esm --target node20 --dts --clean",
+    "type-check": "tsc --noEmit",
+    "dev": "tsup src/index.ts --format esm --target node20 --watch"
+  },
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "^1.12.1"
+  },
+  "devDependencies": {
+    "@types/node": "^22.0.0",
+    "tsup": "^8.3.0",
+    "typescript": "^5.7.0"
+  }
+}