@arizeai/phoenix-mcp 2.1.1 → 2.1.2

package/README.md

@@ -23,7 +23,7 @@ You can use Phoenix MCP Server for:
 - **Datasets**: Explore datasets, and syntesize new examples
 - **Experiments**: Pull experiment results and visualize them with the help of an LLM
 
-Don't se a use-case covered? `@arizeai/phoenix-mcp` is [open-source](https://github.com/Arize-ai/phoenix)! Issues and PRs welcome.
+Don't see a use-case covered? `@arizeai/phoenix-mcp` is [open-source](https://github.com/Arize-ai/phoenix)! Issues and PRs welcome.
 
 ## Installation
 
@@ -50,15 +50,15 @@ This MCP server can be used using `npx` and can be directly integrated with clie
 
 ## Install
 
-This package as managed via a pnpm workspace.
+This package is managed via a pnpm workspace.
 
 ```sh
 // From the /js/ directory
-pnpm install -r
-pnpm build -r
+pnpm install
+pnpm build
 ```
 
-This only needs to be repeated if dependancies change or there is a change to the phoenix-client.
+This only needs to be repeated if dependencies change or there is a change to the phoenix-client.
 
 ### Building
 

package/build/datasetTools.js

@@ -1,6 +1,91 @@
 import z from "zod";
+const LIST_DATASETS_DESCRIPTION = `Get a list of all datasets.
+
+Datasets are collections of 'dataset examples' that each example includes an input, 
+(expected) output, and optional metadata. They are primarily used as inputs for experiments.
+
+Example usage:
+  Show me all available datasets
+
+Expected return:
+  Array of dataset objects with metadata.
+  Example: [
+    {
+      "id": "RGF0YXNldDox",
+      "name": "my-dataset",
+      "description": "A dataset for testing",
+      "metadata": {},
+      "created_at": "2024-03-20T12:00:00Z",
+      "updated_at": "2024-03-20T12:00:00Z"
+    }
+  ]`;
+const GET_DATASET_EXAMPLES_DESCRIPTION = `Get examples from a dataset.
+
+Dataset examples are an array of objects that each include an input, 
+(expected) output, and optional metadata. These examples are typically used to represent 
+input to an application or model (e.g. prompt template variables, a code file, or image) 
+and used to test or benchmark changes.
+
+Example usage:
+  Show me all examples from dataset RGF0YXNldDox
+
+Expected return:
+  Object containing dataset ID, version ID, and array of examples.
+  Example: {
+    "dataset_id": "datasetid1234",
+    "version_id": "datasetversionid1234",
+    "examples": [
+      {
+        "id": "exampleid1234",
+        "input": {
+          "text": "Sample input text"
+        },
+        "output": {
+          "text": "Expected output text"
+        },
+        "metadata": {},
+        "updated_at": "YYYY-MM-DDTHH:mm:ssZ"
+      }
+    ]
+  }`;
+const GET_DATASET_EXPERIMENTS_DESCRIPTION = `List experiments run on a dataset.
+
+Example usage:
+  Show me all experiments run on dataset RGF0YXNldDox
+
+Expected return:
+  Array of experiment objects with metadata.
+  Example: [
+    {
+      "id": "experimentid1234",
+      "dataset_id": "datasetid1234",
+      "dataset_version_id": "datasetversionid1234",
+      "repetitions": 1,
+      "metadata": {},
+      "project_name": "Experiment-abc123",
+      "created_at": "YYYY-MM-DDTHH:mm:ssZ",
+      "updated_at": "YYYY-MM-DDTHH:mm:ssZ"
+    }
+  ]`;
+const ADD_DATASET_EXAMPLES_DESCRIPTION = `Add examples to an existing dataset.
+
+This tool adds one or more examples to an existing dataset. Each example includes an input,
+output, and metadata. The metadata will automatically include information indicating that
+these examples were synthetically generated via MCP. When calling this tool, check existing
+examples using the "get-dataset-examples" tool to ensure that you are not adding duplicate
+examples and following existing patterns for how data should be structured.
+
+Example usage:
+  Look at the analyze "my-dataset" and augment them with new examples to cover relevant edge cases
+
+Expected return:
+  Confirmation of successful addition of examples to the dataset.
+  Example: {
+    "dataset_name": "my-dataset",
+    "message": "Successfully added examples to dataset"
+  }`;
 export const initializeDatasetTools = ({ client, server, }) => {
-    server.tool("list-datasets", "Get a list of all the datasets", {
+    server.tool("list-datasets", LIST_DATASETS_DESCRIPTION, {
         limit: z.number().min(1).max(100).default(100),
     }, async ({ limit }) => {
         const response = await client.GET("/v1/datasets", {
@@ -17,7 +102,7 @@ export const initializeDatasetTools = ({ client, server, }) => {
             ],
         };
     });
-    server.tool("get-dataset-examples", "Get examples from a dataset", {
+    server.tool("get-dataset-examples", GET_DATASET_EXAMPLES_DESCRIPTION, {
         datasetId: z.string(),
     }, async ({ datasetId }) => {
         const response = await client.GET("/v1/datasets/{id}/examples", {
@@ -34,7 +119,7 @@ export const initializeDatasetTools = ({ client, server, }) => {
             ],
         };
     });
-    server.tool("get-dataset-experiments", "List experiments run on a dataset", {
+    server.tool("get-dataset-experiments", GET_DATASET_EXPERIMENTS_DESCRIPTION, {
         datasetId: z.string(),
     }, async ({ datasetId }) => {
         const response = await client.GET("/v1/datasets/{dataset_id}/experiments", {
@@ -51,4 +136,50 @@ export const initializeDatasetTools = ({ client, server, }) => {
             ],
         };
     });
+    server.tool("add-dataset-examples", ADD_DATASET_EXAMPLES_DESCRIPTION, {
+        datasetName: z.string(),
+        examples: z.array(z.object({
+            input: z.record(z.any()),
+            output: z.record(z.any()),
+            metadata: z.record(z.any()).optional(),
+        })),
+    }, async ({ datasetName, examples }) => {
+        // Add MCP metadata to each example
+        const examplesWithMetadata = examples.map((example) => ({
+            ...example,
+            metadata: {
+                ...example.metadata,
+                source: "Synthetic Example added via MCP",
+            },
+        }));
+        const response = await client.POST("/v1/datasets/upload", {
+            body: {
+                action: "append",
+                name: datasetName,
+                inputs: examplesWithMetadata.map((e) => e.input),
+                outputs: examplesWithMetadata.map((e) => e.output),
+                metadata: examplesWithMetadata.map((e) => e.metadata),
+            },
+            params: {
+                query: {
+                    sync: true,
+                },
+            },
+        });
+        if (!response.data?.data?.dataset_id) {
+            throw new Error("Failed to add examples to dataset: No dataset ID received");
+        }
+        return {
+            content: [
+                {
+                    type: "text",
+                    text: JSON.stringify({
+                        dataset_name: datasetName,
+                        dataset_id: response.data.data.dataset_id,
+                        message: "Successfully added examples to dataset",
+                    }, null, 2),
+                },
+            ],
+        };
+    });
 };

package/build/experimentTools.js

@@ -1,6 +1,84 @@
 import z from "zod";
+const LIST_EXPERIMENTS_DESCRIPTION = `Get a list of all the experiments run on a given dataset.
+
+Experiments are collections of experiment runs, each experiment run corresponds to a single 
+dataset example. The dataset example is passed to an implied \`task\` which in turn 
+produces an output.
+
+Example usage:
+  Show me all the experiments I've run on dataset RGF0YXNldDox
+
+Expected return:
+  Array of experiment objects with metadata. 
+  Example: [
+    {
+      "id": "experimentid1234",
+      "dataset_id": "datasetid1234",
+      "dataset_version_id": "datasetversionid1234",
+      "repetitions": 1,
+      "metadata": {},
+      "project_name": "Experiment-abc123",
+      "created_at": "YYYY-MM-DDTHH:mm:ssZ",
+      "updated_at": "YYYY-MM-DDTHH:mm:ssZ"
+    }
+  ]`;
+const GET_EXPERIMENT_DESCRIPTION = `Get an experiment by its ID.
+
+The tool returns experiment metadata in the first content block and a JSON object with the 
+experiment data in the second. The experiment data contains both the results of each 
+experiment run and the annotations made by an evaluator to score or label the results, 
+for example, comparing the output of an experiment run to the expected output from the 
+dataset example.
+
+Example usage:
+  Show me the experiment results for experiment RXhwZXJpbWVudDo4
+
+Expected return:
+  Object containing experiment metadata and results.
+  Example: {
+    "metadata": {
+      "id": "experimentid1234",
+      "dataset_id": "datasetid1234",
+      "dataset_version_id": "datasetversionid1234",
+      "repetitions": 1,
+      "metadata": {},
+      "project_name": "Experiment-abc123",
+      "created_at": "YYYY-MM-DDTHH:mm:ssZ",
+      "updated_at": "YYYY-MM-DDTHH:mm:ssZ"
+    },
+    "experimentResult": [
+      {
+        "example_id": "exampleid1234",
+        "repetition_number": 0,
+        "input": "Sample input text",
+        "reference_output": "Expected output text",
+        "output": "Actual output text",
+        "error": null,
+        "latency_ms": 1000,
+        "start_time": "2025-03-20T12:00:00Z",
+        "end_time": "2025-03-20T12:00:01Z",
+        "trace_id": "trace-123",
+        "prompt_token_count": 10,
+        "completion_token_count": 20,
+        "annotations": [
+          {
+            "name": "quality",
+            "annotator_kind": "HUMAN",
+            "label": "good",
+            "score": 0.9,
+            "explanation": "Output matches expected format",
+            "trace_id": "trace-456",
+            "error": null,
+            "metadata": {},
+            "start_time": "YYYY-MM-DDTHH:mm:ssZ",
+            "end_time": "YYYY-MM-DDTHH:mm:ssZ"
+          }
+        ]
+      }
+    ]
+  }`;
 export const initializeExperimentTools = ({ client, server, }) => {
-    server.tool("list-experiments-for-dataset", "Get a list of all the experiments for a given dataset", {
+    server.tool("list-experiments-for-dataset", LIST_EXPERIMENTS_DESCRIPTION, {
         dataset_id: z.string(),
     }, async ({ dataset_id }) => {
         const response = await client.GET("/v1/datasets/{dataset_id}/experiments", {
@@ -16,7 +94,7 @@ export const initializeExperimentTools = ({ client, server, }) => {
             ],
         };
     });
-    server.tool("get-experiment-by-id", "Get an experiment by its ID. The tool returns experiment metadata in the first content block and a JSON object with the experiment data in the second.", {
+    server.tool("get-experiment-by-id", GET_EXPERIMENT_DESCRIPTION, {
         experiment_id: z.string(),
     }, async ({ experiment_id }) => {
         const [experimentMetadataResponse, experimentDataResponse] = await Promise.all([

package/build/promptSchemas.js

@@ -41,3 +41,20 @@ export const updatePromptSchema = z.object({
 export const deletePromptSchema = z.object({
     prompt_identifier: z.string(),
 });
+export const listPromptVersionsSchema = z.object({
+    prompt_identifier: z.string(),
+    limit: z.number().min(1).max(100).default(100),
+});
+export const getPromptVersionByTagSchema = z.object({
+    prompt_identifier: z.string(),
+    tag_name: z.string(),
+});
+export const listPromptVersionTagsSchema = z.object({
+    prompt_version_id: z.string(),
+    limit: z.number().min(1).max(100).default(100),
+});
+export const addPromptVersionTagSchema = z.object({
+    prompt_version_id: z.string(),
+    name: z.string(),
+    description: z.string().optional(),
+});

package/build/promptTools.js

@@ -1,7 +1,222 @@
 import { createPrompt, promptVersion } from "@arizeai/phoenix-client/prompts";
-import { listPromptsSchema, getLatestPromptSchema, getPromptByIdentifierSchema, getPromptVersionSchema, createPromptSchema, } from "./promptSchemas.js";
+import { listPromptsSchema, getLatestPromptSchema, getPromptByIdentifierSchema, getPromptVersionSchema, createPromptSchema, listPromptVersionsSchema, getPromptVersionByTagSchema, listPromptVersionTagsSchema, addPromptVersionTagSchema, } from "./promptSchemas.js";
+// Tool descriptions as template literals for better readability
+const LIST_PROMPTS_DESCRIPTION = `Get a list of all the prompts.
+
+Prompts (templates, prompt templates) are versioned templates for input messages to an LLM.
+Each prompt includes both the input messages, but also the model and invocation parameters
+to use when generating outputs.
+
+Returns a list of prompt objects with their IDs, names, and descriptions.
+
+Example usage: 
+  List all available prompts
+
+Expected return: 
+  Array of prompt objects with metadata. 
+  Example:  [{
+      "name": "article-summarizer",
+      "description": "Summarizes an article into concise bullet points",
+      "source_prompt_id": null,
+      "id": "promptid1234"
+  }]`;
+const GET_LATEST_PROMPT_DESCRIPTION = `Get the latest version of a prompt. Returns the prompt version with its template, model configuration, and invocation parameters.
+
+Example usage: 
+  Get the latest version of a prompt named 'article-summarizer'
+
+Expected return: 
+  Prompt version object with template and configuration. 
+  Example: {
+    "description": "Initial version",
+    "model_provider": "OPENAI",
+    "model_name": "gpt-3.5-turbo",
+    "template": {
+      "type": "chat",
+      "messages": [
+        {
+          "role": "system",
+          "content": "You are an expert summarizer. Create clear, concise bullet points highlighting the key information."
+        },
+        {
+          "role": "user",
+          "content": "Please summarize the following {{topic}} article:\n\n{{article}}"
+        }
+      ]
+    },
+    "template_type": "CHAT",
+    "template_format": "MUSTACHE",
+    "invocation_parameters": {
+      "type": "openai",
+      "openai": {}
+    },
+    "id": "promptversionid1234"
+  }`;
+const GET_PROMPT_BY_IDENTIFIER_DESCRIPTION = `Get a prompt's latest version by its identifier (name or ID). Returns the prompt version with its template, model configuration, and invocation parameters.
+
+Example usage: 
+  Get the latest version of a prompt with name 'article-summarizer'
+
+Expected return: 
+  Prompt version object with template and configuration. 
+    Example: {
+      "description": "Initial version",
+      "model_provider": "OPENAI",
+      "model_name": "gpt-3.5-turbo",
+      "template": {
+        "type": "chat",
+        "messages": [
+          {
+            "role": "system",
+            "content": "You are an expert summarizer. Create clear, concise bullet points highlighting the key information."
+          },
+          {
+            "role": "user",
+            "content": "Please summarize the following {{topic}} article:\n\n{{article}}"
+          }
+        ]
+      },
+      "template_type": "CHAT",
+      "template_format": "MUSTACHE",
+      "invocation_parameters": {
+        "type": "openai",
+        "openai": {}
+      },
+      "id": "promptversionid1234"
+    }`;
+const GET_PROMPT_VERSION_DESCRIPTION = `Get a specific version of a prompt using its version ID. Returns the prompt version with its template, model configuration, and invocation parameters.
+
+Example usage: 
+  Get a specific prompt version with ID 'promptversionid1234'
+
+Expected return: 
+  Prompt version object with template and configuration. 
+  Example: {
+    "description": "Initial version",
+    "model_provider": "OPENAI",
+    "model_name": "gpt-3.5-turbo",
+    "template": {
+      "type": "chat",
+      "messages": [
+        {
+          "role": "system",
+          "content": "You are an expert summarizer. Create clear, concise bullet points highlighting the key information."
+        },
+        {
+          "role": "user",
+          "content": "Please summarize the following {{topic}} article:\n\n{{article}}"
+        }
+      ]
+    },
+    "template_type": "CHAT",
+    "template_format": "MUSTACHE",
+    "invocation_parameters": {
+      "type": "openai",
+      "openai": {}
+    },
+    "id": "promptversionid1234"
+  }`;
+const UPSERT_PROMPT_DESCRIPTION = `Create or update a prompt with its template and configuration. Creates a new prompt and its initial version with specified model settings.
+
+Example usage: 
+  Create a new prompt named 'email_generator' with a template for generating emails
+
+Expected return: 
+  A confirmation message of successful prompt creation`;
+const LIST_PROMPT_VERSIONS_DESCRIPTION = `Get a list of all versions for a specific prompt. Returns versions with pagination support.
+
+Example usage: 
+  List all versions of a prompt named 'article-summarizer'
+
+Expected return: 
+  Array of prompt version objects with IDs and configuration. 
+  Example: [
+    {
+      "description": "Initial version",
+      "model_provider": "OPENAI",
+      "model_name": "gpt-3.5-turbo",
+      "template": {
+        "type": "chat",
+        "messages": [
+          {
+            "role": "system",
+            "content": "You are an expert summarizer. Create clear, concise bullet points highlighting the key information."
+          },
+          {
+            "role": "user",
+            "content": "Please summarize the following {{topic}} article:\n\n{{article}}"
+          }
+        ]
+      },
+      "template_type": "CHAT",
+      "template_format": "MUSTACHE",
+      "invocation_parameters": {
+        "type": "openai",
+        "openai": {}
+      },
+      "id": "promptversionid1234"
+    }
+  ]`;
+const GET_PROMPT_VERSION_BY_TAG_DESCRIPTION = `Get a prompt version by its tag name. Returns the prompt version with its template, model configuration, and invocation parameters.
+
+Example usage: 
+  Get the 'production' tagged version of prompt 'article-summarizer'
+
+Expected return: 
+  Prompt version object with template and configuration. 
+  Example: {
+      "description": "Initial version",
+      "model_provider": "OPENAI",
+      "model_name": "gpt-3.5-turbo",
+      "template": {
+        "type": "chat",
+        "messages": [
+          {
+            "role": "system",
+            "content": "You are an expert summarizer. Create clear, concise bullet points highlighting the key information."
+          },
+          {
+            "role": "user",
+            "content": "Please summarize the following {{topic}} article:\n\n{{article}}"
+          }
+        ]
+      },
+      "template_type": "CHAT",
+      "template_format": "MUSTACHE",
+      "invocation_parameters": {
+        "type": "openai",
+        "openai": {}
+      },
+      "id": "promptversionid1234"
+    }`;
+const LIST_PROMPT_VERSION_TAGS_DESCRIPTION = `Get a list of all tags for a specific prompt version. Returns tag objects with pagination support.
+
+Example usage: 
+  List all tags associated with prompt version 'promptversionid1234'
+
+Expected return: 
+  Array of tag objects with names and IDs. 
+  Example: [
+    {
+      "name": "staging",
+      "description": "The version deployed to staging",
+      "id": "promptversionid1234"
+    },
+    {
+      "name": "development",
+      "description": "The version deployed for development",
+      "id": "promptversionid1234"
+    }
+  ]`;
+const ADD_PROMPT_VERSION_TAG_DESCRIPTION = `Add a tag to a specific prompt version. The operation returns no content on success (204 status code).
+
+Example usage: 
+  Tag prompt version 'promptversionid1234' with the name 'production'
+
+Expected return: 
+  Confirmation message of successful tag addition`;
 export const initializePromptTools = ({ client, server, }) => {
-    server.tool("list-prompts", "Get a list of all the prompts", listPromptsSchema.shape, async ({ limit }) => {
+    server.tool("list-prompts", LIST_PROMPTS_DESCRIPTION, listPromptsSchema.shape, async ({ limit }) => {
         const response = await client.GET("/v1/prompts", {
             params: {
                 query: {
@@ -18,7 +233,7 @@ export const initializePromptTools = ({ client, server, }) => {
             ],
         };
     });
-    server.tool("get-latest-prompt", "Get the latest prompt", getLatestPromptSchema.shape, async ({ prompt_identifier }) => {
+    server.tool("get-latest-prompt", GET_LATEST_PROMPT_DESCRIPTION, getLatestPromptSchema.shape, async ({ prompt_identifier }) => {
         const response = await client.GET("/v1/prompts/{prompt_identifier}/latest", {
             params: {
                 path: {
@@ -35,7 +250,7 @@ export const initializePromptTools = ({ client, server, }) => {
             ],
         };
     });
-    server.tool("get-prompt-by-identifier", "Get a prompt's latest version by its identifier", getPromptByIdentifierSchema.shape, async ({ prompt_identifier }) => {
+    server.tool("get-prompt-by-identifier", GET_PROMPT_BY_IDENTIFIER_DESCRIPTION, getPromptByIdentifierSchema.shape, async ({ prompt_identifier }) => {
         const response = await client.GET("/v1/prompts/{prompt_identifier}/latest", {
             params: {
                 path: {
@@ -52,7 +267,7 @@ export const initializePromptTools = ({ client, server, }) => {
             ],
         };
     });
-    server.tool("get-prompt-version", "Get a specific version of a prompt given a prompt version id", getPromptVersionSchema.shape, async ({ prompt_version_id }) => {
+    server.tool("get-prompt-version", GET_PROMPT_VERSION_DESCRIPTION, getPromptVersionSchema.shape, async ({ prompt_version_id }) => {
         const response = await client.GET("/v1/prompt_versions/{prompt_version_id}", {
             params: {
                 path: {
@@ -69,7 +284,7 @@ export const initializePromptTools = ({ client, server, }) => {
             ],
         };
     });
-    server.tool("upsert-prompt", "Create or update a prompt", createPromptSchema.shape, async ({ name, description, template, model_provider, model_name, temperature, }) => {
+    server.tool("upsert-prompt", UPSERT_PROMPT_DESCRIPTION, createPromptSchema.shape, async ({ name, description, template, model_provider, model_name, temperature, }) => {
         let promptVersionData;
         switch (model_provider) {
             case "OPENAI":
@@ -153,27 +368,83 @@ export const initializePromptTools = ({ client, server, }) => {
             ],
         };
     });
+    server.tool("list-prompt-versions", LIST_PROMPT_VERSIONS_DESCRIPTION, listPromptVersionsSchema.shape, async ({ prompt_identifier, limit }) => {
+        const response = await client.GET("/v1/prompts/{prompt_identifier}/versions", {
+            params: {
+                path: {
+                    prompt_identifier,
+                },
+                query: {
+                    limit,
+                },
+            },
+        });
+        return {
+            content: [
+                {
+                    type: "text",
+                    text: JSON.stringify(response.data, null, 2),
+                },
+            ],
+        };
+    });
+    server.tool("get-prompt-version-by-tag", GET_PROMPT_VERSION_BY_TAG_DESCRIPTION, getPromptVersionByTagSchema.shape, async ({ prompt_identifier, tag_name }) => {
+        const response = await client.GET("/v1/prompts/{prompt_identifier}/tags/{tag_name}", {
+            params: {
+                path: {
+                    prompt_identifier,
+                    tag_name,
+                },
+            },
+        });
+        return {
+            content: [
+                {
+                    type: "text",
+                    text: JSON.stringify(response.data, null, 2),
+                },
+            ],
+        };
+    });
+    server.tool("list-prompt-version-tags", LIST_PROMPT_VERSION_TAGS_DESCRIPTION, listPromptVersionTagsSchema.shape, async ({ prompt_version_id, limit }) => {
+        const response = await client.GET("/v1/prompt_versions/{prompt_version_id}/tags", {
+            params: {
+                path: {
+                    prompt_version_id,
+                },
+                query: {
+                    limit,
+                },
+            },
+        });
+        return {
+            content: [
+                {
+                    type: "text",
+                    text: JSON.stringify(response.data, null, 2),
+                },
+            ],
+        };
+    });
+    server.tool("add-prompt-version-tag", ADD_PROMPT_VERSION_TAG_DESCRIPTION, addPromptVersionTagSchema.shape, async ({ prompt_version_id, name, description }) => {
+        await client.POST("/v1/prompt_versions/{prompt_version_id}/tags", {
+            params: {
+                path: {
+                    prompt_version_id,
+                },
+            },
+            body: {
+                name,
+                description,
+            },
+        });
+        return {
+            content: [
+                {
+                    type: "text",
+                    text: `Successfully added tag "${name}" to prompt version ${prompt_version_id}`,
+                },
+            ],
+        };
+    });
 };
-//   server.tool(
-//     "delete-prompt",
-//     "Delete a prompt",
-//     deletePromptSchema.shape,
-//     async ({ prompt_identifier }) => {
-//       const response = await client.DELETE("/v1/prompts/{prompt_identifier}", {
-//         params: {
-//           path: {
-//             prompt_identifier,
-//           },
-//         },
-//       });
-//       return {
-//         content: [
-//           {
-//             type: "text",
-//             text: "Prompt deleted successfully",
-//           },
-//         ],
-//       };
-//     }
-//   );
-// };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arizeai/phoenix-mcp",
-  "version": "2.1.1",
+  "version": "2.1.2",
   "description": "A MCP server for Arize Phoenix",
   "bin": {
     "@arizeai/phoenix-mcp": "./build/index.js"