opik-mcp 0.0.1

package/build/utils/tracing-info.js

@@ -0,0 +1,119 @@
+/**
+ * Tracing information module for Opik Comet
+ * Provides detailed information about Opik's tracing capabilities
+ */
+/**
+ * Get information about Opik's tracing capabilities
+ * @param topic Optional topic to get specific information about
+ * @returns Information about the specified topic or an overview if no topic is provided
+ */
+export function getTracingInfo(topic) {
+    // If no topic is provided, return an overview
+    if (!topic) {
+        return {
+            title: 'Opik Tracing Capabilities',
+            description: 'Opik provides comprehensive tracing capabilities to help you understand and analyze your LLM applications. Traces capture the full context of LLM interactions, including inputs, outputs, and metadata.',
+            availableTopics: ['traces', 'spans', 'feedback', 'search', 'visualization'],
+        };
+    }
+    // Return information based on the requested topic
+    switch (topic.toLowerCase()) {
+        case 'traces':
+            return {
+                title: 'Traces',
+                description: 'Traces are complete records of LLM interactions, capturing the full context of a request including inputs, outputs, and metadata. They provide a comprehensive view of how your LLM application is performing.',
+                keyFeatures: [
+                    'Automatic capture of inputs and outputs',
+                    'Metadata collection for context',
+                    'Hierarchical structure with spans',
+                    'Long-term storage and retrieval',
+                    'Integration with evaluation metrics',
+                ],
+                useCases: [
+                    'Debugging LLM application issues',
+                    'Auditing LLM responses',
+                    'Compliance and governance',
+                    'Performance analysis',
+                    'Training data collection',
+                ],
+            };
+        case 'spans':
+            return {
+                title: 'Spans',
+                description: 'Spans are individual units within a trace that represent discrete operations or steps in your LLM application. They help break down complex interactions into manageable pieces for analysis.',
+                keyFeatures: [
+                    'Hierarchical relationship (parent-child)',
+                    'Timing information for performance analysis',
+                    'Custom attributes for context',
+                    'Support for nested operations',
+                    'Automatic correlation with parent traces',
+                ],
+                useCases: [
+                    'Performance bottleneck identification',
+                    'Detailed step-by-step analysis',
+                    'Tracking complex multi-step LLM workflows',
+                    'Measuring time spent in different components',
+                    'Correlating errors with specific operations',
+                ],
+            };
+        case 'feedback':
+            return {
+                title: 'Feedback',
+                description: 'Feedback allows you to annotate traces with human or automated evaluations. This helps you build a dataset of evaluated responses that can be used for model improvement and analysis.',
+                keyFeatures: [
+                    'Binary (thumbs up/down) feedback',
+                    'Structured feedback with categories',
+                    'Free-form text comments',
+                    'Support for multiple feedback sources',
+                    'Integration with evaluation metrics',
+                ],
+                useCases: [
+                    'Building labeled datasets for fine-tuning',
+                    'Identifying patterns in problematic responses',
+                    'Measuring user satisfaction',
+                    'Comparing model versions',
+                    'Creating ground truth for automated evaluations',
+                ],
+            };
+        case 'search':
+            return {
+                title: 'Search',
+                description: 'Opik provides powerful search capabilities to help you find specific traces based on content, metadata, or performance characteristics. This makes it easy to analyze patterns and identify issues.',
+                keyFeatures: [
+                    'Full-text search across inputs and outputs',
+                    'Metadata filtering',
+                    'Time-based queries',
+                    'Performance metric filtering',
+                    'Support for complex boolean queries',
+                ],
+                useCases: [
+                    'Finding examples of specific behaviors',
+                    'Identifying patterns in errors',
+                    'Analyzing performance trends',
+                    'Auditing responses for compliance',
+                    'Building targeted datasets for evaluation',
+                ],
+            };
+        case 'visualization':
+            return {
+                title: 'Visualization',
+                description: 'Opik provides visualization tools to help you understand your traces and spans. These visualizations make it easier to identify patterns, bottlenecks, and issues in your LLM applications.',
+                keyFeatures: [
+                    'Timeline views of traces and spans',
+                    'Performance dashboards',
+                    'Metric trend analysis',
+                    'Comparison views for A/B testing',
+                    'Custom visualization options',
+                ],
+                useCases: [
+                    'Performance monitoring over time',
+                    'Comparing model versions',
+                    'Identifying usage patterns',
+                    'Tracking evaluation metrics',
+                    'Presenting insights to stakeholders',
+                ],
+            };
+        default:
+            return null;
+    }
+}

package/package.json

@@ -0,0 +1,79 @@
+{
+  "name": "opik-mcp",
+  "version": "0.0.1",
+  "description": "MCP server to interact with Opik - Enables automated prompt optimization",
+  "type": "module",
+  "bin": {
+    "opik-mcp": "build/index.js"
+  },
+  "scripts": {
+    "build": "tsc && shx chmod +x build/*.js && shx cp -r client build/",
+    "test": "node --experimental-vm-modules node_modules/jest/bin/jest.js",
+    "test:transport": "node --experimental-vm-modules node_modules/jest/bin/jest.js --forceExit --detectOpenHandles tests/transports",
+    "api:build": "tsc",
+    "api:test": "node build/test-client.js",
+    "simple": "node build/simple-mcp.js",
+    "lint": "eslint . --ext .ts",
+    "lint:fix": "eslint . --ext .ts --fix",
+    "format": "prettier --write \"src/**/*.ts\" \"tests/**/*.ts\"",
+    "check": "npm run lint && npm run test",
+    "start": "node build/index.js",
+    "start:sse": "node build/cli.js serve --transport sse",
+    "start:stdio": "node build/cli.js serve --transport stdio",
+    "dev:sse": "tsc && node build/cli.js serve --transport sse"
+  },
+  "files": [
+    "build"
+  ],
+  "author": "Opik",
+  "license": "Apache-2.0",
+  "dependencies": {
+    "dotenv": "^16.4.5",
+    "yargs": "^17.7.2",
+    "zod": "^3.24.2"
+  },
+  "devDependencies": {
+    "@jest/globals": "^29.7.0",
+    "@modelcontextprotocol/sdk": "^1.6.1",
+    "@types/cors": "^2.8.17",
+    "@types/express": "^5.0.0",
+    "@types/jest": "^29.5.12",
+    "@types/node": "^22.13.9",
+    "@typescript-eslint/eslint-plugin": "^7.4.0",
+    "@typescript-eslint/parser": "^7.4.0",
+    "cors": "^2.8.5",
+    "eslint": "^8.57.0",
+    "eslint-config-prettier": "^9.1.0",
+    "eslint-plugin-prettier": "^5.1.3",
+    "express": "^5.0.1",
+    "jest": "^29.7.0",
+    "jest-environment-jsdom": "^29.7.0",
+    "node-fetch": "^3.3.2",
+    "pre-commit": "^1.2.2",
+    "prettier": "^3.2.5",
+    "shx": "^0.3.4",
+    "ts-jest": "^29.1.2",
+    "typescript": "^5.8.2"
+  },
+  "jest": {
+    "preset": "ts-jest/presets/default-esm",
+    "extensionsToTreatAsEsm": [
+      ".ts"
+    ],
+    "transform": {
+      "^.+\\.tsx?$": [
+        "ts-jest",
+        {
+          "useESM": true
+        }
+      ]
+    },
+    "moduleNameMapper": {
+      "^(\\.{1,2}/.*)\\.js$": "$1"
+    }
+  },
+  "pre-commit": [
+    "lint",
+    "test"
+  ]
+}