opik-mcp 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/build/index.js ADDED
@@ -0,0 +1,1847 @@
1
+ import fs from 'fs';
2
+ // Import other modules
3
+ import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
4
+ import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
5
+ import { z } from 'zod';
6
+ // Import custom transports
7
+ import { SSEServerTransport } from './transports/sse-transport.js';
8
+ // Import environment variables loader - no console output
9
+ import './utils/env.js';
10
+ // Setup file-based logging
11
+ const logFile = '/tmp/opik-mcp.log';
12
+ // Import configuration
13
+ import configImport from './config.js';
14
+ const config = configImport;
15
+ // Define logging functions
16
+ function logToFile(message) {
17
+ // Only log if debug mode is enabled
18
+ if (!config?.debugMode)
19
+ return;
20
+ try {
21
+ const timestamp = new Date().toISOString();
22
+ fs.appendFileSync(logFile, `[${timestamp}] ${message}\n`);
23
+ }
24
+ catch (error) {
25
+ // Silently fail if we can't write to the log file
26
+ }
27
+ }
28
+ // Only initialize log file if debug mode is enabled
29
+ if (config.debugMode) {
30
+ try {
31
+ fs.writeFileSync(logFile, `Opik MCP Server Started: ${new Date().toISOString()}\n`);
32
+ // Log process info
33
+ logToFile(`Process ID: ${process.pid}, Node Version: ${process.version}`);
34
+ logToFile(`Arguments: ${process.argv.join(' ')}`);
35
+ logToFile(`Loaded configuration: API=${config.apiBaseUrl}, Workspace=${config.workspaceName || 'None'}`);
36
+ // Register error handlers
37
+ process.on('uncaughtException', err => {
38
+ logToFile(`UNCAUGHT EXCEPTION: ${err.message}`);
39
+ logToFile(err.stack || 'No stack trace');
40
+ });
41
+ process.on('unhandledRejection', reason => {
42
+ logToFile(`UNHANDLED REJECTION: ${reason}`);
43
+ });
44
+ process.on('exit', code => {
45
+ logToFile(`Process exiting with code ${code}`);
46
+ });
47
+ }
48
+ catch (error) {
49
+ // Silently fail if we can't write to the log file
50
+ }
51
+ }
52
+ // Import capabilities module
53
+ import { getEnabledCapabilities, getCapabilitiesDescription } from './utils/capabilities.js';
54
+ // Helper function to make requests to API with file logging
55
+ const makeApiRequest = async (path, options = {}, workspaceName) => {
56
+ // Prepare headers based on configuration
57
+ const API_HEADERS = {
58
+ Accept: 'application/json',
59
+ 'Content-Type': 'application/json',
60
+ authorization: config.apiKey,
61
+ };
62
+ // Add workspace header for cloud version
63
+ if (!config.isSelfHosted) {
64
+ // Use provided workspace name or fall back to config
65
+ const wsName = workspaceName || config.workspaceName;
66
+ if (wsName) {
67
+ // Note: The Opik API expects the workspace name to be the default workspace.
68
+ // Project names like "Therapist Chat" are not valid workspace names.
69
+ // The API will return a 400 error if a non-existent workspace is specified.
70
+ const workspaceNameToUse = wsName.trim();
71
+ logToFile(`DEBUG - Workspace name before setting header: "${workspaceNameToUse}", type: ${typeof workspaceNameToUse}, length: ${workspaceNameToUse.length}`);
72
+ // Use the raw workspace name - do not encode it
73
+ API_HEADERS['Comet-Workspace'] = workspaceNameToUse;
74
+ logToFile(`Using workspace: ${workspaceNameToUse}`);
75
+ }
76
+ }
77
+ const url = `${config.apiBaseUrl}${path}`;
78
+ logToFile(`Making API request to: ${url}`);
79
+ logToFile(`Headers: ${JSON.stringify(API_HEADERS, null, 2)}`);
80
+ try {
81
+ const response = await fetch(url, {
82
+ ...options,
83
+ headers: {
84
+ ...API_HEADERS,
85
+ ...options.headers,
86
+ },
87
+ });
88
+ // Get response body text for better error handling
89
+ const responseText = await response.text();
90
+ let responseData = null;
91
+ // Try to parse the response as JSON
92
+ try {
93
+ responseData = JSON.parse(responseText);
94
+ }
95
+ catch (e) {
96
+ // If it's not valid JSON, use the raw text
97
+ responseData = responseText;
98
+ }
99
+ if (!response.ok) {
100
+ const errorMsg = `HTTP error! status: ${response.status} ${JSON.stringify(responseData)}`;
101
+ logToFile(`API Error: ${errorMsg}`);
102
+ return {
103
+ data: null,
104
+ error: errorMsg,
105
+ };
106
+ }
107
+ return {
108
+ data: responseData,
109
+ error: null,
110
+ };
111
+ }
112
+ catch (error) {
113
+ const errorMessage = error instanceof Error ? error.message : 'Unknown error occurred';
114
+ logToFile(`Error making API request: ${errorMessage}`);
115
+ return {
116
+ data: null,
117
+ error: errorMessage,
118
+ };
119
+ }
120
+ };
121
+ // Create and configure server - no console output here
122
+ const server = new McpServer({
123
+ name: config.mcpName,
124
+ version: config.mcpVersion,
125
+ }, {
126
+ capabilities: {
127
+ resources: {}, // Enable resources capability
128
+ tools: {}, // Enable tools capability
129
+ },
130
+ });
131
+ // Add resources to the MCP server
132
+ if (config.workspaceName) {
133
+ // Define a workspace info resource
134
+ server.resource('workspace-info', 'opik://workspace-info', async () => ({
135
+ contents: [
136
+ {
137
+ uri: 'opik://workspace-info',
138
+ text: JSON.stringify({
139
+ name: config.workspaceName,
140
+ apiUrl: config.apiBaseUrl,
141
+ selfHosted: config.isSelfHosted,
142
+ }, null, 2),
143
+ },
144
+ ],
145
+ }));
146
+ // Define a projects resource that provides the list of projects in the workspace
147
+ server.resource('projects-list', 'opik://projects-list', async () => {
148
+ try {
149
+ const response = await makeApiRequest('/v1/private/projects');
150
+ if (!response.data) {
151
+ return {
152
+ contents: [
153
+ {
154
+ uri: 'opik://projects-list',
155
+ text: `Error: ${response.error || 'Unknown error fetching projects'}`,
156
+ },
157
+ ],
158
+ };
159
+ }
160
+ return {
161
+ contents: [
162
+ {
163
+ uri: 'opik://projects-list',
164
+ text: JSON.stringify(response.data, null, 2),
165
+ },
166
+ ],
167
+ };
168
+ }
169
+ catch (error) {
170
+ logToFile(`Error fetching projects resource: ${error}`);
171
+ return {
172
+ contents: [
173
+ {
174
+ uri: 'opik://projects-list',
175
+ text: `Error: Failed to fetch projects data`,
176
+ },
177
+ ],
178
+ };
179
+ }
180
+ });
181
+ }
182
+ // DO NOT send any protocol messages before server initialization
183
+ // REMOVED: sendProtocolMessage("log", "Initializing Opik MCP Server");
184
+ // Conditionally enable tool categories based on configuration
185
+ if (config.mcpEnablePromptTools) {
186
+ // ----------- PROMPTS TOOLS -----------
187
+ server.tool('list-prompts', 'Get a list of Opik prompts', {
188
+ page: z.number().describe('Page number for pagination'),
189
+ size: z.number().describe('Number of items per page'),
190
+ }, async (args) => {
191
+ const response = await makeApiRequest(`/v1/private/prompts?page=${args.page}&size=${args.size}`);
192
+ if (!response.data) {
193
+ return {
194
+ content: [{ type: 'text', text: response.error || 'Failed to fetch prompts' }],
195
+ };
196
+ }
197
+ return {
198
+ content: [
199
+ {
200
+ type: 'text',
201
+ text: `Found ${response.data.total} prompts (showing page ${response.data.page} of ${Math.ceil(response.data.total / response.data.size)})`,
202
+ },
203
+ {
204
+ type: 'text',
205
+ text: JSON.stringify(response.data.content, null, 2),
206
+ },
207
+ ],
208
+ };
209
+ });
210
+ server.tool('create-prompt', 'Create a new prompt', {
211
+ name: z.string().describe('Name of the prompt'),
212
+ }, async (args) => {
213
+ const { name } = args;
214
+ const response = await makeApiRequest(`/v1/private/prompts`, {
215
+ method: 'POST',
216
+ body: JSON.stringify({ name }),
217
+ });
218
+ return {
219
+ content: [
220
+ {
221
+ type: 'text',
222
+ text: response.error || 'Successfully created prompt',
223
+ },
224
+ ],
225
+ };
226
+ });
227
+ server.tool('create-prompt-version', 'Create a new version of a prompt', {
228
+ name: z.string().describe('Name of the original prompt'),
229
+ template: z.string().describe('Template content for the prompt version'),
230
+ commit_message: z.string().describe('Commit message for the prompt version'),
231
+ }, async (args) => {
232
+ const { name, template, commit_message } = args;
233
+ const response = await makeApiRequest(`/v1/private/prompts/versions`, {
234
+ method: 'POST',
235
+ body: JSON.stringify({
236
+ name,
237
+ version: { template, change_description: commit_message },
238
+ }),
239
+ });
240
+ return {
241
+ content: [
242
+ {
243
+ type: 'text',
244
+ text: response.data
245
+ ? 'Successfully created prompt version'
246
+ : `${response.error} ${JSON.stringify(args)}` || 'Failed to create prompt version',
247
+ },
248
+ ],
249
+ };
250
+ });
251
+ server.tool('get-prompt-by-id', 'Get a single prompt by ID', {
252
+ promptId: z.string().describe('ID of the prompt to fetch'),
253
+ }, async (args) => {
254
+ const { promptId } = args;
255
+ const response = await makeApiRequest(`/v1/private/prompts/${promptId}`);
256
+ if (!response.data) {
257
+ return {
258
+ content: [{ type: 'text', text: response.error || 'Failed to fetch prompt' }],
259
+ };
260
+ }
261
+ return {
262
+ content: [
263
+ {
264
+ type: 'text',
265
+ text: JSON.stringify(response.data, null, 2),
266
+ },
267
+ ],
268
+ };
269
+ });
270
+ server.tool('update-prompt', 'Update a prompt', {
271
+ promptId: z.string().describe('ID of the prompt to update'),
272
+ name: z.string().describe('New name for the prompt'),
273
+ }, async (args) => {
274
+ const { promptId, name } = args;
275
+ const response = await makeApiRequest(`/v1/private/prompts/${promptId}`, {
276
+ method: 'PUT',
277
+ body: JSON.stringify({ name }),
278
+ headers: {
279
+ 'Content-Type': 'application/json',
280
+ },
281
+ });
282
+ return {
283
+ content: [
284
+ {
285
+ type: 'text',
286
+ text: !response.error
287
+ ? 'Successfully updated prompt'
288
+ : response.error || 'Failed to update prompt',
289
+ },
290
+ ],
291
+ };
292
+ });
293
+ server.tool('delete-prompt', 'Delete a prompt', {
294
+ promptId: z.string().describe('ID of the prompt to delete'),
295
+ }, async (args) => {
296
+ const { promptId } = args;
297
+ const response = await makeApiRequest(`/v1/private/prompts/${promptId}`, {
298
+ method: 'DELETE',
299
+ });
300
+ return {
301
+ content: [
302
+ {
303
+ type: 'text',
304
+ text: !response.error
305
+ ? 'Successfully deleted prompt'
306
+ : response.error || 'Failed to delete prompt',
307
+ },
308
+ ],
309
+ };
310
+ });
311
+ }
312
+ // ----------- PROJECTS/WORKSPACES TOOLS -----------
313
+ if (config.mcpEnableProjectTools) {
314
+ server.tool('list-projects', 'Get a list of projects/workspaces', {
315
+ page: z.number().describe('Page number for pagination'),
316
+ size: z.number().describe('Number of items per page'),
317
+ sortBy: z.string().optional().describe('Sort projects by this field'),
318
+ sortOrder: z.string().optional().describe('Sort order (asc or desc)'),
319
+ workspaceName: z.string().optional().describe('Workspace name to use instead of the default'),
320
+ }, async (args) => {
321
+ const { page, size, sortBy, sortOrder, workspaceName } = args;
322
+ // Build query string
323
+ let url = `/v1/private/projects?page=${page}&size=${size}`;
324
+ if (sortBy)
325
+ url += `&sort_by=${sortBy}`;
326
+ if (sortOrder)
327
+ url += `&sort_order=${sortOrder}`;
328
+ const response = await makeApiRequest(url, {}, workspaceName);
329
+ if (!response.data) {
330
+ return {
331
+ content: [{ type: 'text', text: response.error || 'Failed to fetch projects' }],
332
+ };
333
+ }
334
+ return {
335
+ content: [
336
+ {
337
+ type: 'text',
338
+ text: `Found ${response.data.total} projects (showing page ${response.data.page} of ${Math.ceil(response.data.total / response.data.size)})`,
339
+ },
340
+ {
341
+ type: 'text',
342
+ text: JSON.stringify(response.data.content, null, 2),
343
+ },
344
+ ],
345
+ };
346
+ });
347
+ server.tool('get-project-by-id', 'Get a single project by ID', {
348
+ projectId: z.string().describe('ID of the project to fetch'),
349
+ workspaceName: z.string().optional().describe('Workspace name to use instead of the default'),
350
+ }, async (args) => {
351
+ const { projectId, workspaceName } = args;
352
+ const response = await makeApiRequest(`/v1/private/projects/${projectId}`, {}, workspaceName);
353
+ if (!response.data) {
354
+ return {
355
+ content: [{ type: 'text', text: response.error || 'Failed to fetch project' }],
356
+ };
357
+ }
358
+ return {
359
+ content: [
360
+ {
361
+ type: 'text',
362
+ text: JSON.stringify(response.data, null, 2),
363
+ },
364
+ ],
365
+ };
366
+ });
367
+ server.tool('create-project', 'Create a new project/workspace', {
368
+ name: z.string().describe('Name of the project'),
369
+ description: z.string().optional().describe('Description of the project'),
370
+ workspaceName: z.string().optional().describe('Workspace name to use instead of the default'),
371
+ }, async (args) => {
372
+ const { name, description, workspaceName } = args;
373
+ const response = await makeApiRequest(`/v1/private/projects`, {
374
+ method: 'POST',
375
+ body: JSON.stringify({ name, description }),
376
+ }, workspaceName);
377
+ return {
378
+ content: [
379
+ {
380
+ type: 'text',
381
+ text: response.error || 'Successfully created project',
382
+ },
383
+ ],
384
+ };
385
+ });
386
+ server.tool('update-project', 'Update a project', {
387
+ projectId: z.string().describe('ID of the project to update'),
388
+ name: z.string().optional().describe('New project name'),
389
+ workspaceName: z.string().optional().describe('Workspace name to use instead of the default'),
390
+ description: z.string().optional().describe('New project description'),
391
+ }, async (args) => {
392
+ const { projectId, name, description, workspaceName } = args;
393
+ // Build update data
394
+ const updateData = {};
395
+ if (name !== undefined)
396
+ updateData.name = name;
397
+ if (description !== undefined)
398
+ updateData.description = description;
399
+ const response = await makeApiRequest(`/v1/private/projects/${projectId}`, {
400
+ method: 'PATCH',
401
+ body: JSON.stringify(updateData),
402
+ }, workspaceName);
403
+ if (!response.data) {
404
+ return {
405
+ content: [{ type: 'text', text: response.error || 'Failed to update project' }],
406
+ };
407
+ }
408
+ return {
409
+ content: [
410
+ {
411
+ type: 'text',
412
+ text: 'Project successfully updated',
413
+ },
414
+ {
415
+ type: 'text',
416
+ text: JSON.stringify(response.data, null, 2),
417
+ },
418
+ ],
419
+ };
420
+ });
421
+ server.tool('delete-project', 'Delete a project', {
422
+ projectId: z.string().describe('ID of the project to delete'),
423
+ workspaceName: z.string().optional().describe('Workspace name to use instead of the default'),
424
+ }, async (args) => {
425
+ const { projectId, workspaceName } = args;
426
+ const response = await makeApiRequest(`/v1/private/projects/${projectId}`, {
427
+ method: 'DELETE',
428
+ }, workspaceName);
429
+ return {
430
+ content: [
431
+ {
432
+ type: 'text',
433
+ text: !response.error
434
+ ? 'Successfully deleted project'
435
+ : response.error || 'Failed to delete project',
436
+ },
437
+ ],
438
+ };
439
+ });
440
+ }
441
+ // ----------- TRACES TOOLS -----------
442
+ if (config.mcpEnableTraceTools) {
443
+ server.tool('list-traces', 'Get a list of traces', {
444
+ page: z.number().describe('Page number for pagination'),
445
+ size: z.number().describe('Number of items per page'),
446
+ projectId: z.string().optional().describe('Project ID to filter traces'),
447
+ projectName: z.string().optional().describe('Project name to filter traces'),
448
+ workspaceName: z.string().optional().describe('Workspace name to use instead of the default'),
449
+ }, async (args) => {
450
+ const { page, size, projectId, projectName, workspaceName } = args;
451
+ let url = `/v1/private/traces?page=${page}&size=${size}`;
452
+ // Add project filtering - API requires either project_id or project_name
453
+ if (projectId) {
454
+ url += `&project_id=${projectId}`;
455
+ }
456
+ else if (projectName) {
457
+ url += `&project_name=${encodeURIComponent(projectName)}`;
458
+ }
459
+ else {
460
+ // If no project specified, we need to find one for the API to work
461
+ const projectsResponse = await makeApiRequest(`/v1/private/projects?page=1&size=1`, {}, workspaceName);
462
+ if (projectsResponse.data &&
463
+ projectsResponse.data.content &&
464
+ projectsResponse.data.content.length > 0) {
465
+ const firstProject = projectsResponse.data.content[0];
466
+ url += `&project_id=${firstProject.id}`;
467
+ logToFile(`No project specified, using first available: ${firstProject.name} (${firstProject.id})`);
468
+ }
469
+ else {
470
+ return {
471
+ content: [
472
+ {
473
+ type: 'text',
474
+ text: 'Error: No project ID or name provided, and no projects found',
475
+ },
476
+ ],
477
+ };
478
+ }
479
+ }
480
+ const response = await makeApiRequest(url, {}, workspaceName);
481
+ if (!response.data) {
482
+ return {
483
+ content: [{ type: 'text', text: response.error || 'Failed to fetch traces' }],
484
+ };
485
+ }
486
+ return {
487
+ content: [
488
+ {
489
+ type: 'text',
490
+ text: `Found ${response.data.total} traces (showing page ${response.data.page} of ${Math.ceil(response.data.total / response.data.size)})`,
491
+ },
492
+ {
493
+ type: 'text',
494
+ text: JSON.stringify(response.data.content, null, 2),
495
+ },
496
+ ],
497
+ };
498
+ });
499
+ server.tool('get-trace-by-id', 'Get a single trace by ID', {
500
+ traceId: z.string().describe('ID of the trace to fetch'),
501
+ workspaceName: z.string().optional().describe('Workspace name to use instead of the default'),
502
+ }, async (args) => {
503
+ const { traceId, workspaceName } = args;
504
+ const response = await makeApiRequest(`/v1/private/traces/${traceId}`, {}, workspaceName);
505
+ if (!response.data) {
506
+ return {
507
+ content: [{ type: 'text', text: response.error || 'Failed to fetch trace' }],
508
+ };
509
+ }
510
+ // Format the response for better readability
511
+ const formattedResponse = { ...response.data };
512
+ // Format input/output if they're large
513
+ if (formattedResponse.input &&
514
+ typeof formattedResponse.input === 'object' &&
515
+ Object.keys(formattedResponse.input).length > 0) {
516
+ formattedResponse.input = JSON.stringify(formattedResponse.input, null, 2);
517
+ }
518
+ if (formattedResponse.output &&
519
+ typeof formattedResponse.output === 'object' &&
520
+ Object.keys(formattedResponse.output).length > 0) {
521
+ formattedResponse.output = JSON.stringify(formattedResponse.output, null, 2);
522
+ }
523
+ return {
524
+ content: [
525
+ {
526
+ type: 'text',
527
+ text: `Trace Details for ID: ${traceId}`,
528
+ },
529
+ {
530
+ type: 'text',
531
+ text: JSON.stringify(formattedResponse, null, 2),
532
+ },
533
+ ],
534
+ };
535
+ });
536
+ server.tool('get-trace-stats', 'Get statistics for traces', {
537
+ projectId: z.string().optional().describe('Project ID to filter traces'),
538
+ projectName: z.string().optional().describe('Project name to filter traces'),
539
+ startDate: z.string().optional().describe('Start date in ISO format (YYYY-MM-DD)'),
540
+ endDate: z.string().optional().describe('End date in ISO format (YYYY-MM-DD)'),
541
+ workspaceName: z.string().optional().describe('Workspace name to use instead of the default'),
542
+ }, async (args) => {
543
+ const { projectId, projectName, startDate, endDate, workspaceName } = args;
544
+ let url = `/v1/private/traces/stats`;
545
+ // Build query parameters
546
+ const queryParams = [];
547
+ // Add project filtering - API requires either project_id or project_name
548
+ if (projectId) {
549
+ queryParams.push(`project_id=${projectId}`);
550
+ }
551
+ else if (projectName) {
552
+ queryParams.push(`project_name=${encodeURIComponent(projectName)}`);
553
+ }
554
+ else {
555
+ // If no project specified, we need to find one for the API to work
556
+ const projectsResponse = await makeApiRequest(`/v1/private/projects?page=1&size=1`, {}, workspaceName);
557
+ if (projectsResponse.data &&
558
+ projectsResponse.data.content &&
559
+ projectsResponse.data.content.length > 0) {
560
+ const firstProject = projectsResponse.data.content[0];
561
+ queryParams.push(`project_id=${firstProject.id}`);
562
+ logToFile(`No project specified, using first available: ${firstProject.name} (${firstProject.id})`);
563
+ }
564
+ else {
565
+ return {
566
+ content: [
567
+ {
568
+ type: 'text',
569
+ text: 'Error: No project ID or name provided, and no projects found',
570
+ },
571
+ ],
572
+ };
573
+ }
574
+ }
575
+ if (startDate)
576
+ queryParams.push(`start_date=${startDate}`);
577
+ if (endDate)
578
+ queryParams.push(`end_date=${endDate}`);
579
+ if (queryParams.length > 0) {
580
+ url += `?${queryParams.join('&')}`;
581
+ }
582
+ const response = await makeApiRequest(url, {}, workspaceName);
583
+ if (!response.data) {
584
+ return {
585
+ content: [{ type: 'text', text: response.error || 'Failed to fetch trace statistics' }],
586
+ };
587
+ }
588
+ return {
589
+ content: [
590
+ {
591
+ type: 'text',
592
+ text: `Trace Statistics:`,
593
+ },
594
+ {
595
+ type: 'text',
596
+ text: JSON.stringify(response.data, null, 2),
597
+ },
598
+ ],
599
+ };
600
+ });
601
+ }
602
+ // ----------- METRICS TOOLS -----------
603
+ if (config.mcpEnableMetricTools) {
604
+ server.tool('get-metrics', 'Get metrics data', {
605
+ metricName: z.string().optional().describe('Optional metric name to filter'),
606
+ projectId: z.string().optional().describe('Optional project ID to filter metrics'),
607
+ projectName: z.string().optional().describe('Optional project name to filter metrics'),
608
+ startDate: z.string().optional().describe('Start date in ISO format (YYYY-MM-DD)'),
609
+ endDate: z.string().optional().describe('End date in ISO format (YYYY-MM-DD)'),
610
+ }, async (args) => {
611
+ const { metricName, projectId, projectName, startDate, endDate } = args;
612
+ let url = `/v1/private/metrics`;
613
+ const queryParams = [];
614
+ if (metricName)
615
+ queryParams.push(`metric_name=${metricName}`);
616
+ // Add project filtering - API requires either project_id or project_name
617
+ if (projectId) {
618
+ queryParams.push(`project_id=${projectId}`);
619
+ }
620
+ else if (projectName) {
621
+ queryParams.push(`project_name=${encodeURIComponent(projectName)}`);
622
+ }
623
+ else {
624
+ // If no project specified, we need to find one for the API to work
625
+ const projectsResponse = await makeApiRequest(`/v1/private/projects?page=1&size=1`);
626
+ if (projectsResponse.data &&
627
+ projectsResponse.data.content &&
628
+ projectsResponse.data.content.length > 0) {
629
+ const firstProject = projectsResponse.data.content[0];
630
+ queryParams.push(`project_id=${firstProject.id}`);
631
+ logToFile(`No project specified, using first available: ${firstProject.name} (${firstProject.id})`);
632
+ }
633
+ else {
634
+ return {
635
+ content: [
636
+ {
637
+ type: 'text',
638
+ text: 'Error: No project ID or name provided, and no projects found',
639
+ },
640
+ ],
641
+ };
642
+ }
643
+ }
644
+ if (startDate)
645
+ queryParams.push(`start_date=${startDate}`);
646
+ if (endDate)
647
+ queryParams.push(`end_date=${endDate}`);
648
+ if (queryParams.length > 0) {
649
+ url += `?${queryParams.join('&')}`;
650
+ }
651
+ const response = await makeApiRequest(url);
652
+ if (!response.data) {
653
+ return {
654
+ content: [{ type: 'text', text: response.error || 'Failed to fetch metrics' }],
655
+ };
656
+ }
657
+ return {
658
+ content: [
659
+ {
660
+ type: 'text',
661
+ text: JSON.stringify(response.data, null, 2),
662
+ },
663
+ ],
664
+ };
665
+ });
666
+ }
667
+ // ----------- SERVER CONFIGURATION TOOLS -----------
668
+ server.tool('get-server-info', 'Get information about the Opik server configuration', {
669
+ random_string: z.string().optional().describe('Dummy parameter for no-parameter tools'),
670
+ }, async () => {
671
+ // Get capabilities based on current configuration
672
+ const capabilities = getEnabledCapabilities(config);
673
+ const capabilitiesDescription = getCapabilitiesDescription(config);
674
+ return {
675
+ content: [
676
+ {
677
+ type: 'text',
678
+ text: JSON.stringify({
679
+ // API configuration
680
+ apiBaseUrl: config.apiBaseUrl,
681
+ isSelfHosted: config.isSelfHosted,
682
+ hasWorkspace: !!config.workspaceName,
683
+ workspaceName: config.workspaceName || 'none',
684
+ // MCP configuration
685
+ mcpName: config.mcpName,
686
+ mcpVersion: config.mcpVersion,
687
+ mcpDefaultWorkspace: config.mcpDefaultWorkspace,
688
+ enabledTools: {
689
+ prompts: config.mcpEnablePromptTools,
690
+ projects: config.mcpEnableProjectTools,
691
+ traces: config.mcpEnableTraceTools,
692
+ metrics: config.mcpEnableMetricTools,
693
+ },
694
+ serverVersion: 'v1',
695
+ // Capabilities information
696
+ capabilities: capabilities,
697
+ }, null, 2),
698
+ },
699
+ {
700
+ type: 'text',
701
+ text: capabilitiesDescription,
702
+ },
703
+ ],
704
+ };
705
+ });
706
+ // Add a new tool for contextual help about Opik capabilities
707
+ server.tool('get-opik-help', "Get contextual help about Opik Comet's capabilities", {
708
+ topic: z
709
+ .string()
710
+ .describe('The topic to get help about (prompts, projects, traces, metrics, or general)'),
711
+ subtopic: z.string().optional().describe('Optional subtopic for more specific help'),
712
+ }, async (args) => {
713
+ const { topic, subtopic } = args;
714
+ const capabilities = getEnabledCapabilities(config);
715
+ // Normalize topic to lowercase
716
+ const normalizedTopic = topic.toLowerCase();
717
+ // Check if the topic is valid
718
+ if (!['prompts', 'projects', 'traces', 'metrics', 'general'].includes(normalizedTopic)) {
719
+ return {
720
+ content: [
721
+ {
722
+ type: 'text',
723
+ text: `Invalid topic: ${topic}. Valid topics are: prompts, projects, traces, metrics, general.`,
724
+ },
725
+ ],
726
+ };
727
+ }
728
+ // Get the capabilities for the requested topic
729
+ const topicCapabilities = capabilities[normalizedTopic];
730
+ if (!topicCapabilities) {
731
+ return {
732
+ content: [
733
+ {
734
+ type: 'text',
735
+ text: `No information available for topic: ${topic}`,
736
+ },
737
+ ],
738
+ };
739
+ }
740
+ // If it's a general topic request
741
+ if (normalizedTopic === 'general') {
742
+ return {
743
+ content: [
744
+ {
745
+ type: 'text',
746
+ text: `Opik Comet General Information:\n\n` +
747
+ `API Version: ${topicCapabilities.apiVersion}\n` +
748
+ `Authentication: ${topicCapabilities.authentication}\n` +
749
+ `Rate Limit: ${topicCapabilities.rateLimit}\n` +
750
+ `Supported Formats: ${topicCapabilities.supportedFormats?.join(', ') || 'JSON'}`,
751
+ },
752
+ ],
753
+ };
754
+ }
755
+ // For other topics, check if they're available
756
+ const typedCapabilities = topicCapabilities;
757
+ if (!typedCapabilities.available) {
758
+ return {
759
+ content: [
760
+ {
761
+ type: 'text',
762
+ text: `${topic} functionality is not enabled in the current configuration.`,
763
+ },
764
+ ],
765
+ };
766
+ }
767
+ // If a subtopic is specified, provide more specific help
768
+ if (subtopic) {
769
+ const normalizedSubtopic = subtopic.toLowerCase();
770
+ // Handle different subtopics
771
+ switch (normalizedSubtopic) {
772
+ case 'features':
773
+ return {
774
+ content: [
775
+ {
776
+ type: 'text',
777
+ text: `${topic} Features:\n\n` +
778
+ typedCapabilities.features.map((f) => `- ${f}`).join('\n'),
779
+ },
780
+ ],
781
+ };
782
+ case 'limitations':
783
+ return {
784
+ content: [
785
+ {
786
+ type: 'text',
787
+ text: `${topic} Limitations:\n\n` +
788
+ typedCapabilities.limitations.map((l) => `- ${l}`).join('\n'),
789
+ },
790
+ ],
791
+ };
792
+ case 'examples':
793
+ if (typedCapabilities.examples && typedCapabilities.examples.length > 0) {
794
+ return {
795
+ content: [
796
+ {
797
+ type: 'text',
798
+ text: `${topic} Examples:\n\n` +
799
+ typedCapabilities.examples.map((e) => `- ${e}`).join('\n'),
800
+ },
801
+ ],
802
+ };
803
+ }
804
+ else {
805
+ return {
806
+ content: [
807
+ {
808
+ type: 'text',
809
+ text: `No examples available for ${topic}.`,
810
+ },
811
+ ],
812
+ };
813
+ }
814
+ case 'schema':
815
+ if (typedCapabilities.schema) {
816
+ return {
817
+ content: [
818
+ {
819
+ type: 'text',
820
+ text: `${topic} Schema:\n\n` + JSON.stringify(typedCapabilities.schema, null, 2),
821
+ },
822
+ ],
823
+ };
824
+ }
825
+ else {
826
+ return {
827
+ content: [
828
+ {
829
+ type: 'text',
830
+ text: `No schema information available for ${topic}.`,
831
+ },
832
+ ],
833
+ };
834
+ }
835
+ default:
836
+ // Check if the subtopic is a property of the capabilities
837
+ if (typedCapabilities[normalizedSubtopic] !== undefined) {
838
+ const value = typedCapabilities[normalizedSubtopic];
839
+ // Format the value based on its type
840
+ let formattedValue = '';
841
+ if (Array.isArray(value)) {
842
+ formattedValue = value.map((v) => `- ${v}`).join('\n');
843
+ }
844
+ else if (typeof value === 'object') {
845
+ formattedValue = JSON.stringify(value, null, 2);
846
+ }
847
+ else {
848
+ formattedValue = value.toString();
849
+ }
850
+ return {
851
+ content: [
852
+ {
853
+ type: 'text',
854
+ text: `${topic} ${subtopic}:\n\n${formattedValue}`,
855
+ },
856
+ ],
857
+ };
858
+ }
859
+ else {
860
+ return {
861
+ content: [
862
+ {
863
+ type: 'text',
864
+ text: `Invalid subtopic: ${subtopic} for topic: ${topic}`,
865
+ },
866
+ ],
867
+ };
868
+ }
869
+ }
870
+ }
871
+ // If no subtopic is specified, provide general information about the topic
872
+ let response = `${topic.charAt(0).toUpperCase() + topic.slice(1)} Capabilities:\n\n`;
873
+ response += 'Features:\n';
874
+ typedCapabilities.features.forEach((feature) => {
875
+ response += `- ${feature}\n`;
876
+ });
877
+ response += '\nLimitations:\n';
878
+ typedCapabilities.limitations.forEach((limitation) => {
879
+ response += `- ${limitation}\n`;
880
+ });
881
+ // Add topic-specific information
882
+ switch (normalizedTopic) {
883
+ case 'prompts':
884
+ response += `\nVersion Control: ${typedCapabilities.versionControl ? 'Supported' : 'Not Supported'}\n`;
885
+ response += `Template Format: ${typedCapabilities.templateFormat}\n`;
886
+ break;
887
+ case 'projects':
888
+ response += `\nHierarchy Support: ${typedCapabilities.hierarchySupport ? 'Supported' : 'Not Supported'}\n`;
889
+ response += `Sharing Support: ${typedCapabilities.sharingSupport ? 'Supported' : 'Not Supported'}\n`;
890
+ break;
891
+ case 'traces':
892
+ response += `\nData Retention: ${typedCapabilities.dataRetention}\n`;
893
+ response += `Search Capabilities:\n`;
894
+ typedCapabilities.searchCapabilities.forEach((capability) => {
895
+ response += `- ${capability}\n`;
896
+ });
897
+ break;
898
+ case 'metrics':
899
+ response += `\nAvailable Metrics:\n`;
900
+ typedCapabilities.availableMetrics.forEach((metric) => {
901
+ response += `- ${metric}\n`;
902
+ });
903
+ response += `Custom Metrics Support: ${typedCapabilities.customMetricsSupport ? 'Supported' : 'Not Supported'}\n`;
904
+ response += `Visualization Support: ${typedCapabilities.visualizationSupport ? 'Supported' : 'Not Supported'}\n`;
905
+ break;
906
+ }
907
+ // Add examples if available
908
+ if (typedCapabilities.examples && typedCapabilities.examples.length > 0) {
909
+ response += `\nExamples:\n`;
910
+ typedCapabilities.examples.forEach((example) => {
911
+ response += `- ${example}\n`;
912
+ });
913
+ }
914
+ return {
915
+ content: [
916
+ {
917
+ type: 'text',
918
+ text: response,
919
+ },
920
+ ],
921
+ };
922
+ });
923
+ // Add a tool for providing contextual examples of how to use Opik Comet
924
+ server.tool('get-opik-examples', "Get examples of how to use Opik Comet's API for specific tasks", {
925
+ task: z
926
+ .string()
927
+ .describe("The task to get examples for (e.g., 'create prompt', 'analyze traces', 'monitor costs')"),
928
+ }, async (args) => {
929
+ const { task } = args;
930
+ const normalizedTask = task.toLowerCase();
931
+ const examples = {
932
+ // Prompt-related examples
933
+ 'create prompt': {
934
+ description: 'Creating a new prompt template in Opik Comet',
935
+ steps: [
936
+ "1. Use the 'create-prompt' tool to create a new prompt with a name",
937
+ "2. Use the 'create-prompt-version' tool to add content to the prompt",
938
+ "3. Retrieve the prompt using 'get-prompt-by-id' to verify it was created",
939
+ ],
940
+ code: `// Example: Creating a customer service prompt
941
+ const promptName = "Customer Service Greeting";
942
+ const promptTemplate = "Hello {{customer_name}}, thank you for contacting our support. How can I help you today?";
943
+ const commitMessage = "Initial version of customer service greeting";
944
+
945
+ // First create the prompt
946
+ const createResult = await mcp.createPrompt({ name: promptName });
947
+ const promptId = createResult.id;
948
+
949
+ // Then add content as a version
950
+ await mcp.createPromptVersion({
951
+ name: promptName,
952
+ template: promptTemplate,
953
+ commit_message: commitMessage
954
+ });`,
955
+ },
956
+ 'version prompt': {
957
+ description: 'Creating a new version of an existing prompt',
958
+ steps: [
959
+ "1. Use the 'list-prompts' tool to find the prompt you want to version",
960
+ "2. Use the 'create-prompt-version' tool to add a new version with updated content",
961
+ '3. Include a descriptive commit message explaining the changes',
962
+ ],
963
+ code: `// Example: Creating a new version of an existing prompt
964
+ const promptName = "Customer Service Greeting";
965
+ const newTemplate = "Hello {{customer_name}}, thank you for reaching out to our support team. How may I assist you today?";
966
+ const commitMessage = "Improved wording for more professional tone";
967
+
968
+ await mcp.createPromptVersion({
969
+ name: promptName,
970
+ template: newTemplate,
971
+ commit_message: commitMessage
972
+ });`,
973
+ },
974
+ // Project-related examples
975
+ 'create project': {
976
+ description: 'Creating a new project in Opik Comet',
977
+ steps: [
978
+ "1. Use the 'create-project' tool to create a new project with a name and description",
979
+ "2. Retrieve the project using 'get-project-by-id' to verify it was created",
980
+ ],
981
+ code: `// Example: Creating a new project
982
+ const projectName = "Customer Support Bot";
983
+ const projectDescription = "AI assistant for handling customer support inquiries";
984
+
985
+ const createResult = await mcp.createProject({
986
+ name: projectName,
987
+ description: projectDescription
988
+ });
989
+
990
+ // The project ID will be in the response
991
+ const projectId = createResult.id;`,
992
+ },
993
+ 'organize traces': {
994
+ description: 'Organizing traces by project',
995
+ steps: [
996
+ '1. Create projects for different use cases or applications',
997
+ '2. When recording traces, associate them with the appropriate project',
998
+ "3. Use the 'list-traces' tool with project filtering to view traces for a specific project",
999
+ ],
1000
+ code: `// Example: Listing traces for a specific project
1001
+ const projectId = "proj_12345";
1002
+ const page = 1;
1003
+ const size = 10;
1004
+
1005
+ const traces = await mcp.listTraces({
1006
+ page: page,
1007
+ size: size,
1008
+ projectId: projectId
1009
+ });
1010
+
1011
+ // Alternatively, you can filter by project name
1012
+ const projectName = "Customer Support Bot";
1013
+ const tracesByName = await mcp.listTraces({
1014
+ page: page,
1015
+ size: size,
1016
+ projectName: projectName
1017
+ });`,
1018
+ },
1019
+ // Trace-related examples
1020
+ 'log trace': {
1021
+ description: 'Logging a trace with the Opik API',
1022
+ steps: [
1023
+ '1. Create a trace with input and output data',
1024
+ '2. Add spans to the trace to capture detailed steps',
1025
+ '3. Include LLM calls with relevant metadata',
1026
+ ],
1027
+ code: `// Example: Logging a trace with spans
1028
+ // Based on official Opik documentation
1029
+
1030
+ // Python SDK example (for reference)
1031
+ /*
1032
+ from opik import Opik
1033
+
1034
+ client = Opik(project_name="Opik client demo")
1035
+
1036
+ # Create a trace
1037
+ trace = client.trace(
1038
+ name="my_trace",
1039
+ input={"user_question": "Hello, how are you?"},
1040
+ output={"response": "Comment ça va?"}
1041
+ )
1042
+
1043
+ # Add a span
1044
+ trace.span(
1045
+ name="Add prompt template",
1046
+ input={"text": "Hello, how are you?", "prompt_template": "Translate the following text to French: {text}"},
1047
+ output={"text": "Translate the following text to French: hello, how are you?"}
1048
+ )
1049
+
1050
+ # Add an LLM call
1051
+ trace.span(
1052
+ name="llm_call",
1053
+ type="llm",
1054
+ input={"prompt": "Translate the following text to French: hello, how are you?"},
1055
+ output={"response": "Comment ça va?"}
1056
+ )
1057
+ */
1058
+
1059
+ // JavaScript/TypeScript equivalent using the API
1060
+ const projectId = "proj_12345";
1061
+
1062
+ // Create a trace
1063
+ const traceData = {
1064
+ name: "my_trace",
1065
+ project_id: projectId,
1066
+ input: {"user_question": "Hello, how are you?"},
1067
+ output: {"response": "Comment ça va?"}
1068
+ };
1069
+
1070
+ const traceResponse = await fetch("/v1/private/traces", {
1071
+ method: "POST",
1072
+ headers: {
1073
+ "Content-Type": "application/json",
1074
+ "Authorization": "YOUR_API_KEY"
1075
+ },
1076
+ body: JSON.stringify(traceData)
1077
+ });
1078
+
1079
+ const trace = await traceResponse.json();
1080
+ const traceId = trace.id;
1081
+
1082
+ // Add spans to the trace
1083
+ const span1 = {
1084
+ trace_id: traceId,
1085
+ name: "Add prompt template",
1086
+ input: {"text": "Hello, how are you?", "prompt_template": "Translate the following text to French: {text}"},
1087
+ output: {"text": "Translate the following text to French: hello, how are you?"}
1088
+ };
1089
+
1090
+ const span2 = {
1091
+ trace_id: traceId,
1092
+ name: "llm_call",
1093
+ type: "llm",
1094
+ input: {"prompt": "Translate the following text to French: hello, how are you?"},
1095
+ output: {"response": "Comment ça va?"}
1096
+ };
1097
+
1098
+ await fetch("/v1/private/spans", {
1099
+ method: "POST",
1100
+ headers: {
1101
+ "Content-Type": "application/json",
1102
+ "Authorization": "YOUR_API_KEY"
1103
+ },
1104
+ body: JSON.stringify(span1)
1105
+ });
1106
+
1107
+ await fetch("/v1/private/spans", {
1108
+ method: "POST",
1109
+ headers: {
1110
+ "Content-Type": "application/json",
1111
+ "Authorization": "YOUR_API_KEY"
1112
+ },
1113
+ body: JSON.stringify(span2)
1114
+ });`,
1115
+ },
1116
+ 'analyze traces': {
1117
+ description: 'Analyzing trace data to understand usage patterns',
1118
+ steps: [
1119
+ "1. Use the 'list-traces' tool to retrieve traces for a specific project",
1120
+ "2. Use the 'get-trace-stats' tool to get aggregated statistics",
1121
+ '3. Filter by date range to analyze trends over time',
1122
+ ],
1123
+ code: `// Example: Getting trace statistics for a date range
1124
+ const projectId = "proj_12345";
1125
+ const startDate = "2023-01-01";
1126
+ const endDate = "2023-01-31";
1127
+
1128
+ const stats = await mcp.getTraceStats({
1129
+ projectId: projectId,
1130
+ startDate: startDate,
1131
+ endDate: endDate
1132
+ });
1133
+
1134
+ // The response will include aggregated data like:
1135
+ // - Total trace count
1136
+ // - Total token usage
1137
+ // - Cost information
1138
+ // - Daily breakdowns`,
1139
+ },
1140
+ 'view trace details': {
1141
+ description: 'Viewing detailed information about a specific trace',
1142
+ steps: [
1143
+ "1. Use the 'list-traces' tool to find the trace you want to examine",
1144
+ "2. Use the 'get-trace-by-id' tool with the trace ID to get detailed information",
1145
+ '3. Analyze the input, output, and metadata to understand the interaction',
1146
+ ],
1147
+ code: `// Example: Getting detailed information about a trace
1148
+ const traceId = "trace_67890";
1149
+
1150
+ const traceDetails = await mcp.getTraceById({
1151
+ traceId: traceId
1152
+ });
1153
+
1154
+ // The response will include:
1155
+ // - Input and output data
1156
+ // - Token usage
1157
+ // - Timestamps
1158
+ // - Metadata
1159
+ // - Cost information
1160
+ // - Spans (detailed steps within the trace)`,
1161
+ },
1162
+ 'annotate trace': {
1163
+ description: 'Annotating a trace with feedback scores',
1164
+ steps: [
1165
+ "1. Retrieve a trace using 'get-trace-by-id'",
1166
+ '2. Add feedback scores to evaluate the trace quality',
1167
+ '3. Use the feedback for monitoring and improvement',
1168
+ ],
1169
+ code: `// Example: Annotating a trace with feedback scores
1170
+ // Based on Opik documentation
1171
+
1172
+ // Python SDK example (for reference)
1173
+ /*
1174
+ from opik import Opik
1175
+
1176
+ client = Opik(project_name="Opik client demo")
1177
+
1178
+ # Get an existing trace
1179
+ trace = client.get_trace(trace_id="trace_12345")
1180
+
1181
+ # Add feedback scores
1182
+ trace.add_feedback_score(name="relevance", score=0.8)
1183
+ trace.add_feedback_score(name="accuracy", score=0.9)
1184
+ trace.add_feedback_score(name="helpfulness", score=0.7)
1185
+ */
1186
+
1187
+ // JavaScript/TypeScript equivalent using the API
1188
+ const traceId = "trace_12345";
1189
+
1190
+ // Add feedback scores to the trace
1191
+ const feedbackData = {
1192
+ scores: [
1193
+ { name: "relevance", score: 0.8 },
1194
+ { name: "accuracy", score: 0.9 },
1195
+ { name: "helpfulness", score: 0.7 }
1196
+ ]
1197
+ };
1198
+
1199
+ await fetch(\`/v1/private/traces/\${traceId}/feedback\`, {
1200
+ method: "POST",
1201
+ headers: {
1202
+ "Content-Type": "application/json",
1203
+ "Authorization": "YOUR_API_KEY"
1204
+ },
1205
+ body: JSON.stringify(feedbackData)
1206
+ });`,
1207
+ },
1208
+ // Metrics-related examples
1209
+ 'monitor costs': {
1210
+ description: 'Monitoring costs across projects and time periods',
1211
+ steps: [
1212
+ "1. Use the 'get-metrics' tool with the 'cost' metric name",
1213
+ '2. Filter by project and date range to focus on specific usage',
1214
+ '3. Analyze trends to identify cost patterns',
1215
+ ],
1216
+ code: `// Example: Monitoring costs for a specific project
1217
+ const projectId = "proj_12345";
1218
+ const metricName = "cost";
1219
+ const startDate = "2023-01-01";
1220
+ const endDate = "2023-01-31";
1221
+
1222
+ const costMetrics = await mcp.getMetrics({
1223
+ metricName: metricName,
1224
+ projectId: projectId,
1225
+ startDate: startDate,
1226
+ endDate: endDate
1227
+ });
1228
+
1229
+ // The response will include cost data points over time`,
1230
+ },
1231
+ 'track token usage': {
1232
+ description: 'Tracking token usage across different models and projects',
1233
+ steps: [
1234
+ "1. Use the 'get-metrics' tool with token-related metric names",
1235
+ '2. Filter by project and date range to focus on specific usage',
1236
+ '3. Compare prompt tokens vs. completion tokens to optimize usage',
1237
+ ],
1238
+ code: `// Example: Tracking token usage metrics
1239
+ const projectId = "proj_12345";
1240
+ const startDate = "2023-01-01";
1241
+ const endDate = "2023-01-31";
1242
+
1243
+ // Get total token usage
1244
+ const totalTokens = await mcp.getMetrics({
1245
+ metricName: "total_tokens",
1246
+ projectId: projectId,
1247
+ startDate: startDate,
1248
+ endDate: endDate
1249
+ });
1250
+
1251
+ // Get prompt token usage
1252
+ const promptTokens = await mcp.getMetrics({
1253
+ metricName: "prompt_tokens",
1254
+ projectId: projectId,
1255
+ startDate: startDate,
1256
+ endDate: endDate
1257
+ });
1258
+
1259
+ // Get completion token usage
1260
+ const completionTokens = await mcp.getMetrics({
1261
+ metricName: "completion_tokens",
1262
+ projectId: projectId,
1263
+ startDate: startDate,
1264
+ endDate: endDate
1265
+ });`,
1266
+ },
1267
+ 'evaluate llm': {
1268
+ description: "Evaluating LLM outputs using Opik's evaluation metrics",
1269
+ steps: [
1270
+ '1. Set up evaluation metrics for your use case',
1271
+ '2. Apply metrics to trace data to measure performance',
1272
+ '3. Analyze results to identify areas for improvement',
1273
+ ],
1274
+ code: `// Example: Evaluating LLM outputs with metrics
1275
+ // Based on Opik documentation
1276
+
1277
+ // Python SDK example (for reference)
1278
+ /*
1279
+ from opik import evaluate
1280
+ from opik.metrics import Hallucination, AnswerRelevance, ContextPrecision
1281
+
1282
+ # Define evaluation metrics
1283
+ metrics = [
1284
+ Hallucination(),
1285
+ AnswerRelevance(),
1286
+ ContextPrecision()
1287
+ ]
1288
+
1289
+ # Evaluate a response
1290
+ result = evaluate(
1291
+ question="What is the capital of France?",
1292
+ answer="Paris is the capital of France.",
1293
+ context=["Paris is the capital and most populous city of France."],
1294
+ metrics=metrics
1295
+ )
1296
+
1297
+ # Print results
1298
+ print(result.scores)
1299
+ */
1300
+
1301
+ // JavaScript/TypeScript equivalent using the API
1302
+ const evaluationData = {
1303
+ question: "What is the capital of France?",
1304
+ answer: "Paris is the capital of France.",
1305
+ context: ["Paris is the capital and most populous city of France."],
1306
+ metrics: ["hallucination", "answer_relevance", "context_precision"]
1307
+ };
1308
+
1309
+ const evaluationResponse = await fetch("/v1/private/evaluate", {
1310
+ method: "POST",
1311
+ headers: {
1312
+ "Content-Type": "application/json",
1313
+ "Authorization": "YOUR_API_KEY"
1314
+ },
1315
+ body: JSON.stringify(evaluationData)
1316
+ });
1317
+
1318
+ const evaluationResults = await evaluationResponse.json();
1319
+ // The response will include scores for each metric`,
1320
+ },
1321
+ };
1322
+ // Find the closest matching example
1323
+ let bestMatch = null;
1324
+ let bestMatchScore = 0;
1325
+ for (const [key /* example */] of Object.entries(examples)) {
1326
+ // Simple matching algorithm - check if the normalized task contains the key
1327
+ if (normalizedTask.includes(key)) {
1328
+ const score = key.length; // Longer matches are better
1329
+ if (score > bestMatchScore) {
1330
+ bestMatch = key;
1331
+ bestMatchScore = score;
1332
+ }
1333
+ }
1334
+ }
1335
+ // If no match found, provide a list of available examples
1336
+ if (!bestMatch) {
1337
+ return {
1338
+ content: [
1339
+ {
1340
+ type: 'text',
1341
+ text: `No specific example found for "${task}". Available example categories include:\n\n` +
1342
+ Object.keys(examples)
1343
+ .map(key => `- ${key}`)
1344
+ .join('\n') +
1345
+ `\n\nTry asking for one of these specific tasks.`,
1346
+ },
1347
+ ],
1348
+ };
1349
+ }
1350
+ // Return the matched example
1351
+ const matchedExample = examples[bestMatch];
1352
+ return {
1353
+ content: [
1354
+ {
1355
+ type: 'text',
1356
+ text: `Example: ${bestMatch}\n\n` +
1357
+ `Description: ${matchedExample.description}\n\n` +
1358
+ `Steps:\n${matchedExample.steps.join('\n')}\n\n` +
1359
+ `Code Example:\n\`\`\`javascript\n${matchedExample.code}\n\`\`\``,
1360
+ },
1361
+ ],
1362
+ };
1363
+ });
1364
+ // Add a tool for providing information about Opik's tracing capabilities
1365
+ server.tool('get-opik-tracing-info', "Get information about Opik's tracing capabilities and how to use them", {
1366
+ topic: z
1367
+ .string()
1368
+ .optional()
1369
+ .describe("Optional specific tracing topic to get information about (e.g., 'spans', 'distributed', 'multimodal', 'annotations')"),
1370
+ }, async (args) => {
1371
+ const { topic } = args;
1372
+ const tracingInfo = {
1373
+ basic: {
1374
+ name: 'Basic Tracing',
1375
+ description: 'Core tracing functionality for recording LLM interactions with input and output data.',
1376
+ key_features: [
1377
+ 'Record input and output for LLM calls',
1378
+ 'Track token usage and costs',
1379
+ 'Organize traces by project',
1380
+ 'Add metadata to traces',
1381
+ ],
1382
+ use_cases: [
1383
+ 'Monitoring LLM usage in applications',
1384
+ 'Debugging LLM-based systems',
1385
+ 'Cost tracking and optimization',
1386
+ 'Performance monitoring',
1387
+ ],
1388
+ example: `from opik import Opik
1389
+
1390
+ # Initialize Opik client
1391
+ client = Opik(project_name="My Project")
1392
+
1393
+ # Create a trace
1394
+ trace = client.trace(
1395
+ name="simple_query",
1396
+ input={"question": "What is the capital of France?"},
1397
+ output={"answer": "The capital of France is Paris."},
1398
+ metadata={"model": "gpt-4", "temperature": 0.7}
1399
+ )`,
1400
+ related_topics: ['spans', 'annotations', 'metadata'],
1401
+ },
1402
+ spans: {
1403
+ name: 'Spans',
1404
+ description: 'Detailed tracking of steps within a trace to capture the full flow of an LLM interaction.',
1405
+ key_features: [
1406
+ 'Break down traces into logical steps',
1407
+ 'Track intermediate processing',
1408
+ 'Capture the full chain of operations',
1409
+ 'Measure performance of individual steps',
1410
+ ],
1411
+ use_cases: [
1412
+ 'Debugging complex LLM pipelines',
1413
+ 'Performance optimization of multi-step processes',
1414
+ 'Visualizing the flow of information',
1415
+ 'Identifying bottlenecks in processing',
1416
+ ],
1417
+ example: `from opik import Opik
1418
+
1419
+ # Initialize Opik client
1420
+ client = Opik(project_name="RAG Application")
1421
+
1422
+ # Create a trace
1423
+ trace = client.trace(
1424
+ name="rag_query",
1425
+ input={"question": "What is the capital of France?"},
1426
+ output={"answer": "The capital of France is Paris."}
1427
+ )
1428
+
1429
+ # Add spans for each step in the process
1430
+ trace.span(
1431
+ name="query_processing",
1432
+ input={"raw_query": "What is the capital of France?"},
1433
+ output={"processed_query": "capital France"}
1434
+ )
1435
+
1436
+ trace.span(
1437
+ name="document_retrieval",
1438
+ input={"query": "capital France"},
1439
+ output={"documents": ["Paris is the capital of France.", "France is a country in Europe."]}
1440
+ )
1441
+
1442
+ trace.span(
1443
+ name="llm_generation",
1444
+ type="llm",
1445
+ input={"prompt": "Based on these documents, answer: What is the capital of France?\\n\\nDocuments:\\n- Paris is the capital of France.\\n- France is a country in Europe."},
1446
+ output={"response": "The capital of France is Paris."}
1447
+ )`,
1448
+ related_topics: ['basic', 'distributed', 'context'],
1449
+ },
1450
+ distributed: {
1451
+ name: 'Distributed Tracing',
1452
+ description: 'Tracing across multiple services or components in a distributed system.',
1453
+ key_features: [
1454
+ 'Track LLM interactions across service boundaries',
1455
+ 'Maintain context across different components',
1456
+ 'Visualize end-to-end flows',
1457
+ 'Correlate related traces',
1458
+ ],
1459
+ use_cases: [
1460
+ 'Microservices architectures with LLMs',
1461
+ 'Complex multi-component AI systems',
1462
+ 'Cross-service debugging',
1463
+ 'End-to-end performance monitoring',
1464
+ ],
1465
+ example: `# Service 1: Initial request handler
1466
+ from opik import Opik, opik_context
1467
+
1468
+ client = Opik(project_name="Distributed System")
1469
+
1470
+ # Create a trace
1471
+ trace = client.trace(
1472
+ name="user_request",
1473
+ input={"user_query": "What is the capital of France?"}
1474
+ )
1475
+
1476
+ # Get trace headers to pass to the next service
1477
+ trace_headers = opik_context.get_distributed_trace_headers()
1478
+
1479
+ # Pass trace_headers to Service 2 via API call, message queue, etc.
1480
+
1481
+ # -----------------------------------------------
1482
+
1483
+ # Service 2: Document retrieval service
1484
+ from opik import Opik, opik_context
1485
+
1486
+ client = Opik(project_name="Distributed System")
1487
+
1488
+ # Initialize context from received headers
1489
+ opik_context.init_from_headers(received_headers)
1490
+
1491
+ # This span will be automatically associated with the parent trace
1492
+ with client.span(name="document_retrieval") as span:
1493
+ # Retrieve documents
1494
+ documents = retrieve_documents("capital France")
1495
+ span.update(output={"documents": documents})
1496
+
1497
+ # -----------------------------------------------
1498
+
1499
+ # Service 3: LLM service
1500
+ from opik import Opik, opik_context
1501
+
1502
+ client = Opik(project_name="Distributed System")
1503
+
1504
+ # Initialize context from received headers
1505
+ opik_context.init_from_headers(received_headers)
1506
+
1507
+ # This span will be automatically associated with the parent trace
1508
+ with client.span(name="llm_generation", type="llm") as span:
1509
+ # Generate response
1510
+ response = generate_llm_response(documents, "What is the capital of France?")
1511
+ span.update(output={"response": response})
1512
+
1513
+ # Back in Service 1, update the trace with the final output
1514
+ trace.update(output={"answer": "The capital of France is Paris."})`,
1515
+ related_topics: ['spans', 'context', 'opentelemetry'],
1516
+ },
1517
+ multimodal: {
1518
+ name: 'Multimodal Tracing',
1519
+ description: 'Tracing for LLM interactions that involve multiple modalities like text, images, and audio.',
1520
+ key_features: [
1521
+ 'Track inputs and outputs across modalities',
1522
+ 'Support for image, audio, and text data',
1523
+ 'Visualize multimodal interactions',
1524
+ 'Analyze performance across modalities',
1525
+ ],
1526
+ use_cases: [
1527
+ 'Vision-language models (VLMs)',
1528
+ 'Image generation and analysis',
1529
+ 'Audio transcription and processing',
1530
+ 'Multimodal chatbots and assistants',
1531
+ ],
1532
+ example: `from opik import Opik
1533
+ import base64
1534
+
1535
+ # Initialize Opik client
1536
+ client = Opik(project_name="Multimodal App")
1537
+
1538
+ # Load image as base64
1539
+ with open("image.jpg", "rb") as f:
1540
+ image_data = base64.b64encode(f.read()).decode("utf-8")
1541
+
1542
+ # Create a multimodal trace
1543
+ trace = client.trace(
1544
+ name="image_analysis",
1545
+ input={
1546
+ "image": {"mime_type": "image/jpeg", "data": image_data},
1547
+ "question": "What objects are in this image?"
1548
+ },
1549
+ output={"answer": "The image contains a cat sitting on a windowsill."}
1550
+ )
1551
+
1552
+ # Add a span for the vision model
1553
+ trace.span(
1554
+ name="vision_model",
1555
+ type="llm",
1556
+ input={"image": {"mime_type": "image/jpeg", "data": image_data}},
1557
+ output={"description": "A tabby cat sitting on a wooden windowsill looking outside."}
1558
+ )
1559
+
1560
+ # Add a span for the text generation
1561
+ trace.span(
1562
+ name="text_generation",
1563
+ type="llm",
1564
+ input={"prompt": "Based on this description: 'A tabby cat sitting on a wooden windowsill looking outside.', answer: What objects are in this image?"},
1565
+ output={"answer": "The image contains a cat sitting on a windowsill."}
1566
+ )`,
1567
+ related_topics: ['basic', 'spans'],
1568
+ },
1569
+ annotations: {
1570
+ name: 'Trace Annotations',
1571
+ description: 'Adding feedback scores and annotations to traces for evaluation and improvement.',
1572
+ key_features: [
1573
+ 'Add qualitative and quantitative feedback',
1574
+ 'Score trace quality and performance',
1575
+ 'Track user satisfaction',
1576
+ 'Support continuous improvement',
1577
+ ],
1578
+ use_cases: [
1579
+ 'Quality monitoring in production',
1580
+ 'User feedback collection',
1581
+ 'A/B testing of LLM configurations',
1582
+ 'Performance benchmarking',
1583
+ ],
1584
+ example: `from opik import Opik
1585
+
1586
+ # Initialize Opik client
1587
+ client = Opik(project_name="Customer Support")
1588
+
1589
+ # Get an existing trace
1590
+ trace = client.get_trace(trace_id="trace_12345")
1591
+
1592
+ # Add feedback scores
1593
+ trace.add_feedback_score(name="relevance", score=0.8)
1594
+ trace.add_feedback_score(name="accuracy", score=0.9)
1595
+ trace.add_feedback_score(name="helpfulness", score=0.7)
1596
+
1597
+ # Add a qualitative annotation
1598
+ trace.add_annotation(text="Response was helpful but could be more concise.")`,
1599
+ related_topics: ['basic', 'evaluation'],
1600
+ },
1601
+ context: {
1602
+ name: 'Context Management',
1603
+ description: 'Managing trace context throughout the execution flow of an application.',
1604
+ key_features: [
1605
+ 'Automatic context propagation',
1606
+ 'Access current trace and span data',
1607
+ 'Update traces and spans dynamically',
1608
+ 'Support for async and concurrent operations',
1609
+ ],
1610
+ use_cases: [
1611
+ 'Complex application flows',
1612
+ 'Asynchronous processing',
1613
+ 'Middleware integration',
1614
+ 'Framework integration',
1615
+ ],
1616
+ example: `from opik import Opik, opik_context
1617
+
1618
+ # Initialize Opik client
1619
+ client = Opik(project_name="Context Demo")
1620
+
1621
+ # Create a trace
1622
+ with client.trace(name="main_process") as trace:
1623
+ # The trace is automatically set as the current trace
1624
+
1625
+ # Access current trace data
1626
+ trace_data = opik_context.get_current_trace_data()
1627
+
1628
+ # Create a span
1629
+ with client.span(name="subprocess") as span:
1630
+ # The span is automatically set as the current span
1631
+
1632
+ # Access current span data
1633
+ span_data = opik_context.get_current_span_data()
1634
+
1635
+ # Update the current span
1636
+ opik_context.update_current_span(output={"result": "Processed data"})
1637
+
1638
+ # Update the current trace
1639
+ opik_context.update_current_trace(output={"final_result": "Complete"})`,
1640
+ related_topics: ['distributed', 'spans'],
1641
+ },
1642
+ opentelemetry: {
1643
+ name: 'OpenTelemetry Integration',
1644
+ description: 'Integration with the OpenTelemetry standard for distributed tracing.',
1645
+ key_features: [
1646
+ 'Compatibility with OpenTelemetry ecosystem',
1647
+ 'Standard-compliant trace format',
1648
+ 'Integration with existing observability tools',
1649
+ 'Support for mixed tracing environments',
1650
+ ],
1651
+ use_cases: [
1652
+ 'Enterprise observability platforms',
1653
+ 'Integration with existing monitoring systems',
1654
+ 'Standardized tracing across organizations',
1655
+ 'Multi-vendor observability solutions',
1656
+ ],
1657
+ example: `# OpenTelemetry integration example
1658
+ from opentelemetry import trace
1659
+ from opentelemetry.sdk.trace import TracerProvider
1660
+ from opentelemetry.sdk.trace.export import BatchSpanProcessor
1661
+ from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
1662
+ from opik.integrations.opentelemetry import OpikSpanProcessor
1663
+
1664
+ # Set up OpenTelemetry
1665
+ tracer_provider = TracerProvider()
1666
+ trace.set_tracer_provider(tracer_provider)
1667
+
1668
+ # Set up OTLP exporter
1669
+ otlp_exporter = OTLPSpanExporter(endpoint="your-otlp-endpoint")
1670
+ tracer_provider.add_span_processor(BatchSpanProcessor(otlp_exporter))
1671
+
1672
+ # Add Opik span processor
1673
+ opik_processor = OpikSpanProcessor(
1674
+ project_name="OpenTelemetry Demo",
1675
+ api_key="your-opik-api-key"
1676
+ )
1677
+ tracer_provider.add_span_processor(opik_processor)
1678
+
1679
+ # Now OpenTelemetry traces will also be sent to Opik
1680
+ tracer = trace.get_tracer(__name__)
1681
+
1682
+ with tracer.start_as_current_span("main_operation") as span:
1683
+ # This span will be captured by both OpenTelemetry and Opik
1684
+ span.set_attribute("operation.type", "query")
1685
+
1686
+ # Perform operations
1687
+ result = process_data()
1688
+
1689
+ span.set_attribute("operation.result", result)`,
1690
+ related_topics: ['distributed', 'context'],
1691
+ },
1692
+ metadata: {
1693
+ name: 'Trace Metadata',
1694
+ description: 'Adding contextual metadata to traces for richer analysis and filtering.',
1695
+ key_features: [
1696
+ 'Add custom metadata to traces and spans',
1697
+ 'Tag traces for easier filtering',
1698
+ 'Include environment and version information',
1699
+ 'Track business-specific metrics',
1700
+ ],
1701
+ use_cases: [
1702
+ 'Environment-specific analysis',
1703
+ 'Version comparison',
1704
+ 'Business impact tracking',
1705
+ 'Custom categorization',
1706
+ ],
1707
+ example: `from opik import Opik
1708
+
1709
+ # Initialize Opik client
1710
+ client = Opik(project_name="Metadata Demo")
1711
+
1712
+ # Create a trace with rich metadata
1713
+ trace = client.trace(
1714
+ name="product_search",
1715
+ input={"query": "blue running shoes"},
1716
+ output={"results": ["Product 1", "Product 2", "Product 3"]},
1717
+ metadata={
1718
+ "environment": "production",
1719
+ "version": "1.2.3",
1720
+ "user_segment": "premium",
1721
+ "region": "us-west",
1722
+ "experiment_id": "exp_a1b2c3",
1723
+ "business_metrics": {
1724
+ "conversion_rate": 0.12,
1725
+ "average_order_value": 85.50
1726
+ }
1727
+ }
1728
+ )
1729
+
1730
+ # Add tags for easier filtering
1731
+ trace.add_tags(["search", "product", "footwear"])`,
1732
+ related_topics: ['basic', 'annotations'],
1733
+ },
1734
+ };
1735
+ // If a specific topic is requested, return information about that topic
1736
+ if (topic) {
1737
+ const normalizedTopic = topic.toLowerCase();
1738
+ // Try exact match first
1739
+ if (tracingInfo[normalizedTopic]) {
1740
+ const topicData = tracingInfo[normalizedTopic];
1741
+ return {
1742
+ content: [
1743
+ {
1744
+ type: 'text',
1745
+ text: `# ${topicData.name}\n\n` +
1746
+ `**Description:** ${topicData.description}\n\n` +
1747
+ `**Key Features:**\n${topicData.key_features.map(f => `- ${f}`).join('\n')}\n\n` +
1748
+ `**Use Cases:**\n${topicData.use_cases.map(uc => `- ${uc}`).join('\n')}\n\n` +
1749
+ (topicData.example
1750
+ ? `**Example:**\n\`\`\`python\n${topicData.example}\n\`\`\`\n\n`
1751
+ : '') +
1752
+ (topicData.related_topics && topicData.related_topics.length > 0
1753
+ ? `**Related Topics:** ${topicData.related_topics.map(t => `\`${t}\``).join(', ')}`
1754
+ : ''),
1755
+ },
1756
+ ],
1757
+ };
1758
+ }
1759
+ // Try fuzzy match
1760
+ const fuzzyMatches = Object.keys(tracingInfo).filter(k => k.includes(normalizedTopic) || normalizedTopic.includes(k));
1761
+ if (fuzzyMatches.length > 0) {
1762
+ return {
1763
+ content: [
1764
+ {
1765
+ type: 'text',
1766
+ text: `No exact match found for "${topic}". Did you mean one of these?\n\n` +
1767
+ fuzzyMatches.map(m => `- ${tracingInfo[m].name}`).join('\n'),
1768
+ },
1769
+ ],
1770
+ };
1771
+ }
1772
+ // No matches
1773
+ return {
1774
+ content: [
1775
+ {
1776
+ type: 'text',
1777
+ text: `No information found for tracing topic "${topic}". Available topics include:\n\n` +
1778
+ Object.values(tracingInfo)
1779
+ .map(t => `- ${t.name}`)
1780
+ .join('\n'),
1781
+ },
1782
+ ],
1783
+ };
1784
+ }
1785
+ // If no specific topic is requested, return an overview of all tracing capabilities
1786
+ return {
1787
+ content: [
1788
+ {
1789
+ type: 'text',
1790
+ text: `# Opik Tracing Capabilities\n\n` +
1791
+ `Opik provides comprehensive tracing capabilities for LLM applications, allowing you to track, analyze, and improve your AI systems.\n\n` +
1792
+ `## Core Tracing Features\n\n` +
1793
+ Object.values(tracingInfo)
1794
+ .map(t => `### ${t.name}\n${t.description}\n\n**Key Features:**\n${t.key_features.map(f => `- ${f}`).join('\n')}\n`)
1795
+ .join('\n\n') +
1796
+ `\n\n## Getting Started with Tracing\n\n` +
1797
+ `To start using Opik's tracing capabilities:\n\n` +
1798
+ `1. Install the Opik SDK: \`pip install opik\`\n` +
1799
+ `2. Configure your API key: \`opik configure\`\n` +
1800
+ `3. Create your first trace using the \`trace()\` method\n` +
1801
+ `4. Add spans to capture detailed steps in your process\n` +
1802
+ `5. View your traces in the Opik dashboard\n\n` +
1803
+ `For detailed information about a specific tracing topic, use this tool with the \`topic\` parameter.`,
1804
+ },
1805
+ ],
1806
+ };
1807
+ });
1808
+ // Main function to start the server
1809
+ export async function main() {
1810
+ logToFile('Starting main function');
1811
+ // Create the appropriate transport based on configuration
1812
+ let transport;
1813
+ if (config.transport === 'sse') {
1814
+ logToFile(`Creating SSEServerTransport on port ${config.ssePort}`);
1815
+ transport = new SSEServerTransport({
1816
+ port: config.ssePort || 3001,
1817
+ });
1818
+ // Explicitly start the SSE transport
1819
+ logToFile('Starting SSE transport');
1820
+ await transport.start();
1821
+ }
1822
+ else {
1823
+ logToFile('Creating StdioServerTransport');
1824
+ transport = new StdioServerTransport();
1825
+ }
1826
+ // Connect the server to the transport
1827
+ logToFile('Connecting server to transport');
1828
+ server.connect(transport);
1829
+ logToFile('Transport connection established');
1830
+ // Log server status
1831
+ if (config.transport === 'sse') {
1832
+ logToFile(`Opik MCP Server running on SSE (port ${config.ssePort})`);
1833
+ }
1834
+ else {
1835
+ logToFile('Opik MCP Server running on stdio');
1836
+ }
1837
+ logToFile('Main function completed successfully');
1838
+ // Start heartbeat for keeping the process alive
1839
+ setInterval(() => {
1840
+ logToFile('Heartbeat ping');
1841
+ }, 5000);
1842
+ }
1843
+ // Start the server
1844
+ main().catch(error => {
1845
+ logToFile(`Error starting server: ${error}`);
1846
+ process.exit(1);
1847
+ });