langfuse-mcp-extended 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +207 -0
- package/build/index.d.ts +2 -0
- package/build/index.js +1179 -0
- package/build/index.js.map +1 -0
- package/package.json +72 -0
package/build/index.js
ADDED
|
@@ -0,0 +1,1179 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
// src/config.ts
|
|
4
|
+
import { z } from "zod";
|
|
5
|
+
var ConfigSchema = z.object({
|
|
6
|
+
langfuse: z.object({
|
|
7
|
+
publicKey: z.string().min(1, "LANGFUSE_PUBLIC_KEY is required"),
|
|
8
|
+
secretKey: z.string().min(1, "LANGFUSE_SECRET_KEY is required"),
|
|
9
|
+
baseUrl: z.string().url().default("https://cloud.langfuse.com")
|
|
10
|
+
}),
|
|
11
|
+
server: z.object({
|
|
12
|
+
enablePrompts: z.boolean().default(false),
|
|
13
|
+
logLevel: z.enum(["debug", "info", "warn", "error"]).default("info")
|
|
14
|
+
})
|
|
15
|
+
});
|
|
16
|
+
function loadConfig() {
|
|
17
|
+
const result = ConfigSchema.safeParse({
|
|
18
|
+
langfuse: {
|
|
19
|
+
publicKey: process.env.LANGFUSE_PUBLIC_KEY,
|
|
20
|
+
secretKey: process.env.LANGFUSE_SECRET_KEY,
|
|
21
|
+
baseUrl: process.env.LANGFUSE_BASE_URL || "https://cloud.langfuse.com"
|
|
22
|
+
},
|
|
23
|
+
server: {
|
|
24
|
+
enablePrompts: process.env.MCP_ENABLE_PROMPTS === "true",
|
|
25
|
+
logLevel: process.env.LOG_LEVEL || "info"
|
|
26
|
+
}
|
|
27
|
+
});
|
|
28
|
+
if (!result.success) {
|
|
29
|
+
const errors = result.error.errors.map((e) => `${e.path.join(".")}: ${e.message}`);
|
|
30
|
+
throw new Error(`Configuration error:
|
|
31
|
+
${errors.join("\n")}`);
|
|
32
|
+
}
|
|
33
|
+
return result.data;
|
|
34
|
+
}
|
|
35
|
+
function createLogger(config) {
|
|
36
|
+
const levels = { debug: 0, info: 1, warn: 2, error: 3 };
|
|
37
|
+
const currentLevel = levels[config.server.logLevel];
|
|
38
|
+
return {
|
|
39
|
+
debug: (msg, ...args) => {
|
|
40
|
+
if (currentLevel <= 0) console.error(`[DEBUG] ${msg}`, ...args);
|
|
41
|
+
},
|
|
42
|
+
info: (msg, ...args) => {
|
|
43
|
+
if (currentLevel <= 1) console.error(`[INFO] ${msg}`, ...args);
|
|
44
|
+
},
|
|
45
|
+
warn: (msg, ...args) => {
|
|
46
|
+
if (currentLevel <= 2) console.error(`[WARN] ${msg}`, ...args);
|
|
47
|
+
},
|
|
48
|
+
error: (msg, ...args) => {
|
|
49
|
+
if (currentLevel <= 3) console.error(`[ERROR] ${msg}`, ...args);
|
|
50
|
+
}
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
// src/server.ts
|
|
55
|
+
import { Server } from "@modelcontextprotocol/sdk/server/index.js";
|
|
56
|
+
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
57
|
+
import {
|
|
58
|
+
CallToolRequestSchema,
|
|
59
|
+
ListToolsRequestSchema
|
|
60
|
+
} from "@modelcontextprotocol/sdk/types.js";
|
|
61
|
+
|
|
62
|
+
// src/client.ts
|
|
63
|
+
import Langfuse from "langfuse";
|
|
64
|
+
var LangfuseClient = class {
|
|
65
|
+
langfuse;
|
|
66
|
+
logger;
|
|
67
|
+
baseUrl;
|
|
68
|
+
authHeader;
|
|
69
|
+
constructor(config, logger) {
|
|
70
|
+
this.logger = logger;
|
|
71
|
+
this.baseUrl = config.langfuse.baseUrl;
|
|
72
|
+
const credentials = `${config.langfuse.publicKey}:${config.langfuse.secretKey}`;
|
|
73
|
+
this.authHeader = `Basic ${Buffer.from(credentials).toString("base64")}`;
|
|
74
|
+
this.langfuse = new Langfuse({
|
|
75
|
+
publicKey: config.langfuse.publicKey,
|
|
76
|
+
secretKey: config.langfuse.secretKey,
|
|
77
|
+
baseUrl: config.langfuse.baseUrl
|
|
78
|
+
});
|
|
79
|
+
this.logger.debug(`LangfuseClient initialized with baseUrl: ${this.baseUrl}`);
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Access the Langfuse SDK instance
|
|
83
|
+
*/
|
|
84
|
+
get sdk() {
|
|
85
|
+
return this.langfuse;
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Make a direct API request to Langfuse
|
|
89
|
+
*/
|
|
90
|
+
async apiRequest(method, path, body) {
|
|
91
|
+
const url = `${this.baseUrl}${path}`;
|
|
92
|
+
this.logger.debug(`API Request: ${method} ${path}`);
|
|
93
|
+
const response = await fetch(url, {
|
|
94
|
+
method,
|
|
95
|
+
headers: {
|
|
96
|
+
Authorization: this.authHeader,
|
|
97
|
+
"Content-Type": "application/json"
|
|
98
|
+
},
|
|
99
|
+
body: body ? JSON.stringify(body) : void 0
|
|
100
|
+
});
|
|
101
|
+
if (!response.ok) {
|
|
102
|
+
const errorText = await response.text();
|
|
103
|
+
this.logger.error(`API Error: ${response.status} - ${errorText}`);
|
|
104
|
+
throw new LangfuseApiError(response.status, errorText, path);
|
|
105
|
+
}
|
|
106
|
+
const text = await response.text();
|
|
107
|
+
if (!text) {
|
|
108
|
+
return {};
|
|
109
|
+
}
|
|
110
|
+
return JSON.parse(text);
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* Shutdown the client and flush any pending data
|
|
114
|
+
*/
|
|
115
|
+
async shutdown() {
|
|
116
|
+
await this.langfuse.shutdownAsync();
|
|
117
|
+
this.logger.debug("LangfuseClient shutdown complete");
|
|
118
|
+
}
|
|
119
|
+
};
|
|
120
|
+
var LangfuseApiError = class extends Error {
|
|
121
|
+
constructor(statusCode, body, path) {
|
|
122
|
+
super(`Langfuse API Error (${statusCode}) at ${path}: ${body}`);
|
|
123
|
+
this.statusCode = statusCode;
|
|
124
|
+
this.body = body;
|
|
125
|
+
this.path = path;
|
|
126
|
+
this.name = "LangfuseApiError";
|
|
127
|
+
}
|
|
128
|
+
get isNotFound() {
|
|
129
|
+
return this.statusCode === 404;
|
|
130
|
+
}
|
|
131
|
+
get isUnauthorized() {
|
|
132
|
+
return this.statusCode === 401;
|
|
133
|
+
}
|
|
134
|
+
get isRateLimited() {
|
|
135
|
+
return this.statusCode === 429;
|
|
136
|
+
}
|
|
137
|
+
};
|
|
138
|
+
|
|
139
|
+
// src/tools/traces/listTraces.ts
|
|
140
|
+
import { z as z2 } from "zod";
|
|
141
|
+
|
|
142
|
+
// src/utils/errors.ts
|
|
143
|
+
function formatSuccess(data, summary) {
|
|
144
|
+
const content = [];
|
|
145
|
+
if (summary) {
|
|
146
|
+
content.push({ type: "text", text: summary });
|
|
147
|
+
}
|
|
148
|
+
content.push({
|
|
149
|
+
type: "text",
|
|
150
|
+
text: JSON.stringify(data, null, 2)
|
|
151
|
+
});
|
|
152
|
+
return { content };
|
|
153
|
+
}
|
|
154
|
+
function formatError(message) {
|
|
155
|
+
return {
|
|
156
|
+
content: [{ type: "text", text: `Error: ${message}` }],
|
|
157
|
+
isError: true
|
|
158
|
+
};
|
|
159
|
+
}
|
|
160
|
+
function handleToolError(error) {
|
|
161
|
+
if (error instanceof LangfuseApiError) {
|
|
162
|
+
if (error.isNotFound) {
|
|
163
|
+
return formatError(`Resource not found: ${error.path}`);
|
|
164
|
+
}
|
|
165
|
+
if (error.isUnauthorized) {
|
|
166
|
+
return formatError("Authentication failed. Check your API keys.");
|
|
167
|
+
}
|
|
168
|
+
if (error.isRateLimited) {
|
|
169
|
+
return formatError("Rate limited. Please try again later.");
|
|
170
|
+
}
|
|
171
|
+
return formatError(`API Error (${error.statusCode}): ${error.body}`);
|
|
172
|
+
}
|
|
173
|
+
if (error instanceof Error && error.name === "ZodError") {
|
|
174
|
+
return formatError(`Validation error: ${error.message}`);
|
|
175
|
+
}
|
|
176
|
+
if (error instanceof Error) {
|
|
177
|
+
return formatError(error.message);
|
|
178
|
+
}
|
|
179
|
+
console.error("Unexpected error:", error);
|
|
180
|
+
return formatError("An unexpected error occurred");
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
// src/tools/registry.ts
|
|
184
|
+
function defineTool(definition) {
|
|
185
|
+
return definition;
|
|
186
|
+
}
|
|
187
|
+
var ToolRegistry = class {
|
|
188
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
189
|
+
tools = /* @__PURE__ */ new Map();
|
|
190
|
+
client;
|
|
191
|
+
config;
|
|
192
|
+
logger;
|
|
193
|
+
constructor(client, config, logger) {
|
|
194
|
+
this.client = client;
|
|
195
|
+
this.config = config;
|
|
196
|
+
this.logger = logger;
|
|
197
|
+
}
|
|
198
|
+
/**
|
|
199
|
+
* Register a tool in the registry
|
|
200
|
+
*/
|
|
201
|
+
register(tool) {
|
|
202
|
+
if (tool.enabled && !tool.enabled(this.config)) {
|
|
203
|
+
this.logger.debug(`Tool ${tool.name} is disabled by configuration`);
|
|
204
|
+
return;
|
|
205
|
+
}
|
|
206
|
+
this.tools.set(tool.name, tool);
|
|
207
|
+
this.logger.debug(`Registered tool: ${tool.name}`);
|
|
208
|
+
}
|
|
209
|
+
/**
|
|
210
|
+
* Register multiple tools
|
|
211
|
+
*/
|
|
212
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
213
|
+
registerAll(tools) {
|
|
214
|
+
for (const tool of tools) {
|
|
215
|
+
this.register(tool);
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
/**
|
|
219
|
+
* Get all registered tools in MCP format
|
|
220
|
+
*/
|
|
221
|
+
getTools() {
|
|
222
|
+
return Array.from(this.tools.values()).map((tool) => ({
|
|
223
|
+
name: tool.name,
|
|
224
|
+
description: tool.description,
|
|
225
|
+
inputSchema: this.zodToJsonSchema(tool.inputSchema)
|
|
226
|
+
}));
|
|
227
|
+
}
|
|
228
|
+
/**
|
|
229
|
+
* Execute a tool by name
|
|
230
|
+
*/
|
|
231
|
+
async execute(name, args) {
|
|
232
|
+
const tool = this.tools.get(name);
|
|
233
|
+
if (!tool) {
|
|
234
|
+
return {
|
|
235
|
+
content: [{ type: "text", text: `Unknown tool: ${name}` }],
|
|
236
|
+
isError: true
|
|
237
|
+
};
|
|
238
|
+
}
|
|
239
|
+
try {
|
|
240
|
+
const parseResult = tool.inputSchema.safeParse(args);
|
|
241
|
+
if (!parseResult.success) {
|
|
242
|
+
const errors = parseResult.error.errors.map((e) => `${e.path.join(".")}: ${e.message}`).join(", ");
|
|
243
|
+
return {
|
|
244
|
+
content: [{ type: "text", text: `Invalid input: ${errors}` }],
|
|
245
|
+
isError: true
|
|
246
|
+
};
|
|
247
|
+
}
|
|
248
|
+
this.logger.debug(`Executing tool: ${name}`);
|
|
249
|
+
return await tool.handler(parseResult.data, this.client, this.logger);
|
|
250
|
+
} catch (error) {
|
|
251
|
+
this.logger.error(`Error executing tool ${name}:`, error);
|
|
252
|
+
return handleToolError(error);
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
/**
|
|
256
|
+
* Convert Zod schema to JSON Schema for MCP
|
|
257
|
+
*/
|
|
258
|
+
zodToJsonSchema(schema) {
|
|
259
|
+
const jsonSchema = {
|
|
260
|
+
type: "object",
|
|
261
|
+
properties: {},
|
|
262
|
+
required: []
|
|
263
|
+
};
|
|
264
|
+
if ("shape" in schema && typeof schema.shape === "object") {
|
|
265
|
+
const shape = schema.shape;
|
|
266
|
+
const properties = {};
|
|
267
|
+
const required = [];
|
|
268
|
+
for (const [key, value] of Object.entries(shape)) {
|
|
269
|
+
properties[key] = this.zodTypeToJsonSchema(value);
|
|
270
|
+
if (!value.isOptional()) {
|
|
271
|
+
required.push(key);
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
jsonSchema.properties = properties;
|
|
275
|
+
if (required.length > 0) {
|
|
276
|
+
jsonSchema.required = required;
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
return jsonSchema;
|
|
280
|
+
}
|
|
281
|
+
/**
|
|
282
|
+
* Convert individual Zod type to JSON Schema
|
|
283
|
+
*/
|
|
284
|
+
zodTypeToJsonSchema(zodType) {
|
|
285
|
+
const schema = {};
|
|
286
|
+
if (zodType.description) {
|
|
287
|
+
schema.description = zodType.description;
|
|
288
|
+
}
|
|
289
|
+
let innerType = zodType;
|
|
290
|
+
if ("unwrap" in zodType && typeof zodType.unwrap === "function") {
|
|
291
|
+
innerType = zodType.unwrap();
|
|
292
|
+
}
|
|
293
|
+
const typeName = innerType._def?.typeName;
|
|
294
|
+
switch (typeName) {
|
|
295
|
+
case "ZodString":
|
|
296
|
+
schema.type = "string";
|
|
297
|
+
break;
|
|
298
|
+
case "ZodNumber":
|
|
299
|
+
schema.type = "number";
|
|
300
|
+
break;
|
|
301
|
+
case "ZodBoolean":
|
|
302
|
+
schema.type = "boolean";
|
|
303
|
+
break;
|
|
304
|
+
case "ZodArray":
|
|
305
|
+
schema.type = "array";
|
|
306
|
+
if (innerType._def?.type) {
|
|
307
|
+
schema.items = this.zodTypeToJsonSchema(innerType._def.type);
|
|
308
|
+
}
|
|
309
|
+
break;
|
|
310
|
+
case "ZodObject":
|
|
311
|
+
schema.type = "object";
|
|
312
|
+
break;
|
|
313
|
+
case "ZodEnum":
|
|
314
|
+
schema.type = "string";
|
|
315
|
+
if (innerType._def?.values) {
|
|
316
|
+
schema.enum = innerType._def.values;
|
|
317
|
+
}
|
|
318
|
+
break;
|
|
319
|
+
default:
|
|
320
|
+
schema.type = "string";
|
|
321
|
+
}
|
|
322
|
+
return schema;
|
|
323
|
+
}
|
|
324
|
+
};
|
|
325
|
+
|
|
326
|
+
// src/utils/pagination.ts
|
|
327
|
+
function buildQueryString(params) {
|
|
328
|
+
const query = new URLSearchParams();
|
|
329
|
+
for (const [key, value] of Object.entries(params)) {
|
|
330
|
+
if (value === void 0) continue;
|
|
331
|
+
if (Array.isArray(value)) {
|
|
332
|
+
for (const item of value) {
|
|
333
|
+
query.append(key, item);
|
|
334
|
+
}
|
|
335
|
+
} else if (typeof value === "boolean") {
|
|
336
|
+
query.set(key, value ? "true" : "false");
|
|
337
|
+
} else {
|
|
338
|
+
query.set(key, String(value));
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
const queryString = query.toString();
|
|
342
|
+
return queryString ? `?${queryString}` : "";
|
|
343
|
+
}
|
|
344
|
+
function formatPaginationSummary(count, total, page, hasMore) {
|
|
345
|
+
const parts = [];
|
|
346
|
+
if (total !== void 0) {
|
|
347
|
+
parts.push(`Showing ${count} of ${total} items`);
|
|
348
|
+
} else {
|
|
349
|
+
parts.push(`Found ${count} items`);
|
|
350
|
+
}
|
|
351
|
+
if (page !== void 0) {
|
|
352
|
+
parts.push(`(page ${page})`);
|
|
353
|
+
}
|
|
354
|
+
if (hasMore) {
|
|
355
|
+
parts.push("- more available");
|
|
356
|
+
}
|
|
357
|
+
return parts.join(" ");
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
// src/tools/traces/listTraces.ts
|
|
361
|
+
var inputSchema = z2.object({
|
|
362
|
+
page: z2.number().int().min(1).optional().describe("Page number (1-indexed)"),
|
|
363
|
+
limit: z2.number().int().min(1).max(100).optional().default(50).describe("Items per page"),
|
|
364
|
+
name: z2.string().optional().describe("Filter by trace name"),
|
|
365
|
+
userId: z2.string().optional().describe("Filter by user ID"),
|
|
366
|
+
sessionId: z2.string().optional().describe("Filter by session ID"),
|
|
367
|
+
tags: z2.array(z2.string()).optional().describe("Filter by tags (AND logic)"),
|
|
368
|
+
fromTimestamp: z2.string().optional().describe("Start of time range (ISO 8601)"),
|
|
369
|
+
toTimestamp: z2.string().optional().describe("End of time range (ISO 8601)"),
|
|
370
|
+
environment: z2.string().optional().describe("Filter by environment"),
|
|
371
|
+
orderBy: z2.enum(["timestamp", "latency", "totalCost"]).optional().describe("Sort field")
|
|
372
|
+
});
|
|
373
|
+
var listTraces = defineTool({
|
|
374
|
+
name: "listTraces",
|
|
375
|
+
description: "List traces with filtering and pagination. Returns trace metadata including latency, cost, and observation/score counts.",
|
|
376
|
+
inputSchema,
|
|
377
|
+
handler: async (input, client, logger) => {
|
|
378
|
+
logger.debug("listTraces called with:", input);
|
|
379
|
+
const queryParams = {
|
|
380
|
+
page: input.page,
|
|
381
|
+
limit: input.limit,
|
|
382
|
+
name: input.name,
|
|
383
|
+
userId: input.userId,
|
|
384
|
+
sessionId: input.sessionId,
|
|
385
|
+
tags: input.tags,
|
|
386
|
+
fromTimestamp: input.fromTimestamp,
|
|
387
|
+
toTimestamp: input.toTimestamp,
|
|
388
|
+
environment: input.environment,
|
|
389
|
+
orderBy: input.orderBy
|
|
390
|
+
};
|
|
391
|
+
const query = buildQueryString(queryParams);
|
|
392
|
+
const response = await client.apiRequest("GET", `/api/public/traces${query}`);
|
|
393
|
+
const summary = formatPaginationSummary(
|
|
394
|
+
response.data.length,
|
|
395
|
+
response.meta.totalItems,
|
|
396
|
+
response.meta.page,
|
|
397
|
+
response.meta.page < response.meta.totalPages
|
|
398
|
+
);
|
|
399
|
+
return formatSuccess(response, summary);
|
|
400
|
+
}
|
|
401
|
+
});
|
|
402
|
+
|
|
403
|
+
// src/tools/traces/getTrace.ts
|
|
404
|
+
import { z as z3 } from "zod";
|
|
405
|
+
var inputSchema2 = z3.object({
|
|
406
|
+
traceId: z3.string().min(1).describe("The unique trace identifier")
|
|
407
|
+
});
|
|
408
|
+
var getTrace = defineTool({
|
|
409
|
+
name: "getTrace",
|
|
410
|
+
description: "Get a specific trace with full details including all observations and scores.",
|
|
411
|
+
inputSchema: inputSchema2,
|
|
412
|
+
handler: async (input, client, logger) => {
|
|
413
|
+
logger.debug("getTrace called with:", input);
|
|
414
|
+
const response = await client.apiRequest(
|
|
415
|
+
"GET",
|
|
416
|
+
`/api/public/traces/${encodeURIComponent(input.traceId)}`
|
|
417
|
+
);
|
|
418
|
+
const observationCount = response.observations?.length ?? 0;
|
|
419
|
+
const scoreCount = response.scores?.length ?? 0;
|
|
420
|
+
const summary = `Trace "${response.name || response.id}" with ${observationCount} observations and ${scoreCount} scores`;
|
|
421
|
+
return formatSuccess(response, summary);
|
|
422
|
+
}
|
|
423
|
+
});
|
|
424
|
+
|
|
425
|
+
// src/tools/traces/deleteTrace.ts
|
|
426
|
+
import { z as z4 } from "zod";
|
|
427
|
+
var inputSchema3 = z4.object({
|
|
428
|
+
traceId: z4.string().min(1).describe("The unique trace identifier to delete")
|
|
429
|
+
});
|
|
430
|
+
var deleteTrace = defineTool({
|
|
431
|
+
name: "deleteTrace",
|
|
432
|
+
description: "Delete a specific trace. This action is irreversible.",
|
|
433
|
+
inputSchema: inputSchema3,
|
|
434
|
+
handler: async (input, client, logger) => {
|
|
435
|
+
logger.debug("deleteTrace called with:", input);
|
|
436
|
+
await client.apiRequest("DELETE", `/api/public/traces/${encodeURIComponent(input.traceId)}`);
|
|
437
|
+
return formatSuccess(
|
|
438
|
+
{ deleted: true, traceId: input.traceId },
|
|
439
|
+
`Successfully deleted trace: ${input.traceId}`
|
|
440
|
+
);
|
|
441
|
+
}
|
|
442
|
+
});
|
|
443
|
+
|
|
444
|
+
// src/tools/observations/listObservations.ts
|
|
445
|
+
import { z as z5 } from "zod";
|
|
446
|
+
var inputSchema4 = z5.object({
|
|
447
|
+
cursor: z5.string().optional().describe("Pagination cursor from previous response"),
|
|
448
|
+
limit: z5.number().int().min(1).max(100).optional().default(50).describe("Items per page"),
|
|
449
|
+
traceId: z5.string().optional().describe("Filter by trace ID"),
|
|
450
|
+
name: z5.string().optional().describe("Filter by observation name"),
|
|
451
|
+
type: z5.enum(["GENERATION", "SPAN", "EVENT"]).optional().describe("Filter by observation type"),
|
|
452
|
+
parentObservationId: z5.string().optional().describe("Filter by parent observation"),
|
|
453
|
+
fromStartTime: z5.string().optional().describe("Start of time range (ISO 8601)"),
|
|
454
|
+
toStartTime: z5.string().optional().describe("End of time range (ISO 8601)"),
|
|
455
|
+
userId: z5.string().optional().describe("Filter by user ID"),
|
|
456
|
+
version: z5.string().optional().describe("Filter by version")
|
|
457
|
+
});
|
|
458
|
+
var listObservations = defineTool({
|
|
459
|
+
name: "listObservations",
|
|
460
|
+
description: "List observations (generations, spans, events) with cursor-based pagination. Includes usage metrics, costs, and latency.",
|
|
461
|
+
inputSchema: inputSchema4,
|
|
462
|
+
handler: async (input, client, logger) => {
|
|
463
|
+
logger.debug("listObservations called with:", input);
|
|
464
|
+
const queryParams = {
|
|
465
|
+
page: input.cursor ? parseInt(input.cursor, 10) : 1,
|
|
466
|
+
limit: input.limit,
|
|
467
|
+
traceId: input.traceId,
|
|
468
|
+
name: input.name,
|
|
469
|
+
type: input.type,
|
|
470
|
+
parentObservationId: input.parentObservationId,
|
|
471
|
+
fromStartTime: input.fromStartTime,
|
|
472
|
+
toStartTime: input.toStartTime,
|
|
473
|
+
userId: input.userId,
|
|
474
|
+
version: input.version
|
|
475
|
+
};
|
|
476
|
+
const query = buildQueryString(queryParams);
|
|
477
|
+
const response = await client.apiRequest(
|
|
478
|
+
"GET",
|
|
479
|
+
`/api/public/observations${query}`
|
|
480
|
+
);
|
|
481
|
+
const summary = formatPaginationSummary(
|
|
482
|
+
response.data.length,
|
|
483
|
+
response.meta.totalItems,
|
|
484
|
+
response.meta.page,
|
|
485
|
+
response.meta.page < response.meta.totalPages
|
|
486
|
+
);
|
|
487
|
+
const result = {
|
|
488
|
+
...response,
|
|
489
|
+
nextCursor: response.meta.page < response.meta.totalPages ? String(response.meta.page + 1) : void 0
|
|
490
|
+
};
|
|
491
|
+
return formatSuccess(result, summary);
|
|
492
|
+
}
|
|
493
|
+
});
|
|
494
|
+
|
|
495
|
+
// src/tools/observations/getObservation.ts
|
|
496
|
+
import { z as z6 } from "zod";
|
|
497
|
+
var inputSchema5 = z6.object({
|
|
498
|
+
observationId: z6.string().min(1).describe("The unique observation identifier")
|
|
499
|
+
});
|
|
500
|
+
var getObservation = defineTool({
|
|
501
|
+
name: "getObservation",
|
|
502
|
+
description: "Get a specific observation with all details including input/output, usage, costs, and timing.",
|
|
503
|
+
inputSchema: inputSchema5,
|
|
504
|
+
handler: async (input, client, logger) => {
|
|
505
|
+
logger.debug("getObservation called with:", input);
|
|
506
|
+
const response = await client.apiRequest(
|
|
507
|
+
"GET",
|
|
508
|
+
`/api/public/observations/${encodeURIComponent(input.observationId)}`
|
|
509
|
+
);
|
|
510
|
+
const typeLabel = response.type.toLowerCase();
|
|
511
|
+
const modelInfo = response.model ? ` using ${response.model}` : "";
|
|
512
|
+
const summary = `${typeLabel} "${response.name || response.id}"${modelInfo}`;
|
|
513
|
+
return formatSuccess(response, summary);
|
|
514
|
+
}
|
|
515
|
+
});
|
|
516
|
+
|
|
517
|
+
// src/tools/scores/createScore.ts
|
|
518
|
+
import { z as z7 } from "zod";
|
|
519
|
+
var inputSchema6 = z7.object({
|
|
520
|
+
traceId: z7.string().min(1).describe("ID of the trace to score"),
|
|
521
|
+
observationId: z7.string().optional().describe("Optional: ID of specific observation to score"),
|
|
522
|
+
name: z7.string().min(1).describe("Name of the score (e.g., 'accuracy', 'relevance')"),
|
|
523
|
+
value: z7.union([z7.number(), z7.string()]).describe("Score value - numeric for NUMERIC type, string for CATEGORICAL/BOOLEAN"),
|
|
524
|
+
dataType: z7.enum(["NUMERIC", "CATEGORICAL", "BOOLEAN"]).optional().describe("Type of score value"),
|
|
525
|
+
comment: z7.string().optional().describe("Optional comment explaining the score"),
|
|
526
|
+
configId: z7.string().optional().describe("Optional score config ID for validation"),
|
|
527
|
+
id: z7.string().optional().describe("Optional custom ID for idempotency")
|
|
528
|
+
});
|
|
529
|
+
var createScore = defineTool({
|
|
530
|
+
name: "createScore",
|
|
531
|
+
description: "Create a score for a trace or observation. Supports numeric, boolean, and categorical score types.",
|
|
532
|
+
inputSchema: inputSchema6,
|
|
533
|
+
handler: async (input, client, logger) => {
|
|
534
|
+
logger.debug("createScore called with:", input);
|
|
535
|
+
const response = await client.apiRequest("POST", "/api/public/scores", {
|
|
536
|
+
traceId: input.traceId,
|
|
537
|
+
observationId: input.observationId,
|
|
538
|
+
name: input.name,
|
|
539
|
+
value: input.value,
|
|
540
|
+
dataType: input.dataType,
|
|
541
|
+
comment: input.comment,
|
|
542
|
+
configId: input.configId,
|
|
543
|
+
id: input.id
|
|
544
|
+
});
|
|
545
|
+
const targetDesc = input.observationId ? `observation ${input.observationId}` : `trace ${input.traceId}`;
|
|
546
|
+
const summary = `Created score "${input.name}" = ${input.value} for ${targetDesc}`;
|
|
547
|
+
return formatSuccess(response, summary);
|
|
548
|
+
}
|
|
549
|
+
});
|
|
550
|
+
|
|
551
|
+
// src/tools/scores/listScores.ts
|
|
552
|
+
import { z as z8 } from "zod";
|
|
553
|
+
var inputSchema7 = z8.object({
|
|
554
|
+
page: z8.number().int().min(1).optional().describe("Page number (1-indexed)"),
|
|
555
|
+
limit: z8.number().int().min(1).max(100).optional().default(50).describe("Items per page"),
|
|
556
|
+
traceId: z8.string().optional().describe("Filter by trace ID"),
|
|
557
|
+
observationId: z8.string().optional().describe("Filter by observation ID"),
|
|
558
|
+
name: z8.string().optional().describe("Filter by score name"),
|
|
559
|
+
source: z8.enum(["API", "ANNOTATION", "EVAL"]).optional().describe("Filter by score source"),
|
|
560
|
+
dataType: z8.enum(["NUMERIC", "CATEGORICAL", "BOOLEAN"]).optional().describe("Filter by data type"),
|
|
561
|
+
configId: z8.string().optional().describe("Filter by score config ID"),
|
|
562
|
+
fromTimestamp: z8.string().optional().describe("Start of time range (ISO 8601)"),
|
|
563
|
+
toTimestamp: z8.string().optional().describe("End of time range (ISO 8601)"),
|
|
564
|
+
userId: z8.string().optional().describe("Filter by user ID who created the trace"),
|
|
565
|
+
operator: z8.enum(["<", ">", "<=", ">=", "!=", "="]).optional().describe("Comparison operator for value filter"),
|
|
566
|
+
value: z8.number().optional().describe("Value to compare against (requires operator)")
|
|
567
|
+
});
|
|
568
|
+
var listScores = defineTool({
|
|
569
|
+
name: "listScores",
|
|
570
|
+
description: "List scores with filtering. Supports both trace and observation scores.",
|
|
571
|
+
inputSchema: inputSchema7,
|
|
572
|
+
handler: async (input, client, logger) => {
|
|
573
|
+
logger.debug("listScores called with:", input);
|
|
574
|
+
const queryParams = {
|
|
575
|
+
page: input.page,
|
|
576
|
+
limit: input.limit,
|
|
577
|
+
traceId: input.traceId,
|
|
578
|
+
observationId: input.observationId,
|
|
579
|
+
name: input.name,
|
|
580
|
+
source: input.source,
|
|
581
|
+
dataType: input.dataType,
|
|
582
|
+
configId: input.configId,
|
|
583
|
+
fromTimestamp: input.fromTimestamp,
|
|
584
|
+
toTimestamp: input.toTimestamp,
|
|
585
|
+
userId: input.userId,
|
|
586
|
+
operator: input.operator,
|
|
587
|
+
value: input.value
|
|
588
|
+
};
|
|
589
|
+
const query = buildQueryString(queryParams);
|
|
590
|
+
const response = await client.apiRequest(
|
|
591
|
+
"GET",
|
|
592
|
+
`/api/public/v2/scores${query}`
|
|
593
|
+
);
|
|
594
|
+
const summary = formatPaginationSummary(
|
|
595
|
+
response.data.length,
|
|
596
|
+
response.meta.totalItems,
|
|
597
|
+
response.meta.page,
|
|
598
|
+
response.meta.page < response.meta.totalPages
|
|
599
|
+
);
|
|
600
|
+
return formatSuccess(response, summary);
|
|
601
|
+
}
|
|
602
|
+
});
|
|
603
|
+
|
|
604
|
+
// src/tools/scores/getScore.ts
|
|
605
|
+
import { z as z9 } from "zod";
|
|
606
|
+
var inputSchema8 = z9.object({
|
|
607
|
+
scoreId: z9.string().min(1).describe("The unique score identifier")
|
|
608
|
+
});
|
|
609
|
+
var getScore = defineTool({
|
|
610
|
+
name: "getScore",
|
|
611
|
+
description: "Get a specific score by ID.",
|
|
612
|
+
inputSchema: inputSchema8,
|
|
613
|
+
handler: async (input, client, logger) => {
|
|
614
|
+
logger.debug("getScore called with:", input);
|
|
615
|
+
const response = await client.apiRequest(
|
|
616
|
+
"GET",
|
|
617
|
+
`/api/public/v2/scores/${encodeURIComponent(input.scoreId)}`
|
|
618
|
+
);
|
|
619
|
+
const summary = `Score "${response.name}" = ${response.value} (${response.dataType})`;
|
|
620
|
+
return formatSuccess(response, summary);
|
|
621
|
+
}
|
|
622
|
+
});
|
|
623
|
+
|
|
624
|
+
// src/tools/scores/deleteScore.ts
|
|
625
|
+
import { z as z10 } from "zod";
|
|
626
|
+
var inputSchema9 = z10.object({
|
|
627
|
+
scoreId: z10.string().min(1).describe("The unique score identifier to delete")
|
|
628
|
+
});
|
|
629
|
+
var deleteScore = defineTool({
|
|
630
|
+
name: "deleteScore",
|
|
631
|
+
description: "Delete a specific score. This action is irreversible.",
|
|
632
|
+
inputSchema: inputSchema9,
|
|
633
|
+
handler: async (input, client, logger) => {
|
|
634
|
+
logger.debug("deleteScore called with:", input);
|
|
635
|
+
await client.apiRequest("DELETE", `/api/public/scores/${encodeURIComponent(input.scoreId)}`);
|
|
636
|
+
return formatSuccess(
|
|
637
|
+
{ deleted: true, scoreId: input.scoreId },
|
|
638
|
+
`Successfully deleted score: ${input.scoreId}`
|
|
639
|
+
);
|
|
640
|
+
}
|
|
641
|
+
});
|
|
642
|
+
|
|
643
|
+
// src/tools/score-configs/createScoreConfig.ts
|
|
644
|
+
import { z as z11 } from "zod";
|
|
645
|
+
var categorySchema = z11.object({
|
|
646
|
+
value: z11.number().describe("Numeric value for this category"),
|
|
647
|
+
label: z11.string().describe("Display label for this category")
|
|
648
|
+
});
|
|
649
|
+
var inputSchema10 = z11.object({
|
|
650
|
+
name: z11.string().min(1).describe("Unique name for the score configuration"),
|
|
651
|
+
dataType: z11.enum(["NUMERIC", "CATEGORICAL", "BOOLEAN"]).describe("Type of score values this config accepts"),
|
|
652
|
+
minValue: z11.number().optional().describe("Minimum allowed value (NUMERIC only)"),
|
|
653
|
+
maxValue: z11.number().optional().describe("Maximum allowed value (NUMERIC only)"),
|
|
654
|
+
categories: z11.array(categorySchema).optional().describe("Category definitions (CATEGORICAL only)"),
|
|
655
|
+
description: z11.string().optional().describe("Description of what this score measures")
|
|
656
|
+
});
|
|
657
|
+
var createScoreConfig = defineTool({
|
|
658
|
+
name: "createScoreConfig",
|
|
659
|
+
description: "Create a score configuration that defines validation rules for scores. Supports numeric ranges, categorical values, and boolean types.",
|
|
660
|
+
inputSchema: inputSchema10,
|
|
661
|
+
handler: async (input, client, logger) => {
|
|
662
|
+
logger.debug("createScoreConfig called with:", input);
|
|
663
|
+
const response = await client.apiRequest(
|
|
664
|
+
"POST",
|
|
665
|
+
"/api/public/score-configs",
|
|
666
|
+
{
|
|
667
|
+
name: input.name,
|
|
668
|
+
dataType: input.dataType,
|
|
669
|
+
minValue: input.minValue,
|
|
670
|
+
maxValue: input.maxValue,
|
|
671
|
+
categories: input.categories,
|
|
672
|
+
description: input.description
|
|
673
|
+
}
|
|
674
|
+
);
|
|
675
|
+
let summary = `Created score config "${input.name}" (${input.dataType})`;
|
|
676
|
+
if (input.dataType === "NUMERIC" && (input.minValue !== void 0 || input.maxValue !== void 0)) {
|
|
677
|
+
summary += ` with range [${input.minValue ?? "-\u221E"}, ${input.maxValue ?? "\u221E"}]`;
|
|
678
|
+
} else if (input.dataType === "CATEGORICAL" && input.categories) {
|
|
679
|
+
summary += ` with ${input.categories.length} categories`;
|
|
680
|
+
}
|
|
681
|
+
return formatSuccess(response, summary);
|
|
682
|
+
}
|
|
683
|
+
});
|
|
684
|
+
|
|
685
|
+
// src/tools/score-configs/listScoreConfigs.ts
|
|
686
|
+
import { z as z12 } from "zod";
|
|
687
|
+
var inputSchema11 = z12.object({
|
|
688
|
+
page: z12.number().int().min(1).optional().describe("Page number (1-indexed)"),
|
|
689
|
+
limit: z12.number().int().min(1).max(100).optional().default(50).describe("Items per page")
|
|
690
|
+
});
|
|
691
|
+
var listScoreConfigs = defineTool({
|
|
692
|
+
name: "listScoreConfigs",
|
|
693
|
+
description: "List all score configurations in the project.",
|
|
694
|
+
inputSchema: inputSchema11,
|
|
695
|
+
handler: async (input, client, logger) => {
|
|
696
|
+
logger.debug("listScoreConfigs called with:", input);
|
|
697
|
+
const queryParams = {
|
|
698
|
+
page: input.page,
|
|
699
|
+
limit: input.limit
|
|
700
|
+
};
|
|
701
|
+
const query = buildQueryString(queryParams);
|
|
702
|
+
const response = await client.apiRequest(
|
|
703
|
+
"GET",
|
|
704
|
+
`/api/public/score-configs${query}`
|
|
705
|
+
);
|
|
706
|
+
const summary = formatPaginationSummary(
|
|
707
|
+
response.data.length,
|
|
708
|
+
response.meta.totalItems,
|
|
709
|
+
response.meta.page,
|
|
710
|
+
response.meta.page < response.meta.totalPages
|
|
711
|
+
);
|
|
712
|
+
return formatSuccess(response, summary);
|
|
713
|
+
}
|
|
714
|
+
});
|
|
715
|
+
|
|
716
|
+
// src/tools/score-configs/getScoreConfig.ts
|
|
717
|
+
import { z as z13 } from "zod";
|
|
718
|
+
var inputSchema12 = z13.object({
|
|
719
|
+
configId: z13.string().min(1).describe("The unique score config identifier")
|
|
720
|
+
});
|
|
721
|
+
var getScoreConfig = defineTool({
|
|
722
|
+
name: "getScoreConfig",
|
|
723
|
+
description: "Get a specific score configuration by ID.",
|
|
724
|
+
inputSchema: inputSchema12,
|
|
725
|
+
handler: async (input, client, logger) => {
|
|
726
|
+
logger.debug("getScoreConfig called with:", input);
|
|
727
|
+
const response = await client.apiRequest(
|
|
728
|
+
"GET",
|
|
729
|
+
`/api/public/score-configs/${encodeURIComponent(input.configId)}`
|
|
730
|
+
);
|
|
731
|
+
let summary = `Score config "${response.name}" (${response.dataType})`;
|
|
732
|
+
if (response.dataType === "NUMERIC") {
|
|
733
|
+
const min = response.minValue ?? "-\u221E";
|
|
734
|
+
const max = response.maxValue ?? "\u221E";
|
|
735
|
+
summary += ` [${min}, ${max}]`;
|
|
736
|
+
} else if (response.dataType === "CATEGORICAL" && response.categories) {
|
|
737
|
+
summary += ` with ${response.categories.length} categories`;
|
|
738
|
+
}
|
|
739
|
+
return formatSuccess(response, summary);
|
|
740
|
+
}
|
|
741
|
+
});
|
|
742
|
+
|
|
743
|
+
// src/tools/datasets/createDataset.ts
|
|
744
|
+
import { z as z14 } from "zod";
|
|
745
|
+
var inputSchema13 = z14.object({
|
|
746
|
+
name: z14.string().min(1).describe("Unique name for the dataset"),
|
|
747
|
+
description: z14.string().optional().describe("Description of the dataset"),
|
|
748
|
+
metadata: z14.record(z14.unknown()).optional().describe("Additional metadata as key-value pairs")
|
|
749
|
+
});
|
|
750
|
+
var createDataset = defineTool({
|
|
751
|
+
name: "createDataset",
|
|
752
|
+
description: "Create a new dataset for evaluation. Datasets contain items with expected inputs/outputs for testing LLM applications.",
|
|
753
|
+
inputSchema: inputSchema13,
|
|
754
|
+
handler: async (input, client, logger) => {
|
|
755
|
+
logger.debug("createDataset called with:", input);
|
|
756
|
+
const response = await client.apiRequest(
|
|
757
|
+
"POST",
|
|
758
|
+
"/api/public/v2/datasets",
|
|
759
|
+
{
|
|
760
|
+
name: input.name,
|
|
761
|
+
description: input.description,
|
|
762
|
+
metadata: input.metadata
|
|
763
|
+
}
|
|
764
|
+
);
|
|
765
|
+
const summary = `Created dataset "${input.name}"${input.description ? `: ${input.description}` : ""}`;
|
|
766
|
+
return formatSuccess(response, summary);
|
|
767
|
+
}
|
|
768
|
+
});
|
|
769
|
+
|
|
770
|
+
// src/tools/datasets/listDatasets.ts
|
|
771
|
+
import { z as z15 } from "zod";
|
|
772
|
+
var inputSchema14 = z15.object({
|
|
773
|
+
page: z15.number().int().min(1).optional().describe("Page number (1-indexed)"),
|
|
774
|
+
limit: z15.number().int().min(1).max(100).optional().default(50).describe("Items per page")
|
|
775
|
+
});
|
|
776
|
+
var listDatasets = defineTool({
|
|
777
|
+
name: "listDatasets",
|
|
778
|
+
description: "List all datasets in the project.",
|
|
779
|
+
inputSchema: inputSchema14,
|
|
780
|
+
handler: async (input, client, logger) => {
|
|
781
|
+
logger.debug("listDatasets called with:", input);
|
|
782
|
+
const queryParams = {
|
|
783
|
+
page: input.page,
|
|
784
|
+
limit: input.limit
|
|
785
|
+
};
|
|
786
|
+
const query = buildQueryString(queryParams);
|
|
787
|
+
const response = await client.apiRequest(
|
|
788
|
+
"GET",
|
|
789
|
+
`/api/public/v2/datasets${query}`
|
|
790
|
+
);
|
|
791
|
+
const summary = formatPaginationSummary(
|
|
792
|
+
response.data.length,
|
|
793
|
+
response.meta.totalItems,
|
|
794
|
+
response.meta.page,
|
|
795
|
+
response.meta.page < response.meta.totalPages
|
|
796
|
+
);
|
|
797
|
+
return formatSuccess(response, summary);
|
|
798
|
+
}
|
|
799
|
+
});
|
|
800
|
+
|
|
801
|
+
// src/tools/datasets/getDataset.ts
|
|
802
|
+
import { z as z16 } from "zod";
|
|
803
|
+
var inputSchema15 = z16.object({
|
|
804
|
+
datasetName: z16.string().min(1).describe("The name of the dataset to retrieve")
|
|
805
|
+
});
|
|
806
|
+
var getDataset = defineTool({
|
|
807
|
+
name: "getDataset",
|
|
808
|
+
description: "Get a dataset by name including its items and run names.",
|
|
809
|
+
inputSchema: inputSchema15,
|
|
810
|
+
handler: async (input, client, logger) => {
|
|
811
|
+
logger.debug("getDataset called with:", input);
|
|
812
|
+
const response = await client.apiRequest(
|
|
813
|
+
"GET",
|
|
814
|
+
`/api/public/v2/datasets/${encodeURIComponent(input.datasetName)}`
|
|
815
|
+
);
|
|
816
|
+
const itemCount = response.items?.length ?? 0;
|
|
817
|
+
const runCount = response.runs?.length ?? 0;
|
|
818
|
+
const summary = `Dataset "${response.name}" with ${itemCount} items and ${runCount} runs`;
|
|
819
|
+
return formatSuccess(response, summary);
|
|
820
|
+
}
|
|
821
|
+
});
|
|
822
|
+
|
|
823
|
+
// src/tools/datasets/createDatasetItem.ts
|
|
824
|
+
import { z as z17 } from "zod";
|
|
825
|
+
var inputSchema16 = z17.object({
|
|
826
|
+
datasetName: z17.string().min(1).describe("Name of the dataset to add the item to"),
|
|
827
|
+
input: z17.unknown().describe("The input for this dataset item (any JSON value)"),
|
|
828
|
+
expectedOutput: z17.unknown().optional().describe("The expected output for evaluation"),
|
|
829
|
+
metadata: z17.record(z17.unknown()).optional().describe("Additional metadata"),
|
|
830
|
+
sourceTraceId: z17.string().optional().describe("Link to source trace that generated this item"),
|
|
831
|
+
sourceObservationId: z17.string().optional().describe("Link to source observation (requires sourceTraceId)"),
|
|
832
|
+
id: z17.string().optional().describe("Custom ID for idempotent upsert"),
|
|
833
|
+
status: z17.enum(["ACTIVE", "ARCHIVED"]).optional().describe("Item status (defaults to ACTIVE)")
|
|
834
|
+
});
|
|
835
|
+
var createDatasetItem = defineTool({
|
|
836
|
+
name: "createDatasetItem",
|
|
837
|
+
description: "Create or upsert a dataset item. Use id parameter for idempotent updates.",
|
|
838
|
+
inputSchema: inputSchema16,
|
|
839
|
+
handler: async (input, client, logger) => {
|
|
840
|
+
logger.debug("createDatasetItem called with:", input);
|
|
841
|
+
const response = await client.apiRequest(
|
|
842
|
+
"POST",
|
|
843
|
+
"/api/public/dataset-items",
|
|
844
|
+
{
|
|
845
|
+
datasetName: input.datasetName,
|
|
846
|
+
input: input.input,
|
|
847
|
+
expectedOutput: input.expectedOutput,
|
|
848
|
+
metadata: input.metadata,
|
|
849
|
+
sourceTraceId: input.sourceTraceId,
|
|
850
|
+
sourceObservationId: input.sourceObservationId,
|
|
851
|
+
id: input.id,
|
|
852
|
+
status: input.status
|
|
853
|
+
}
|
|
854
|
+
);
|
|
855
|
+
const summary = `Created dataset item ${response.id} in "${input.datasetName}"`;
|
|
856
|
+
return formatSuccess(response, summary);
|
|
857
|
+
}
|
|
858
|
+
});
|
|
859
|
+
|
|
860
|
+
// src/tools/datasets/listDatasetItems.ts
|
|
861
|
+
import { z as z18 } from "zod";
|
|
862
|
+
var inputSchema17 = z18.object({
|
|
863
|
+
datasetName: z18.string().min(1).describe("Name of the dataset to list items from"),
|
|
864
|
+
page: z18.number().int().min(1).optional().describe("Page number (1-indexed)"),
|
|
865
|
+
limit: z18.number().int().min(1).max(100).optional().default(50).describe("Items per page"),
|
|
866
|
+
sourceTraceId: z18.string().optional().describe("Filter by source trace ID"),
|
|
867
|
+
sourceObservationId: z18.string().optional().describe("Filter by source observation ID")
|
|
868
|
+
});
|
|
869
|
+
var listDatasetItems = defineTool({
|
|
870
|
+
name: "listDatasetItems",
|
|
871
|
+
description: "List items in a dataset with optional filtering.",
|
|
872
|
+
inputSchema: inputSchema17,
|
|
873
|
+
handler: async (input, client, logger) => {
|
|
874
|
+
logger.debug("listDatasetItems called with:", input);
|
|
875
|
+
const queryParams = {
|
|
876
|
+
datasetName: input.datasetName,
|
|
877
|
+
page: input.page,
|
|
878
|
+
limit: input.limit,
|
|
879
|
+
sourceTraceId: input.sourceTraceId,
|
|
880
|
+
sourceObservationId: input.sourceObservationId
|
|
881
|
+
};
|
|
882
|
+
const query = buildQueryString(queryParams);
|
|
883
|
+
const response = await client.apiRequest(
|
|
884
|
+
"GET",
|
|
885
|
+
`/api/public/dataset-items${query}`
|
|
886
|
+
);
|
|
887
|
+
const summary = formatPaginationSummary(
|
|
888
|
+
response.data.length,
|
|
889
|
+
response.meta.totalItems,
|
|
890
|
+
response.meta.page,
|
|
891
|
+
response.meta.page < response.meta.totalPages
|
|
892
|
+
);
|
|
893
|
+
return formatSuccess(response, summary);
|
|
894
|
+
}
|
|
895
|
+
});
|
|
896
|
+
|
|
897
|
+
// src/tools/datasets/getDatasetItem.ts
|
|
898
|
+
import { z as z19 } from "zod";
|
|
899
|
+
var inputSchema18 = z19.object({
|
|
900
|
+
datasetItemId: z19.string().min(1).describe("The unique dataset item identifier")
|
|
901
|
+
});
|
|
902
|
+
var getDatasetItem = defineTool({
|
|
903
|
+
name: "getDatasetItem",
|
|
904
|
+
description: "Get a specific dataset item by ID.",
|
|
905
|
+
inputSchema: inputSchema18,
|
|
906
|
+
handler: async (input, client, logger) => {
|
|
907
|
+
logger.debug("getDatasetItem called with:", input);
|
|
908
|
+
const response = await client.apiRequest(
|
|
909
|
+
"GET",
|
|
910
|
+
`/api/public/dataset-items/${encodeURIComponent(input.datasetItemId)}`
|
|
911
|
+
);
|
|
912
|
+
const summary = `Dataset item ${response.id} from "${response.datasetName}" (${response.status})`;
|
|
913
|
+
return formatSuccess(response, summary);
|
|
914
|
+
}
|
|
915
|
+
});
|
|
916
|
+
|
|
917
|
+
// src/tools/datasets/deleteDatasetItem.ts
|
|
918
|
+
import { z as z20 } from "zod";
|
|
919
|
+
var inputSchema19 = z20.object({
|
|
920
|
+
datasetItemId: z20.string().min(1).describe("The unique dataset item identifier to delete")
|
|
921
|
+
});
|
|
922
|
+
var deleteDatasetItem = defineTool({
|
|
923
|
+
name: "deleteDatasetItem",
|
|
924
|
+
description: "Delete a specific dataset item. This action is irreversible.",
|
|
925
|
+
inputSchema: inputSchema19,
|
|
926
|
+
handler: async (input, client, logger) => {
|
|
927
|
+
logger.debug("deleteDatasetItem called with:", input);
|
|
928
|
+
await client.apiRequest(
|
|
929
|
+
"DELETE",
|
|
930
|
+
`/api/public/dataset-items/${encodeURIComponent(input.datasetItemId)}`
|
|
931
|
+
);
|
|
932
|
+
return formatSuccess(
|
|
933
|
+
{ deleted: true, datasetItemId: input.datasetItemId },
|
|
934
|
+
`Successfully deleted dataset item: ${input.datasetItemId}`
|
|
935
|
+
);
|
|
936
|
+
}
|
|
937
|
+
});
|
|
938
|
+
|
|
939
|
+
// src/tools/datasets/createDatasetRunItem.ts
|
|
940
|
+
import { z as z21 } from "zod";
|
|
941
|
+
var inputSchema20 = z21.object({
|
|
942
|
+
runName: z21.string().min(1).describe("Name of the dataset run"),
|
|
943
|
+
runDescription: z21.string().optional().describe("Description of the run"),
|
|
944
|
+
datasetItemId: z21.string().min(1).describe("ID of the dataset item being evaluated"),
|
|
945
|
+
traceId: z21.string().min(1).describe("ID of the trace that processed this item"),
|
|
946
|
+
observationId: z21.string().optional().describe("ID of specific observation (if applicable)"),
|
|
947
|
+
metadata: z21.record(z21.unknown()).optional().describe("Additional run item metadata")
|
|
948
|
+
});
|
|
949
|
+
var createDatasetRunItem = defineTool({
|
|
950
|
+
name: "createDatasetRunItem",
|
|
951
|
+
description: "Create a dataset run item linking a trace/observation to a dataset item for evaluation.",
|
|
952
|
+
inputSchema: inputSchema20,
|
|
953
|
+
handler: async (input, client, logger) => {
|
|
954
|
+
logger.debug("createDatasetRunItem called with:", input);
|
|
955
|
+
const response = await client.apiRequest(
|
|
956
|
+
"POST",
|
|
957
|
+
"/api/public/dataset-run-items",
|
|
958
|
+
{
|
|
959
|
+
runName: input.runName,
|
|
960
|
+
runDescription: input.runDescription,
|
|
961
|
+
datasetItemId: input.datasetItemId,
|
|
962
|
+
traceId: input.traceId,
|
|
963
|
+
observationId: input.observationId,
|
|
964
|
+
metadata: input.metadata
|
|
965
|
+
}
|
|
966
|
+
);
|
|
967
|
+
const summary = `Created run item in "${input.runName}" linking trace ${input.traceId} to item ${input.datasetItemId}`;
|
|
968
|
+
return formatSuccess(response, summary);
|
|
969
|
+
}
|
|
970
|
+
});
|
|
971
|
+
|
|
972
|
+
// src/tools/datasets/listDatasetRuns.ts
|
|
973
|
+
import { z as z22 } from "zod";
|
|
974
|
+
var inputSchema21 = z22.object({
|
|
975
|
+
datasetName: z22.string().min(1).describe("Name of the dataset to list runs from"),
|
|
976
|
+
page: z22.number().int().min(1).optional().describe("Page number (1-indexed)"),
|
|
977
|
+
limit: z22.number().int().min(1).max(100).optional().default(50).describe("Items per page")
|
|
978
|
+
});
|
|
979
|
+
var listDatasetRuns = defineTool({
|
|
980
|
+
name: "listDatasetRuns",
|
|
981
|
+
description: "List all runs for a specific dataset.",
|
|
982
|
+
inputSchema: inputSchema21,
|
|
983
|
+
handler: async (input, client, logger) => {
|
|
984
|
+
logger.debug("listDatasetRuns called with:", input);
|
|
985
|
+
const queryParams = {
|
|
986
|
+
page: input.page,
|
|
987
|
+
limit: input.limit
|
|
988
|
+
};
|
|
989
|
+
const query = buildQueryString(queryParams);
|
|
990
|
+
const response = await client.apiRequest(
|
|
991
|
+
"GET",
|
|
992
|
+
`/api/public/datasets/${encodeURIComponent(input.datasetName)}/runs${query}`
|
|
993
|
+
);
|
|
994
|
+
const summary = formatPaginationSummary(
|
|
995
|
+
response.data.length,
|
|
996
|
+
response.meta.totalItems,
|
|
997
|
+
response.meta.page,
|
|
998
|
+
response.meta.page < response.meta.totalPages
|
|
999
|
+
);
|
|
1000
|
+
return formatSuccess(response, summary);
|
|
1001
|
+
}
|
|
1002
|
+
});
|
|
1003
|
+
|
|
1004
|
+
// src/tools/datasets/getDatasetRun.ts
|
|
1005
|
+
import { z as z23 } from "zod";
|
|
1006
|
+
var inputSchema22 = z23.object({
|
|
1007
|
+
datasetName: z23.string().min(1).describe("Name of the dataset"),
|
|
1008
|
+
runName: z23.string().min(1).describe("Name of the run to retrieve")
|
|
1009
|
+
});
|
|
1010
|
+
var getDatasetRun = defineTool({
|
|
1011
|
+
name: "getDatasetRun",
|
|
1012
|
+
description: "Get a specific dataset run by name including its run items.",
|
|
1013
|
+
inputSchema: inputSchema22,
|
|
1014
|
+
handler: async (input, client, logger) => {
|
|
1015
|
+
logger.debug("getDatasetRun called with:", input);
|
|
1016
|
+
const response = await client.apiRequest(
|
|
1017
|
+
"GET",
|
|
1018
|
+
`/api/public/datasets/${encodeURIComponent(input.datasetName)}/runs/${encodeURIComponent(input.runName)}`
|
|
1019
|
+
);
|
|
1020
|
+
const itemCount = response.datasetRunItems?.length ?? 0;
|
|
1021
|
+
const summary = `Run "${response.name}" with ${itemCount} items`;
|
|
1022
|
+
return formatSuccess(response, summary);
|
|
1023
|
+
}
|
|
1024
|
+
});
|
|
1025
|
+
|
|
1026
|
+
// src/tools/sessions/listSessions.ts
|
|
1027
|
+
import { z as z24 } from "zod";
|
|
1028
|
+
var inputSchema23 = z24.object({
|
|
1029
|
+
page: z24.number().int().min(1).optional().describe("Page number (1-indexed)"),
|
|
1030
|
+
limit: z24.number().int().min(1).max(100).optional().default(50).describe("Items per page"),
|
|
1031
|
+
fromTimestamp: z24.string().optional().describe("Start of time range (ISO 8601)"),
|
|
1032
|
+
toTimestamp: z24.string().optional().describe("End of time range (ISO 8601)")
|
|
1033
|
+
});
|
|
1034
|
+
var listSessions = defineTool({
|
|
1035
|
+
name: "listSessions",
|
|
1036
|
+
description: "List all sessions. Sessions group multiple traces from the same user interaction.",
|
|
1037
|
+
inputSchema: inputSchema23,
|
|
1038
|
+
handler: async (input, client, logger) => {
|
|
1039
|
+
logger.debug("listSessions called with:", input);
|
|
1040
|
+
const queryParams = {
|
|
1041
|
+
page: input.page,
|
|
1042
|
+
limit: input.limit,
|
|
1043
|
+
fromTimestamp: input.fromTimestamp,
|
|
1044
|
+
toTimestamp: input.toTimestamp
|
|
1045
|
+
};
|
|
1046
|
+
const query = buildQueryString(queryParams);
|
|
1047
|
+
const response = await client.apiRequest(
|
|
1048
|
+
"GET",
|
|
1049
|
+
`/api/public/sessions${query}`
|
|
1050
|
+
);
|
|
1051
|
+
const summary = formatPaginationSummary(
|
|
1052
|
+
response.data.length,
|
|
1053
|
+
response.meta.totalItems,
|
|
1054
|
+
response.meta.page,
|
|
1055
|
+
response.meta.page < response.meta.totalPages
|
|
1056
|
+
);
|
|
1057
|
+
const result = {
|
|
1058
|
+
...response,
|
|
1059
|
+
nextCursor: response.meta.page < response.meta.totalPages ? String(response.meta.page + 1) : void 0
|
|
1060
|
+
};
|
|
1061
|
+
return formatSuccess(result, summary);
|
|
1062
|
+
}
|
|
1063
|
+
});
|
|
1064
|
+
|
|
1065
|
+
// src/tools/sessions/getSession.ts
|
|
1066
|
+
import { z as z25 } from "zod";
|
|
1067
|
+
var inputSchema24 = z25.object({
|
|
1068
|
+
sessionId: z25.string().min(1).describe("The unique session identifier")
|
|
1069
|
+
});
|
|
1070
|
+
var getSession = defineTool({
|
|
1071
|
+
name: "getSession",
|
|
1072
|
+
description: "Get a specific session by ID including its traces.",
|
|
1073
|
+
inputSchema: inputSchema24,
|
|
1074
|
+
handler: async (input, client, logger) => {
|
|
1075
|
+
logger.debug("getSession called with:", input);
|
|
1076
|
+
const response = await client.apiRequest(
|
|
1077
|
+
"GET",
|
|
1078
|
+
`/api/public/sessions/${encodeURIComponent(input.sessionId)}`
|
|
1079
|
+
);
|
|
1080
|
+
const traceCount = response.traces?.length ?? 0;
|
|
1081
|
+
const summary = `Session ${response.id} with ${traceCount} traces`;
|
|
1082
|
+
return formatSuccess(response, summary);
|
|
1083
|
+
}
|
|
1084
|
+
});
|
|
1085
|
+
|
|
1086
|
+
// src/tools/index.ts
|
|
1087
|
+
var allTools = {
|
|
1088
|
+
traces: [listTraces, getTrace, deleteTrace],
|
|
1089
|
+
observations: [listObservations, getObservation],
|
|
1090
|
+
scores: [createScore, listScores, getScore, deleteScore],
|
|
1091
|
+
scoreConfigs: [createScoreConfig, listScoreConfigs, getScoreConfig],
|
|
1092
|
+
datasets: [
|
|
1093
|
+
createDataset,
|
|
1094
|
+
listDatasets,
|
|
1095
|
+
getDataset,
|
|
1096
|
+
createDatasetItem,
|
|
1097
|
+
listDatasetItems,
|
|
1098
|
+
getDatasetItem,
|
|
1099
|
+
deleteDatasetItem,
|
|
1100
|
+
createDatasetRunItem,
|
|
1101
|
+
listDatasetRuns,
|
|
1102
|
+
getDatasetRun
|
|
1103
|
+
],
|
|
1104
|
+
sessions: [listSessions, getSession]
|
|
1105
|
+
// prompts: [], // TODO: Phase 5 (optional)
|
|
1106
|
+
};
|
|
1107
|
+
function getAllTools() {
|
|
1108
|
+
return Object.values(allTools).flat();
|
|
1109
|
+
}
|
|
1110
|
+
|
|
1111
|
+
// src/server.ts
|
|
1112
|
+
var SERVER_NAME = "langfuse-mcp-server";
|
|
1113
|
+
var SERVER_VERSION = "0.1.0";
|
|
1114
|
+
function createServer(config, logger) {
|
|
1115
|
+
const langfuseClient = new LangfuseClient(config, logger);
|
|
1116
|
+
const toolRegistry = new ToolRegistry(langfuseClient, config, logger);
|
|
1117
|
+
toolRegistry.registerAll(getAllTools());
|
|
1118
|
+
const server = new Server(
|
|
1119
|
+
{
|
|
1120
|
+
name: SERVER_NAME,
|
|
1121
|
+
version: SERVER_VERSION
|
|
1122
|
+
},
|
|
1123
|
+
{
|
|
1124
|
+
capabilities: {
|
|
1125
|
+
tools: {}
|
|
1126
|
+
}
|
|
1127
|
+
}
|
|
1128
|
+
);
|
|
1129
|
+
server.setRequestHandler(ListToolsRequestSchema, async () => {
|
|
1130
|
+
logger.debug("Received list tools request");
|
|
1131
|
+
return {
|
|
1132
|
+
tools: toolRegistry.getTools()
|
|
1133
|
+
};
|
|
1134
|
+
});
|
|
1135
|
+
server.setRequestHandler(CallToolRequestSchema, async (request) => {
|
|
1136
|
+
const { name, arguments: args } = request.params;
|
|
1137
|
+
logger.info(`Tool called: ${name}`);
|
|
1138
|
+
const result = await toolRegistry.execute(name, args ?? {});
|
|
1139
|
+
return result;
|
|
1140
|
+
});
|
|
1141
|
+
server.onerror = (error) => {
|
|
1142
|
+
logger.error("Server error:", error);
|
|
1143
|
+
};
|
|
1144
|
+
return {
|
|
1145
|
+
server,
|
|
1146
|
+
langfuseClient,
|
|
1147
|
+
toolRegistry
|
|
1148
|
+
};
|
|
1149
|
+
}
|
|
1150
|
+
async function runServer(config, logger) {
|
|
1151
|
+
logger.info(`Starting ${SERVER_NAME} v${SERVER_VERSION}`);
|
|
1152
|
+
logger.info(`Langfuse URL: ${config.langfuse.baseUrl}`);
|
|
1153
|
+
const { server, langfuseClient } = createServer(config, logger);
|
|
1154
|
+
const transport = new StdioServerTransport();
|
|
1155
|
+
const shutdown = async () => {
|
|
1156
|
+
logger.info("Shutting down server...");
|
|
1157
|
+
await langfuseClient.shutdown();
|
|
1158
|
+
await server.close();
|
|
1159
|
+
process.exit(0);
|
|
1160
|
+
};
|
|
1161
|
+
process.on("SIGINT", shutdown);
|
|
1162
|
+
process.on("SIGTERM", shutdown);
|
|
1163
|
+
await server.connect(transport);
|
|
1164
|
+
logger.info("Server connected and ready");
|
|
1165
|
+
}
|
|
1166
|
+
|
|
1167
|
+
// src/index.ts
|
|
1168
|
+
async function main() {
|
|
1169
|
+
try {
|
|
1170
|
+
const config = loadConfig();
|
|
1171
|
+
const logger = createLogger(config);
|
|
1172
|
+
await runServer(config, logger);
|
|
1173
|
+
} catch (error) {
|
|
1174
|
+
console.error("Failed to start server:", error);
|
|
1175
|
+
process.exit(1);
|
|
1176
|
+
}
|
|
1177
|
+
}
|
|
1178
|
+
main().catch(console.error);
|
|
1179
|
+
//# sourceMappingURL=index.js.map
|