@inspectr/mcplab-mcp-server 1.2.3 → 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -0
- package/dist/runtime.contracts.test.d.ts +2 -0
- package/dist/runtime.contracts.test.d.ts.map +1 -0
- package/dist/runtime.contracts.test.js +187 -0
- package/dist/runtime.contracts.test.js.map +1 -0
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +385 -363
- package/dist/runtime.js.map +1 -1
- package/package.json +9 -5
package/dist/runtime.js
CHANGED
|
@@ -5,7 +5,7 @@ import { basename, dirname, extname, resolve, join, sep, relative } from 'node:p
|
|
|
5
5
|
import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
|
6
6
|
import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js';
|
|
7
7
|
import { isInitializeRequest } from '@modelcontextprotocol/sdk/types.js';
|
|
8
|
-
import { loadConfig, runAll, selectScenarios } from '@inspectr/mcplab-core';
|
|
8
|
+
import { loadConfig, runAll, selectScenarios, loadOrBuildSearchIndex, indexNeedsRefresh, getResultsIndexPaths, resolveRunArtifactPath, searchDocs, getContext } from '@inspectr/mcplab-core';
|
|
9
9
|
import { renderReport } from '@inspectr/mcplab-reporting';
|
|
10
10
|
import { parse as parseYaml, stringify as stringifyYaml } from 'yaml';
|
|
11
11
|
import { z } from 'zod';
|
|
@@ -15,6 +15,7 @@ const SERVER_VERSION = typeof PACKAGE_JSON.version === 'string' && PACKAGE_JSON.
|
|
|
15
15
|
? PACKAGE_JSON.version
|
|
16
16
|
: '0.0.0';
|
|
17
17
|
const SERVER_ICON_URL = 'https://mcplab.inspectr.dev/favicon.svg';
|
|
18
|
+
const SERVER_OWNED_ROOTS = resolveServerOwnedRoots();
|
|
18
19
|
const DEFAULT_MCP_PATH = '/mcp';
|
|
19
20
|
const DEFAULT_MCP_PORT = 3011;
|
|
20
21
|
const DEFAULT_MCP_HOST = '127.0.0.1';
|
|
@@ -36,6 +37,8 @@ const ResultsMetadataSchema = z
|
|
|
36
37
|
mcp_server_versions: z.record(z.string())
|
|
37
38
|
})
|
|
38
39
|
.passthrough();
|
|
40
|
+
const ResultsQueryStatusSchema = z.enum(['passed', 'failed', 'all']);
|
|
41
|
+
const ResultsQuerySourceSchema = z.enum(['results', 'trace', 'summary']);
|
|
39
42
|
const MetricSummarySchema = z.object({
|
|
40
43
|
total_runs: z.number().int().nonnegative(),
|
|
41
44
|
passed_runs: z.number().int().nonnegative(),
|
|
@@ -97,10 +100,28 @@ const AgentEntrySchema = z.object({
|
|
|
97
100
|
max_tokens: z.number().int().positive().optional(),
|
|
98
101
|
system: z.string().optional()
|
|
99
102
|
});
|
|
103
|
+
const LibraryServerEntryContentSchema = z
|
|
104
|
+
.object({
|
|
105
|
+
transport: z.string().optional(),
|
|
106
|
+
url: z.string().optional(),
|
|
107
|
+
auth: GenericObjectSchema.optional(),
|
|
108
|
+
name: z.string().optional(),
|
|
109
|
+
description: z.string().optional(),
|
|
110
|
+
tags: z.array(z.string()).optional()
|
|
111
|
+
})
|
|
112
|
+
.passthrough();
|
|
113
|
+
const LibraryServerEntrySchema = z.object({
|
|
114
|
+
id: z.string(),
|
|
115
|
+
entry: LibraryServerEntryContentSchema.optional()
|
|
116
|
+
});
|
|
117
|
+
const LibraryAgentEntrySchema = z.object({
|
|
118
|
+
id: z.string(),
|
|
119
|
+
entry: AgentEntrySchema.optional()
|
|
120
|
+
});
|
|
100
121
|
const LibraryEntrySchema = z.object({
|
|
101
122
|
bundleRoot: z.string(),
|
|
102
|
-
servers: z.
|
|
103
|
-
agents: z.
|
|
123
|
+
servers: z.array(LibraryServerEntrySchema),
|
|
124
|
+
agents: z.array(LibraryAgentEntrySchema),
|
|
104
125
|
scenarios: z.array(LibraryScenarioEntrySchema)
|
|
105
126
|
});
|
|
106
127
|
const ServerAuthSchema = z.union([
|
|
@@ -155,6 +176,41 @@ const ConfigSummarySchema = z.object({
|
|
|
155
176
|
extract_count: z.number().int().nonnegative()
|
|
156
177
|
}))
|
|
157
178
|
});
|
|
179
|
+
const ResolvedScenarioSchema = z.object({
|
|
180
|
+
id: z.string(),
|
|
181
|
+
servers: z.array(z.string()),
|
|
182
|
+
agent: z.string().optional(),
|
|
183
|
+
prompt: z.string().optional(),
|
|
184
|
+
eval: z
|
|
185
|
+
.object({
|
|
186
|
+
type: z.string(),
|
|
187
|
+
assertions: z.array(z.string()).optional(),
|
|
188
|
+
rubric: GenericObjectSchema.optional()
|
|
189
|
+
})
|
|
190
|
+
.passthrough()
|
|
191
|
+
.optional(),
|
|
192
|
+
extract: z
|
|
193
|
+
.array(z
|
|
194
|
+
.object({
|
|
195
|
+
name: z.string(),
|
|
196
|
+
from: z.string().optional(),
|
|
197
|
+
regex: z.string().optional(),
|
|
198
|
+
path: z.string().optional(),
|
|
199
|
+
expression: z.string().optional(),
|
|
200
|
+
transform: z.string().optional()
|
|
201
|
+
})
|
|
202
|
+
.passthrough())
|
|
203
|
+
.optional()
|
|
204
|
+
});
|
|
205
|
+
const RunDefaultsSchema = z
|
|
206
|
+
.object({
|
|
207
|
+
selected_agents: z.array(z.string()).optional(),
|
|
208
|
+
runs_per_scenario: z.number().int().positive().optional(),
|
|
209
|
+
timeout_ms: z.number().int().positive().optional(),
|
|
210
|
+
retries: z.number().int().nonnegative().optional(),
|
|
211
|
+
concurrency: z.number().int().positive().optional()
|
|
212
|
+
})
|
|
213
|
+
.passthrough();
|
|
158
214
|
const ToolAnalysisListItemSchema = z.object({
|
|
159
215
|
report_id: z.string(),
|
|
160
216
|
path: z.string().optional(),
|
|
@@ -280,37 +336,98 @@ const WriteMarkdownReportSuccessSchema = z.object({
|
|
|
280
336
|
overwritten: z.boolean().describe('True when an existing file was replaced.'),
|
|
281
337
|
workspace_root: z.string().describe('Workspace root used for path safety validation.')
|
|
282
338
|
});
|
|
339
|
+
const WriteMarkdownReportErrorCodeSchema = z.enum([
|
|
340
|
+
'PATH_ESCAPE',
|
|
341
|
+
'PERMISSION_DENIED',
|
|
342
|
+
'FILE_EXISTS',
|
|
343
|
+
'INVALID_EXTENSION',
|
|
344
|
+
'PARENT_DIR_MISSING',
|
|
345
|
+
'IO_ERROR'
|
|
346
|
+
]);
|
|
283
347
|
const WriteMarkdownReportErrorSchema = z.object({
|
|
284
348
|
ok: z.literal(false),
|
|
285
|
-
error_code:
|
|
286
|
-
'PATH_ESCAPE',
|
|
287
|
-
'PERMISSION_DENIED',
|
|
288
|
-
'FILE_EXISTS',
|
|
289
|
-
'INVALID_EXTENSION',
|
|
290
|
-
'PARENT_DIR_MISSING',
|
|
291
|
-
'IO_ERROR'
|
|
292
|
-
]),
|
|
349
|
+
error_code: WriteMarkdownReportErrorCodeSchema,
|
|
293
350
|
error_message: z.string(),
|
|
294
351
|
attempted_path: z.string().optional(),
|
|
295
352
|
violated_constraint: z.string().optional()
|
|
296
353
|
});
|
|
297
|
-
const
|
|
298
|
-
.
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
354
|
+
const WriteMarkdownReportOutputSchema = z.object({
|
|
355
|
+
ok: z.boolean(),
|
|
356
|
+
path: z.string().optional(),
|
|
357
|
+
bytes: z.number().int().nonnegative().optional(),
|
|
358
|
+
chars: z.number().int().nonnegative().optional(),
|
|
359
|
+
overwritten: z.boolean().optional(),
|
|
360
|
+
workspace_root: z.string().optional(),
|
|
361
|
+
error_code: WriteMarkdownReportErrorCodeSchema.optional(),
|
|
362
|
+
error_message: z.string().optional(),
|
|
363
|
+
attempted_path: z.string().optional(),
|
|
364
|
+
violated_constraint: z.string().optional()
|
|
365
|
+
});
|
|
366
|
+
const SafeRelativePathSchema = z
|
|
367
|
+
.string()
|
|
368
|
+
.max(200)
|
|
369
|
+
.regex(/^(?!\/)(?![A-Za-z]:)(?!.*(?:^|\/)\.\.(?:\/|$))[\w./-]+$/, 'Must be a relative workspace path without ".." segments or absolute prefixes.');
|
|
370
|
+
const GenerateServerEntryInputCoreSchema = z.object({
|
|
371
|
+
id: z.string().describe('Server id key (kebab-case recommended).'),
|
|
372
|
+
url: z.string().describe('MCP server URL (Streamable HTTP endpoint).'),
|
|
373
|
+
transport: z.enum(['http']).optional().describe('MCPLab transport type (currently http).')
|
|
374
|
+
});
|
|
375
|
+
const GenerateServerEntryInputBaseSchema = GenerateServerEntryInputCoreSchema.extend({
|
|
376
|
+
auth_type: z
|
|
377
|
+
.enum(['none', 'bearer', 'api_key', 'oauth_client_credentials'])
|
|
378
|
+
.optional()
|
|
379
|
+
.describe('Authentication mode.'),
|
|
380
|
+
bearer_token: z
|
|
381
|
+
.string()
|
|
382
|
+
.optional()
|
|
383
|
+
.describe('Direct bearer token value or ${VAR} env reference when auth_type=bearer.'),
|
|
384
|
+
bearer_env: z.string().optional().describe('Env var for bearer token when auth_type=bearer.'),
|
|
385
|
+
api_key_header_name: z
|
|
386
|
+
.string()
|
|
387
|
+
.optional()
|
|
388
|
+
.describe('Header name for API key auth (default: X-API-Key).'),
|
|
389
|
+
api_key_value: z
|
|
390
|
+
.string()
|
|
391
|
+
.optional()
|
|
392
|
+
.describe('API key value or ${VAR} env reference when auth_type=api_key.'),
|
|
393
|
+
oauth_token_url: z
|
|
394
|
+
.string()
|
|
395
|
+
.optional()
|
|
396
|
+
.describe('OAuth token URL when auth_type=oauth_client_credentials.'),
|
|
397
|
+
oauth_client_id_env: z.string().optional().describe('OAuth client id env var.'),
|
|
398
|
+
oauth_client_secret_env: z.string().optional().describe('OAuth client secret env var.'),
|
|
399
|
+
oauth_scope: z.string().optional().describe('Optional OAuth scope.'),
|
|
400
|
+
oauth_audience: z.string().optional().describe('Optional OAuth audience.')
|
|
401
|
+
});
|
|
402
|
+
const GenerateServerEntryPublicInputSchema = z.union([
|
|
403
|
+
GenerateServerEntryInputCoreSchema.extend({
|
|
404
|
+
auth_type: z.enum(['none']).optional()
|
|
405
|
+
}),
|
|
406
|
+
GenerateServerEntryInputCoreSchema.extend({
|
|
407
|
+
auth_type: z.literal('bearer'),
|
|
408
|
+
bearer_token: z.string(),
|
|
409
|
+
bearer_env: z.string().optional()
|
|
410
|
+
}),
|
|
411
|
+
GenerateServerEntryInputCoreSchema.extend({
|
|
412
|
+
auth_type: z.literal('bearer'),
|
|
413
|
+
bearer_token: z.string().optional(),
|
|
414
|
+
bearer_env: z.string()
|
|
415
|
+
}),
|
|
416
|
+
GenerateServerEntryInputCoreSchema.extend({
|
|
417
|
+
auth_type: z.literal('api_key'),
|
|
418
|
+
api_key_header_name: z.string().optional(),
|
|
419
|
+
api_key_value: z.string()
|
|
420
|
+
}),
|
|
421
|
+
GenerateServerEntryInputCoreSchema.extend({
|
|
422
|
+
auth_type: z.literal('oauth_client_credentials'),
|
|
423
|
+
oauth_token_url: z.string(),
|
|
424
|
+
oauth_client_id_env: z.string(),
|
|
425
|
+
oauth_client_secret_env: z.string(),
|
|
426
|
+
oauth_scope: z.string().optional(),
|
|
427
|
+
oauth_audience: z.string().optional()
|
|
428
|
+
})
|
|
429
|
+
]);
|
|
430
|
+
const GenerateServerEntryInputSchema = GenerateServerEntryInputBaseSchema.superRefine((value, ctx) => {
|
|
314
431
|
const authType = value.auth_type ?? 'none';
|
|
315
432
|
if (authType === 'bearer') {
|
|
316
433
|
if (!value.bearer_token && !value.bearer_env) {
|
|
@@ -433,10 +550,12 @@ export function registerTools(server) {
|
|
|
433
550
|
};
|
|
434
551
|
registerTool('mcplab_write_markdown_report', {
|
|
435
552
|
description: 'Write a Markdown (.md or .markdown) file to a path within the current workspace. Returns structured output with ok:true and the resolved path on success. On failure, returns ok:false with an error_code from: PATH_ESCAPE (path traversal attempt), FILE_EXISTS (file already exists and overwrite is false), PERMISSION_DENIED, PARENT_DIR_MISSING (create_dirs is false and parent does not exist), INVALID_EXTENSION (not .md or .markdown), IO_ERROR.',
|
|
436
|
-
outputSchema:
|
|
553
|
+
outputSchema: WriteMarkdownReportOutputSchema,
|
|
437
554
|
inputSchema: {
|
|
438
555
|
output_path: z
|
|
439
556
|
.string()
|
|
557
|
+
.max(200)
|
|
558
|
+
.regex(/^[^\0]+\.(?:md|markdown)$/i, 'output_path must end with .md or .markdown.')
|
|
440
559
|
.describe("Target .md/.markdown path. Use a relative path (e.g. mcplab/reports/my-report.md) — relative paths are always safe and resolve against the server's working directory (process.cwd()). Absolute paths are accepted only if they stay inside that directory; any path that escapes it is rejected with error_code PATH_ESCAPE."),
|
|
441
560
|
markdown: z
|
|
442
561
|
.string()
|
|
@@ -575,7 +694,11 @@ export function registerTools(server) {
|
|
|
575
694
|
reports_dir: z.string(),
|
|
576
695
|
run_id_filter: z.string().optional(),
|
|
577
696
|
query: z.string().optional(),
|
|
697
|
+
offset: z.number().int().min(0),
|
|
698
|
+
limit: z.number().int().positive().max(200),
|
|
699
|
+
returned: z.number().int().nonnegative(),
|
|
578
700
|
total_matching: z.number().int().nonnegative(),
|
|
701
|
+
next_offset: z.number().int().min(0).nullable(),
|
|
579
702
|
items: z.array(z.object({
|
|
580
703
|
path: z.string(),
|
|
581
704
|
relativePath: z.string(),
|
|
@@ -585,10 +708,6 @@ export function registerTools(server) {
|
|
|
585
708
|
}))
|
|
586
709
|
},
|
|
587
710
|
inputSchema: {
|
|
588
|
-
reports_dir: z
|
|
589
|
-
.string()
|
|
590
|
-
.optional()
|
|
591
|
-
.describe('Markdown reports root (default mcplab/reports).'),
|
|
592
711
|
run_id: z
|
|
593
712
|
.string()
|
|
594
713
|
.optional()
|
|
@@ -602,12 +721,18 @@ export function registerTools(server) {
|
|
|
602
721
|
.int()
|
|
603
722
|
.positive()
|
|
604
723
|
.max(200)
|
|
605
|
-
.
|
|
606
|
-
.describe('Max reports to return (default 20).')
|
|
724
|
+
.default(20)
|
|
725
|
+
.describe('Max reports to return (default 20).'),
|
|
726
|
+
offset: z
|
|
727
|
+
.number()
|
|
728
|
+
.int()
|
|
729
|
+
.min(0)
|
|
730
|
+
.default(0)
|
|
731
|
+
.describe('Pagination offset into matching results (default 0).')
|
|
607
732
|
}
|
|
608
|
-
}, async ({
|
|
733
|
+
}, async ({ run_id, query, limit, offset }) => {
|
|
609
734
|
return withToolHandling(async () => {
|
|
610
|
-
const root = resolveMarkdownReportsDir(
|
|
735
|
+
const root = resolveMarkdownReportsDir();
|
|
611
736
|
const all = listMarkdownReportsFromDisk(root);
|
|
612
737
|
const runFilter = String(run_id ?? '').trim();
|
|
613
738
|
const searchQuery = String(query ?? '')
|
|
@@ -624,18 +749,25 @@ export function registerTools(server) {
|
|
|
624
749
|
const hay = `${item.path}\n${item.relativePath}\n${item.name}`.toLowerCase();
|
|
625
750
|
return hay.includes(searchQuery);
|
|
626
751
|
});
|
|
627
|
-
const
|
|
752
|
+
const start = Math.max(0, offset ?? 0);
|
|
753
|
+
const pageSize = limit ?? 20;
|
|
754
|
+
const capped = filtered.slice(start, start + pageSize);
|
|
755
|
+
const nextOffset = start + capped.length < filtered.length ? start + capped.length : null;
|
|
628
756
|
return ok(`Found ${capped.length}/${filtered.length} markdown report(s) in ${root}`, {
|
|
629
757
|
reports_dir: root,
|
|
630
758
|
run_id_filter: runFilter || undefined,
|
|
631
759
|
query: searchQuery || undefined,
|
|
760
|
+
offset: start,
|
|
761
|
+
limit: pageSize,
|
|
762
|
+
returned: capped.length,
|
|
632
763
|
total_matching: filtered.length,
|
|
764
|
+
next_offset: nextOffset,
|
|
633
765
|
items: capped
|
|
634
766
|
});
|
|
635
767
|
});
|
|
636
768
|
});
|
|
637
769
|
registerTool('mcplab_read_markdown_report', {
|
|
638
|
-
description: 'Read a saved markdown report by relative path (under mcplab/reports
|
|
770
|
+
description: 'Read a saved markdown report by reports-root-relative path (under mcplab/reports), with optional truncation.',
|
|
639
771
|
outputSchema: {
|
|
640
772
|
reports_dir: z.string(),
|
|
641
773
|
path: z.string(),
|
|
@@ -647,13 +779,7 @@ export function registerTools(server) {
|
|
|
647
779
|
content: z.string()
|
|
648
780
|
},
|
|
649
781
|
inputSchema: {
|
|
650
|
-
path:
|
|
651
|
-
.string()
|
|
652
|
-
.describe('Report path (relative to reports root or workspace-relative, e.g. mcplab/reports/... ).'),
|
|
653
|
-
reports_dir: z
|
|
654
|
-
.string()
|
|
655
|
-
.optional()
|
|
656
|
-
.describe('Markdown reports root (default mcplab/reports).'),
|
|
782
|
+
path: SafeRelativePathSchema.describe('Report path relative to the reports root (e.g. team/run-2026-05-03.md). Do not prefix with mcplab/reports/.'),
|
|
657
783
|
max_chars: z
|
|
658
784
|
.number()
|
|
659
785
|
.int()
|
|
@@ -661,9 +787,10 @@ export function registerTools(server) {
|
|
|
661
787
|
.optional()
|
|
662
788
|
.describe('Optional truncation for markdown content preview (default 20000).')
|
|
663
789
|
}
|
|
664
|
-
}, async ({ path,
|
|
790
|
+
}, async ({ path, max_chars }) => {
|
|
665
791
|
return withToolHandling(async () => {
|
|
666
|
-
|
|
792
|
+
validateWorkspaceRelativePath(path, 'path');
|
|
793
|
+
const root = resolveMarkdownReportsDir();
|
|
667
794
|
const targetPath = resolveMarkdownReportPath(root, path);
|
|
668
795
|
if (!isMarkdownReportExt(targetPath)) {
|
|
669
796
|
throw new Error('path must point to a .md or .markdown file');
|
|
@@ -692,10 +819,6 @@ export function registerTools(server) {
|
|
|
692
819
|
description: 'List reusable MCPLab library entries (servers, agents, scenarios) from a bundle root such as mcplab/ or examples/libraries/.',
|
|
693
820
|
outputSchema: LibraryEntrySchema,
|
|
694
821
|
inputSchema: {
|
|
695
|
-
bundleRoot: z
|
|
696
|
-
.string()
|
|
697
|
-
.optional()
|
|
698
|
-
.describe('Optional library bundle root. Defaults to mcplab/ or examples/libraries/ if present.'),
|
|
699
822
|
kind: z
|
|
700
823
|
.enum(['all', 'servers', 'agents', 'scenarios'])
|
|
701
824
|
.optional()
|
|
@@ -705,16 +828,18 @@ export function registerTools(server) {
|
|
|
705
828
|
.optional()
|
|
706
829
|
.describe('Include parsed YAML content for each item (larger output).')
|
|
707
830
|
}
|
|
708
|
-
}, async ({
|
|
831
|
+
}, async ({ kind, includeContent }) => {
|
|
709
832
|
return withToolHandling(async () => {
|
|
710
|
-
const root = resolveBundleRoot(
|
|
833
|
+
const root = resolveBundleRoot();
|
|
711
834
|
const data = readLibrary(root, Boolean(includeContent));
|
|
712
835
|
const selectedKind = kind ?? 'all';
|
|
713
836
|
const structured = selectedKind === 'all'
|
|
714
837
|
? data
|
|
715
838
|
: {
|
|
716
839
|
bundleRoot: data.bundleRoot,
|
|
717
|
-
|
|
840
|
+
servers: selectedKind === 'servers' ? data.servers : [],
|
|
841
|
+
agents: selectedKind === 'agents' ? data.agents : [],
|
|
842
|
+
scenarios: selectedKind === 'scenarios' ? data.scenarios : []
|
|
718
843
|
};
|
|
719
844
|
return ok(`Loaded MCPLab library from ${root}`, structured);
|
|
720
845
|
});
|
|
@@ -730,13 +855,12 @@ export function registerTools(server) {
|
|
|
730
855
|
content: GenericObjectSchema
|
|
731
856
|
},
|
|
732
857
|
inputSchema: {
|
|
733
|
-
bundleRoot: z.string().optional().describe('Optional library bundle root path.'),
|
|
734
858
|
kind: z.enum(['servers', 'agents', 'scenarios']).describe('Library category.'),
|
|
735
859
|
id: z.string().describe('Entry id (for scenarios this is scenario.id, not filename).')
|
|
736
860
|
}
|
|
737
|
-
}, async ({
|
|
861
|
+
}, async ({ kind, id }) => {
|
|
738
862
|
return withToolHandling(async () => {
|
|
739
|
-
const root = resolveBundleRoot(
|
|
863
|
+
const root = resolveBundleRoot();
|
|
740
864
|
const item = getLibraryItem(root, kind, id);
|
|
741
865
|
return ok(`Loaded ${kind.slice(0, -1)} '${id}' from ${root}`, item);
|
|
742
866
|
});
|
|
@@ -748,39 +872,7 @@ export function registerTools(server) {
|
|
|
748
872
|
entry: ServerEntrySchema,
|
|
749
873
|
yaml: z.string()
|
|
750
874
|
},
|
|
751
|
-
inputSchema:
|
|
752
|
-
id: z.string().describe('Server id key (kebab-case recommended).'),
|
|
753
|
-
url: z.string().describe('MCP server URL (Streamable HTTP endpoint).'),
|
|
754
|
-
transport: z.enum(['http']).optional().describe('MCPLab transport type (currently http).'),
|
|
755
|
-
auth_type: z
|
|
756
|
-
.enum(['none', 'bearer', 'api_key', 'oauth_client_credentials'])
|
|
757
|
-
.optional()
|
|
758
|
-
.describe('Authentication mode.'),
|
|
759
|
-
bearer_token: z
|
|
760
|
-
.string()
|
|
761
|
-
.optional()
|
|
762
|
-
.describe('Direct bearer token value or ${VAR} env reference when auth_type=bearer.'),
|
|
763
|
-
bearer_env: z
|
|
764
|
-
.string()
|
|
765
|
-
.optional()
|
|
766
|
-
.describe('Env var for bearer token when auth_type=bearer.'),
|
|
767
|
-
api_key_header_name: z
|
|
768
|
-
.string()
|
|
769
|
-
.optional()
|
|
770
|
-
.describe('Header name for API key auth (default: X-API-Key).'),
|
|
771
|
-
api_key_value: z
|
|
772
|
-
.string()
|
|
773
|
-
.optional()
|
|
774
|
-
.describe('API key value or ${VAR} env reference when auth_type=api_key.'),
|
|
775
|
-
oauth_token_url: z
|
|
776
|
-
.string()
|
|
777
|
-
.optional()
|
|
778
|
-
.describe('OAuth token URL when auth_type=oauth_client_credentials.'),
|
|
779
|
-
oauth_client_id_env: z.string().optional().describe('OAuth client id env var.'),
|
|
780
|
-
oauth_client_secret_env: z.string().optional().describe('OAuth client secret env var.'),
|
|
781
|
-
oauth_scope: z.string().optional().describe('Optional OAuth scope.'),
|
|
782
|
-
oauth_audience: z.string().optional().describe('Optional OAuth audience.')
|
|
783
|
-
}
|
|
875
|
+
inputSchema: GenerateServerEntryPublicInputSchema
|
|
784
876
|
}, async (input) => {
|
|
785
877
|
return withToolHandling(async () => {
|
|
786
878
|
const parsed = GenerateServerEntryInputSchema.parse(input);
|
|
@@ -904,33 +996,27 @@ export function registerTools(server) {
|
|
|
904
996
|
resolved_config: z.object({
|
|
905
997
|
servers: z.record(GenericObjectSchema),
|
|
906
998
|
agents: z.record(AgentEntrySchema),
|
|
907
|
-
scenarios: z.array(
|
|
908
|
-
run_defaults:
|
|
999
|
+
scenarios: z.array(ResolvedScenarioSchema),
|
|
1000
|
+
run_defaults: RunDefaultsSchema.optional()
|
|
909
1001
|
})
|
|
910
1002
|
},
|
|
911
1003
|
inputSchema: {
|
|
912
1004
|
config_path: z.string().describe('Path to MCPLab eval YAML config.'),
|
|
913
|
-
bundle_root: z
|
|
914
|
-
.string()
|
|
915
|
-
.optional()
|
|
916
|
-
.describe('Optional bundle root override for refs resolution.'),
|
|
917
1005
|
scenario_id: z
|
|
918
1006
|
.string()
|
|
919
1007
|
.optional()
|
|
920
1008
|
.describe('Optional single scenario id to validate selection.')
|
|
921
1009
|
}
|
|
922
|
-
}, async ({ config_path,
|
|
1010
|
+
}, async ({ config_path, scenario_id }) => {
|
|
923
1011
|
return withToolHandling(async () => {
|
|
924
1012
|
const loaded = loadConfig(resolve(config_path), {
|
|
925
|
-
bundleRoot:
|
|
1013
|
+
bundleRoot: resolveBundleRoot()
|
|
926
1014
|
});
|
|
927
1015
|
const selected = selectScenarios(loaded.config, scenario_id);
|
|
928
1016
|
const summary = summarizeConfig(selected);
|
|
929
1017
|
return ok(`Validated config ${config_path}`, {
|
|
930
1018
|
configPath: resolve(config_path),
|
|
931
|
-
bundleRoot:
|
|
932
|
-
? resolve(bundle_root)
|
|
933
|
-
: detectLikelyBundleRoot(resolve(config_path)),
|
|
1019
|
+
bundleRoot: detectLikelyBundleRoot(resolve(config_path)),
|
|
934
1020
|
hash: loaded.hash,
|
|
935
1021
|
summary,
|
|
936
1022
|
resolved_config: selected
|
|
@@ -959,26 +1045,18 @@ export function registerTools(server) {
|
|
|
959
1045
|
},
|
|
960
1046
|
inputSchema: {
|
|
961
1047
|
config_path: z.string().describe('Path to MCPLab eval YAML config.'),
|
|
962
|
-
bundle_root: z
|
|
963
|
-
.string()
|
|
964
|
-
.optional()
|
|
965
|
-
.describe('Optional bundle root override for library refs.'),
|
|
966
1048
|
scenario_id: z.string().optional().describe('Optional scenario id to run.'),
|
|
967
1049
|
runs_per_scenario: z
|
|
968
1050
|
.number()
|
|
969
1051
|
.int()
|
|
970
1052
|
.positive()
|
|
971
1053
|
.optional()
|
|
972
|
-
.describe('Runs per scenario (default 1).')
|
|
973
|
-
runs_dir: z
|
|
974
|
-
.string()
|
|
975
|
-
.optional()
|
|
976
|
-
.describe('Output directory for run artifacts (default mcplab/results/evaluation-runs).')
|
|
1054
|
+
.describe('Runs per scenario (default 1).')
|
|
977
1055
|
}
|
|
978
|
-
}, async ({ config_path,
|
|
1056
|
+
}, async ({ config_path, scenario_id, runs_per_scenario }) => {
|
|
979
1057
|
return withToolHandling(async () => {
|
|
980
1058
|
const loaded = loadConfig(resolve(config_path), {
|
|
981
|
-
bundleRoot:
|
|
1059
|
+
bundleRoot: resolveBundleRoot()
|
|
982
1060
|
});
|
|
983
1061
|
const selected = selectScenarios(loaded.config, scenario_id);
|
|
984
1062
|
const executable = expandConfigForAgents(selected, selected.run_defaults?.selected_agents);
|
|
@@ -987,7 +1065,7 @@ export function registerTools(server) {
|
|
|
987
1065
|
scenarioId: scenario_id,
|
|
988
1066
|
configHash: loaded.hash,
|
|
989
1067
|
cliVersion: `mcplab-mcp-server/${SERVER_VERSION}`,
|
|
990
|
-
runsDir:
|
|
1068
|
+
runsDir: resolveRunsDir()
|
|
991
1069
|
});
|
|
992
1070
|
const reportHtml = renderReport(results);
|
|
993
1071
|
const allRuns = results.scenarios.flatMap((scenario) => scenario.runs);
|
|
@@ -1024,54 +1102,6 @@ export function registerTools(server) {
|
|
|
1024
1102
|
});
|
|
1025
1103
|
});
|
|
1026
1104
|
});
|
|
1027
|
-
registerTool('mcplab_search_runs', {
|
|
1028
|
-
description: 'Search MCPLab run artifact directories with results.json summary metrics. Use the optional query parameter to filter by run_id, path, or summary fields (case-insensitive substring). Set include_summary=false to skip reading results.json for faster listing.',
|
|
1029
|
-
outputSchema: {
|
|
1030
|
-
runsDir: z.string(),
|
|
1031
|
-
query: z.string().optional(),
|
|
1032
|
-
total_matching: z.number().int().nonnegative(),
|
|
1033
|
-
runs: z.array(RunListEntrySchema)
|
|
1034
|
-
},
|
|
1035
|
-
inputSchema: {
|
|
1036
|
-
runs_dir: z
|
|
1037
|
-
.string()
|
|
1038
|
-
.optional()
|
|
1039
|
-
.describe('Runs directory (default mcplab/results/evaluation-runs).'),
|
|
1040
|
-
query: z
|
|
1041
|
-
.string()
|
|
1042
|
-
.optional()
|
|
1043
|
-
.describe('Optional case-insensitive search query across run id/path and summary fields.'),
|
|
1044
|
-
limit: z
|
|
1045
|
-
.number()
|
|
1046
|
-
.int()
|
|
1047
|
-
.positive()
|
|
1048
|
-
.max(100)
|
|
1049
|
-
.default(10)
|
|
1050
|
-
.describe('Max runs to return. Defaults to 10.'),
|
|
1051
|
-
include_summary: z
|
|
1052
|
-
.boolean()
|
|
1053
|
-
.default(true)
|
|
1054
|
-
.describe('Read results.json summary for each run when available. Defaults to true.')
|
|
1055
|
-
}
|
|
1056
|
-
}, async ({ runs_dir, query, limit, include_summary }) => {
|
|
1057
|
-
return withToolHandling(async () => {
|
|
1058
|
-
const base = resolveRunsDir(runs_dir);
|
|
1059
|
-
const entries = listRunsWithFallback(base, undefined, include_summary);
|
|
1060
|
-
const searchQuery = String(query ?? '')
|
|
1061
|
-
.trim()
|
|
1062
|
-
.toLowerCase();
|
|
1063
|
-
const filtered = searchQuery
|
|
1064
|
-
? entries.filter((entry) => searchableText(entry).includes(searchQuery))
|
|
1065
|
-
: entries;
|
|
1066
|
-
const capped = filtered.slice(0, limit);
|
|
1067
|
-
return ok(`Found ${filtered.length} run(s) in ${base}`, {
|
|
1068
|
-
runsDir: base,
|
|
1069
|
-
query: searchQuery || undefined,
|
|
1070
|
-
total_matching: filtered.length,
|
|
1071
|
-
runs: capped
|
|
1072
|
-
});
|
|
1073
|
-
});
|
|
1074
|
-
});
|
|
1075
1105
|
registerTool('mcplab_aggregate_runs', {
|
|
1076
1106
|
description: 'Aggregate metrics across historical MCPLab runs with compact summary-first output.',
|
|
1077
1107
|
outputSchema: z.object({
|
|
@@ -1095,10 +1125,6 @@ export function registerTools(server) {
|
|
|
1095
1125
|
details: z.array(AggregateRowSchema).optional()
|
|
1096
1126
|
}),
|
|
1097
1127
|
inputSchema: {
|
|
1098
|
-
runs_dir: z
|
|
1099
|
-
.string()
|
|
1100
|
-
.optional()
|
|
1101
|
-
.describe('Runs directory (default mcplab/results/evaluation-runs).'),
|
|
1102
1128
|
run_ids: z
|
|
1103
1129
|
.array(z.string())
|
|
1104
1130
|
.optional()
|
|
@@ -1128,10 +1154,9 @@ export function registerTools(server) {
|
|
|
1128
1154
|
.optional()
|
|
1129
1155
|
.describe('Include full grouped rows. Defaults to false (summary-first).')
|
|
1130
1156
|
}
|
|
1131
|
-
}, async ({
|
|
1157
|
+
}, async ({ run_ids, latest_n, scenario_ids, agents, group_by, top_n, include_details }) => {
|
|
1132
1158
|
return withToolHandling(async () => {
|
|
1133
1159
|
const loaded = loadRunsForAnalysis({
|
|
1134
|
-
runsDirInput: runs_dir,
|
|
1135
1160
|
runIds: run_ids,
|
|
1136
1161
|
latestN: latest_n ?? 20
|
|
1137
1162
|
});
|
|
@@ -1189,10 +1214,6 @@ export function registerTools(server) {
|
|
|
1189
1214
|
details: z.array(CompareRowSchema).optional()
|
|
1190
1215
|
}),
|
|
1191
1216
|
inputSchema: {
|
|
1192
|
-
runs_dir: z
|
|
1193
|
-
.string()
|
|
1194
|
-
.optional()
|
|
1195
|
-
.describe('Runs directory (default mcplab/results/evaluation-runs).'),
|
|
1196
1217
|
left_run_id: z.string().describe("Left run id or 'LATEST'."),
|
|
1197
1218
|
right_run_id: z.string().describe("Right run id or 'LATEST'."),
|
|
1198
1219
|
scenario_ids: z.array(z.string()).optional().describe('Optional scenario id filter.'),
|
|
@@ -1209,9 +1230,9 @@ export function registerTools(server) {
|
|
|
1209
1230
|
.optional()
|
|
1210
1231
|
.describe('Include full classification rows. Defaults to false (summary-first).')
|
|
1211
1232
|
}
|
|
1212
|
-
}, async ({
|
|
1233
|
+
}, async ({ left_run_id, right_run_id, scenario_ids, agents, top_n, include_details }) => {
|
|
1213
1234
|
return withToolHandling(async () => {
|
|
1214
|
-
const base = resolveRunsDir(
|
|
1235
|
+
const base = resolveRunsDir();
|
|
1215
1236
|
const left = loadSingleRunForAnalysis(base, left_run_id);
|
|
1216
1237
|
const right = loadSingleRunForAnalysis(base, right_run_id);
|
|
1217
1238
|
const report = buildCompareRunsReport({
|
|
@@ -1233,10 +1254,6 @@ export function registerTools(server) {
|
|
|
1233
1254
|
items: z.array(ToolAnalysisListItemSchema)
|
|
1234
1255
|
},
|
|
1235
1256
|
inputSchema: {
|
|
1236
|
-
tool_analysis_results_dir: z
|
|
1237
|
-
.string()
|
|
1238
|
-
.optional()
|
|
1239
|
-
.describe('Directory containing saved tool analysis report folders.'),
|
|
1240
1257
|
query: z
|
|
1241
1258
|
.string()
|
|
1242
1259
|
.optional()
|
|
@@ -1249,9 +1266,9 @@ export function registerTools(server) {
|
|
|
1249
1266
|
.default(20)
|
|
1250
1267
|
.describe('Max reports to return. Defaults to 20.')
|
|
1251
1268
|
}
|
|
1252
|
-
}, async ({
|
|
1269
|
+
}, async ({ query, limit }) => {
|
|
1253
1270
|
return withToolHandling(async () => {
|
|
1254
|
-
const baseDir = resolveToolAnalysisResultsDir(
|
|
1271
|
+
const baseDir = resolveToolAnalysisResultsDir();
|
|
1255
1272
|
const reports = listToolAnalysisReportsFromDiskWithFallback(baseDir, undefined);
|
|
1256
1273
|
const searchQuery = String(query ?? '')
|
|
1257
1274
|
.trim()
|
|
@@ -1280,10 +1297,6 @@ export function registerTools(server) {
|
|
|
1280
1297
|
}),
|
|
1281
1298
|
inputSchema: {
|
|
1282
1299
|
report_id: z.string().describe("Report id directory name (or 'LATEST')."),
|
|
1283
|
-
tool_analysis_results_dir: z
|
|
1284
|
-
.string()
|
|
1285
|
-
.optional()
|
|
1286
|
-
.describe('Directory containing saved tool analysis reports (default mcplab/results/tool-analysis).'),
|
|
1287
1300
|
max_chars: z
|
|
1288
1301
|
.number()
|
|
1289
1302
|
.int()
|
|
@@ -1295,9 +1308,9 @@ export function registerTools(server) {
|
|
|
1295
1308
|
.optional()
|
|
1296
1309
|
.describe('Include the full parsed record in structured content. Defaults to true.')
|
|
1297
1310
|
}
|
|
1298
|
-
}, async ({ report_id,
|
|
1311
|
+
}, async ({ report_id, max_chars, include_record }) => {
|
|
1299
1312
|
return withToolHandling(async () => {
|
|
1300
|
-
const baseDir = resolveToolAnalysisResultsDir(
|
|
1313
|
+
const baseDir = resolveToolAnalysisResultsDir();
|
|
1301
1314
|
const resolvedReportId = report_id === 'LATEST'
|
|
1302
1315
|
? latestToolAnalysisReportIdWithFallback(baseDir)
|
|
1303
1316
|
: report_id.trim();
|
|
@@ -1336,22 +1349,18 @@ export function registerTools(server) {
|
|
|
1336
1349
|
},
|
|
1337
1350
|
inputSchema: {
|
|
1338
1351
|
report_id: z.string().describe('Report id directory name to delete.'),
|
|
1339
|
-
tool_analysis_results_dir: z
|
|
1340
|
-
.string()
|
|
1341
|
-
.optional()
|
|
1342
|
-
.describe('Directory containing saved tool analysis reports (default mcplab/results/tool-analysis).'),
|
|
1343
1352
|
dry_run: z
|
|
1344
1353
|
.boolean()
|
|
1345
|
-
.
|
|
1346
|
-
.describe('If true, return what would be deleted without deleting anything.'),
|
|
1354
|
+
.default(false)
|
|
1355
|
+
.describe('If true, return what would be deleted without deleting anything. Defaults to false.'),
|
|
1347
1356
|
confirm: z
|
|
1348
1357
|
.boolean()
|
|
1349
|
-
.
|
|
1350
|
-
.describe('Must be true to execute deletion when dry_run is false.')
|
|
1358
|
+
.default(false)
|
|
1359
|
+
.describe('Must be true to execute deletion when dry_run is false. If confirm is false, deletion is rejected with an error.')
|
|
1351
1360
|
}
|
|
1352
|
-
}, async ({ report_id,
|
|
1361
|
+
}, async ({ report_id, dry_run, confirm }) => {
|
|
1353
1362
|
return withToolHandling(async () => {
|
|
1354
|
-
const baseDir = resolveToolAnalysisResultsDir(
|
|
1363
|
+
const baseDir = resolveToolAnalysisResultsDir();
|
|
1355
1364
|
const dirPath = toolAnalysisReportDirPathWithFallback(baseDir, report_id.trim());
|
|
1356
1365
|
const existed = existsSync(dirPath);
|
|
1357
1366
|
const isDryRun = Boolean(dry_run);
|
|
@@ -1401,10 +1410,6 @@ export function registerTools(server) {
|
|
|
1401
1410
|
items: z.array(FlattenedTraceItemSchema)
|
|
1402
1411
|
},
|
|
1403
1412
|
inputSchema: {
|
|
1404
|
-
runs_dir: z
|
|
1405
|
-
.string()
|
|
1406
|
-
.optional()
|
|
1407
|
-
.describe('Runs directory (default mcplab/results/evaluation-runs).'),
|
|
1408
1413
|
run_id: z.string().describe("Run id directory name or 'LATEST'."),
|
|
1409
1414
|
event_types: z
|
|
1410
1415
|
.array(z.string())
|
|
@@ -1420,9 +1425,9 @@ export function registerTools(server) {
|
|
|
1420
1425
|
.optional()
|
|
1421
1426
|
.describe('Max items to return (default 200).')
|
|
1422
1427
|
}
|
|
1423
|
-
}, async ({
|
|
1428
|
+
}, async ({ run_id, event_types, scenario_id, agent, limit }) => {
|
|
1424
1429
|
return withToolHandling(async () => {
|
|
1425
|
-
const { runId, records, legacyDetected } = readScenarioRunTraceRecordsForRun(
|
|
1430
|
+
const { runId, records, legacyDetected } = readScenarioRunTraceRecordsForRun(run_id);
|
|
1426
1431
|
const typeSet = event_types?.length
|
|
1427
1432
|
? new Set(event_types)
|
|
1428
1433
|
: null;
|
|
@@ -1464,10 +1469,6 @@ export function registerTools(server) {
|
|
|
1464
1469
|
}))
|
|
1465
1470
|
},
|
|
1466
1471
|
inputSchema: {
|
|
1467
|
-
runs_dir: z
|
|
1468
|
-
.string()
|
|
1469
|
-
.optional()
|
|
1470
|
-
.describe('Runs directory (default mcplab/results/evaluation-runs).'),
|
|
1471
1472
|
run_id: z.string().describe("Run id directory name or 'LATEST'."),
|
|
1472
1473
|
scenario_id: z.string().optional().describe('Optional scenario id filter.'),
|
|
1473
1474
|
agent: z.string().optional().describe('Optional agent filter.'),
|
|
@@ -1479,9 +1480,9 @@ export function registerTools(server) {
|
|
|
1479
1480
|
.optional()
|
|
1480
1481
|
.describe('Optional truncation per final answer text (default 8000).')
|
|
1481
1482
|
}
|
|
1482
|
-
}, async ({
|
|
1483
|
+
}, async ({ run_id, scenario_id, agent, max_chars_per_answer }) => {
|
|
1483
1484
|
return withToolHandling(async () => {
|
|
1484
|
-
const { runId, records, legacyDetected } = readScenarioRunTraceRecordsForRun(
|
|
1485
|
+
const { runId, records, legacyDetected } = readScenarioRunTraceRecordsForRun(run_id);
|
|
1485
1486
|
const maxChars = max_chars_per_answer ?? 8000;
|
|
1486
1487
|
const items = records
|
|
1487
1488
|
.filter((record) => (!scenario_id || record.scenario_id === scenario_id) &&
|
|
@@ -1518,10 +1519,6 @@ export function registerTools(server) {
|
|
|
1518
1519
|
timeline: z.array(ConversationTimelineItemSchema)
|
|
1519
1520
|
},
|
|
1520
1521
|
inputSchema: {
|
|
1521
|
-
runs_dir: z
|
|
1522
|
-
.string()
|
|
1523
|
-
.optional()
|
|
1524
|
-
.describe('Runs directory (default mcplab/results/evaluation-runs).'),
|
|
1525
1522
|
run_id: z.string().describe("Run id directory name or 'LATEST'."),
|
|
1526
1523
|
scenario_id: z.string().describe('Scenario id to filter.'),
|
|
1527
1524
|
agent: z.string().describe('Agent name to filter.'),
|
|
@@ -1540,9 +1537,9 @@ export function registerTools(server) {
|
|
|
1540
1537
|
.optional()
|
|
1541
1538
|
.describe('Max chars for text fields (default 4000).')
|
|
1542
1539
|
}
|
|
1543
|
-
}, async ({
|
|
1540
|
+
}, async ({ run_id, scenario_id, agent, max_items, max_text_chars }) => {
|
|
1544
1541
|
return withToolHandling(async () => {
|
|
1545
|
-
const { runId, records, legacyDetected } = readScenarioRunTraceRecordsForRun(
|
|
1542
|
+
const { runId, records, legacyDetected } = readScenarioRunTraceRecordsForRun(run_id);
|
|
1546
1543
|
const textMax = max_text_chars ?? 4000;
|
|
1547
1544
|
const record = records.find((r) => r.scenario_id === scenario_id && r.agent === agent);
|
|
1548
1545
|
const timeline = record
|
|
@@ -1561,15 +1558,11 @@ export function registerTools(server) {
|
|
|
1561
1558
|
description: 'Search scenario_run trace content for a text query and return matching message/block items.',
|
|
1562
1559
|
outputSchema: {
|
|
1563
1560
|
run_id: z.string(),
|
|
1564
|
-
query: z.string(),
|
|
1561
|
+
query: z.string().trim().min(1),
|
|
1565
1562
|
legacy_trace_detected: z.boolean().optional(),
|
|
1566
1563
|
matches: z.array(FlattenedTraceItemSchema)
|
|
1567
1564
|
},
|
|
1568
1565
|
inputSchema: {
|
|
1569
|
-
runs_dir: z
|
|
1570
|
-
.string()
|
|
1571
|
-
.optional()
|
|
1572
|
-
.describe('Runs directory (default mcplab/results/evaluation-runs).'),
|
|
1573
1566
|
run_id: z.string().describe("Run id directory name or 'LATEST'."),
|
|
1574
1567
|
query: z.string().describe('Case-insensitive text query.'),
|
|
1575
1568
|
event_types: z
|
|
@@ -1584,12 +1577,12 @@ export function registerTools(server) {
|
|
|
1584
1577
|
.optional()
|
|
1585
1578
|
.describe('Max matches to return (default 50).')
|
|
1586
1579
|
}
|
|
1587
|
-
}, async ({
|
|
1580
|
+
}, async ({ run_id, query, event_types, limit }) => {
|
|
1588
1581
|
return withToolHandling(async () => {
|
|
1589
1582
|
const q = query.trim().toLowerCase();
|
|
1590
1583
|
if (!q)
|
|
1591
1584
|
throw new Error('query is required');
|
|
1592
|
-
const { runId, records, legacyDetected } = readScenarioRunTraceRecordsForRun(
|
|
1585
|
+
const { runId, records, legacyDetected } = readScenarioRunTraceRecordsForRun(run_id);
|
|
1593
1586
|
const typeSet = event_types?.length
|
|
1594
1587
|
? new Set(event_types)
|
|
1595
1588
|
: null;
|
|
@@ -1632,15 +1625,11 @@ export function registerTools(server) {
|
|
|
1632
1625
|
}))
|
|
1633
1626
|
},
|
|
1634
1627
|
inputSchema: {
|
|
1635
|
-
runs_dir: z
|
|
1636
|
-
.string()
|
|
1637
|
-
.optional()
|
|
1638
|
-
.describe('Runs directory (default mcplab/results/evaluation-runs).'),
|
|
1639
1628
|
run_id: z.string().describe("Run id directory name or 'LATEST'.")
|
|
1640
1629
|
}
|
|
1641
|
-
}, async ({
|
|
1630
|
+
}, async ({ run_id }) => {
|
|
1642
1631
|
return withToolHandling(async () => {
|
|
1643
|
-
const { runId, records, legacyDetected } = readScenarioRunTraceRecordsForRun(
|
|
1632
|
+
const { runId, records, legacyDetected } = readScenarioRunTraceRecordsForRun(run_id);
|
|
1644
1633
|
const messageRoleCounts = {};
|
|
1645
1634
|
const blockTypeCounts = {};
|
|
1646
1635
|
const toolUsage = {};
|
|
@@ -1711,10 +1700,6 @@ export function registerTools(server) {
|
|
|
1711
1700
|
.optional()
|
|
1712
1701
|
},
|
|
1713
1702
|
inputSchema: {
|
|
1714
|
-
runs_dir: z
|
|
1715
|
-
.string()
|
|
1716
|
-
.optional()
|
|
1717
|
-
.describe('Runs directory (default mcplab/results/evaluation-runs).'),
|
|
1718
1703
|
run_id: z.string().describe('Run id directory name or LATEST.'),
|
|
1719
1704
|
artifact: z
|
|
1720
1705
|
.enum([
|
|
@@ -1744,15 +1729,15 @@ export function registerTools(server) {
|
|
|
1744
1729
|
.optional()
|
|
1745
1730
|
.describe('1-indexed line to stop reading at (inclusive).')
|
|
1746
1731
|
}
|
|
1747
|
-
}, async ({
|
|
1732
|
+
}, async ({ run_id, artifact, max_chars, line_start, line_end }) => {
|
|
1748
1733
|
return withToolHandling(async () => {
|
|
1749
|
-
const base = resolveRunsDir(
|
|
1734
|
+
const base = resolveRunsDir();
|
|
1750
1735
|
const readBase = resolveExistingRunReadDir(base, run_id === 'LATEST' ? undefined : run_id);
|
|
1751
1736
|
const resolvedRunId = run_id === 'LATEST' ? latestRunId(readBase) : run_id;
|
|
1752
1737
|
if (!resolvedRunId) {
|
|
1753
1738
|
throw new Error(`No runs found in ${base}`);
|
|
1754
1739
|
}
|
|
1755
|
-
const fullPath =
|
|
1740
|
+
const fullPath = resolveRunArtifactPath(readBase, resolvedRunId, artifact);
|
|
1756
1741
|
if (!existsSync(fullPath)) {
|
|
1757
1742
|
throw new Error(`Artifact not found: ${fullPath}`);
|
|
1758
1743
|
}
|
|
@@ -1793,111 +1778,132 @@ export function registerTools(server) {
|
|
|
1793
1778
|
return ok(`Read ${artifact} from run ${resolvedRunId}`, structured);
|
|
1794
1779
|
});
|
|
1795
1780
|
});
|
|
1796
|
-
registerTool('
|
|
1797
|
-
description: '
|
|
1781
|
+
registerTool('mcplab_results_index', {
|
|
1782
|
+
description: 'Build or refresh local MCPLab results search index under mcplab/results/.index for LLM-first querying.',
|
|
1783
|
+
outputSchema: {
|
|
1784
|
+
runs_dir: z.string(),
|
|
1785
|
+
rebuilt: z.boolean(),
|
|
1786
|
+
doc_count: z.number().int().nonnegative(),
|
|
1787
|
+
index_path: z.string(),
|
|
1788
|
+
manifest_path: z.string()
|
|
1789
|
+
},
|
|
1790
|
+
inputSchema: {
|
|
1791
|
+
rebuild: z.boolean().optional().describe('Force full index rebuild.')
|
|
1792
|
+
}
|
|
1793
|
+
}, async ({ rebuild }) => {
|
|
1794
|
+
return withToolHandling(async () => {
|
|
1795
|
+
const runsDir = resolveRunsDir();
|
|
1796
|
+
const wasStale = indexNeedsRefresh(runsDir);
|
|
1797
|
+
const docs = loadOrBuildSearchIndex(runsDir, Boolean(rebuild));
|
|
1798
|
+
const paths = getResultsIndexPaths(runsDir);
|
|
1799
|
+
return ok(`Results index ready (${docs.length} docs).`, {
|
|
1800
|
+
runs_dir: runsDir,
|
|
1801
|
+
rebuilt: Boolean(rebuild) || wasStale,
|
|
1802
|
+
doc_count: docs.length,
|
|
1803
|
+
index_path: paths.indexPath,
|
|
1804
|
+
manifest_path: paths.manifestPath
|
|
1805
|
+
});
|
|
1806
|
+
});
|
|
1807
|
+
});
|
|
1808
|
+
registerTool('mcplab_results_search', {
|
|
1809
|
+
description: 'Search MCPLab run results in compact LLM-first format. Auto-refreshes index when artifacts changed.',
|
|
1798
1810
|
outputSchema: {
|
|
1799
|
-
run_id: z.string(),
|
|
1800
|
-
artifact: z.enum([
|
|
1801
|
-
'results.json',
|
|
1802
|
-
'summary.md',
|
|
1803
|
-
'trace.jsonl',
|
|
1804
|
-
'resolved-config.yaml',
|
|
1805
|
-
'report.html'
|
|
1806
|
-
]),
|
|
1807
1811
|
query: z.string(),
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
|
|
1813
|
-
|
|
1814
|
-
|
|
1815
|
-
|
|
1816
|
-
|
|
1817
|
-
|
|
1818
|
-
|
|
1819
|
-
|
|
1812
|
+
runs_dir: z.string(),
|
|
1813
|
+
total_hits: z.number().int().nonnegative(),
|
|
1814
|
+
hits: z.array(z.object({
|
|
1815
|
+
run_id: z.string(),
|
|
1816
|
+
scenario_id: z.string().optional(),
|
|
1817
|
+
agent: z.string().optional(),
|
|
1818
|
+
status: z.enum(['passed', 'failed']).optional(),
|
|
1819
|
+
source: ResultsQuerySourceSchema,
|
|
1820
|
+
file: z.string(),
|
|
1821
|
+
line_start: z.number().int().positive().optional(),
|
|
1822
|
+
line_end: z.number().int().positive().optional(),
|
|
1823
|
+
snippet: z.string(),
|
|
1824
|
+
score: z.number(),
|
|
1825
|
+
context_command: z.string().optional()
|
|
1820
1826
|
}))
|
|
1821
1827
|
},
|
|
1822
1828
|
inputSchema: {
|
|
1823
|
-
|
|
1824
|
-
|
|
1829
|
+
query: z.string().trim().min(1).describe('Search query.'),
|
|
1830
|
+
status: ResultsQueryStatusSchema.optional().describe('Filter by status (default all).'),
|
|
1831
|
+
source: z
|
|
1832
|
+
.array(ResultsQuerySourceSchema)
|
|
1825
1833
|
.optional()
|
|
1826
|
-
.describe('
|
|
1834
|
+
.describe('Sources to search (default results,trace,summary).'),
|
|
1835
|
+
scenario: z.string().optional().describe('Filter by scenario id.'),
|
|
1836
|
+
agent: z.string().optional().describe('Filter by agent id.'),
|
|
1837
|
+
limit: z.number().int().positive().max(100).optional().describe('Max hits (default 10).')
|
|
1838
|
+
}
|
|
1839
|
+
}, async ({ query, status, source, scenario, agent, limit }) => {
|
|
1840
|
+
return withToolHandling(async () => {
|
|
1841
|
+
const runsDir = resolveRunsDir();
|
|
1842
|
+
const docs = loadOrBuildSearchIndex(runsDir, false);
|
|
1843
|
+
const hits = searchDocs(docs, {
|
|
1844
|
+
query,
|
|
1845
|
+
status: status ?? 'all',
|
|
1846
|
+
source: source && source.length > 0 ? source : ['results', 'trace', 'summary'],
|
|
1847
|
+
scenario,
|
|
1848
|
+
agent,
|
|
1849
|
+
limit: limit ?? 10
|
|
1850
|
+
});
|
|
1851
|
+
return ok(`Found ${hits.length} result hit(s).`, {
|
|
1852
|
+
query,
|
|
1853
|
+
runs_dir: runsDir,
|
|
1854
|
+
total_hits: hits.length,
|
|
1855
|
+
hits
|
|
1856
|
+
});
|
|
1857
|
+
});
|
|
1858
|
+
});
|
|
1859
|
+
registerTool('mcplab_results_context', {
|
|
1860
|
+
description: 'Fetch focused context for a scenario/run from results, trace, or summary. Returns bounded excerpts only. around is trace-line context and is only valid with source=trace (or when source is omitted).',
|
|
1861
|
+
outputSchema: {
|
|
1862
|
+
run_id: z.string(),
|
|
1863
|
+
scenario_id: z.string(),
|
|
1864
|
+
source: z.enum(['results', 'trace', 'summary', 'mixed']),
|
|
1865
|
+
line_start: z.number().int().positive().optional(),
|
|
1866
|
+
line_end: z.number().int().positive().optional(),
|
|
1867
|
+
excerpt: z.string()
|
|
1868
|
+
},
|
|
1869
|
+
inputSchema: {
|
|
1827
1870
|
run_id: z.string().describe("Run id directory name or 'LATEST'."),
|
|
1828
|
-
|
|
1829
|
-
|
|
1830
|
-
|
|
1831
|
-
|
|
1832
|
-
'trace.jsonl',
|
|
1833
|
-
'resolved-config.yaml',
|
|
1834
|
-
'report.html'
|
|
1835
|
-
])
|
|
1836
|
-
.describe('Artifact filename to search.'),
|
|
1837
|
-
query: z.string().describe('Text to search for (case-insensitive by default).'),
|
|
1838
|
-
context_lines: z
|
|
1871
|
+
scenario_id: z.string().describe('Scenario id to focus.'),
|
|
1872
|
+
source: ResultsQuerySourceSchema.optional().describe('Context source; default mixed.'),
|
|
1873
|
+
around: z.number().int().positive().optional().describe('Trace line center.'),
|
|
1874
|
+
before: z
|
|
1839
1875
|
.number()
|
|
1840
1876
|
.int()
|
|
1841
1877
|
.min(0)
|
|
1842
|
-
.max(
|
|
1878
|
+
.max(200)
|
|
1843
1879
|
.optional()
|
|
1844
|
-
.describe('Lines
|
|
1845
|
-
|
|
1880
|
+
.describe('Lines before around for trace (default 20).'),
|
|
1881
|
+
after: z
|
|
1846
1882
|
.number()
|
|
1847
1883
|
.int()
|
|
1848
|
-
.
|
|
1849
|
-
.max(
|
|
1884
|
+
.min(0)
|
|
1885
|
+
.max(200)
|
|
1850
1886
|
.optional()
|
|
1851
|
-
.describe('
|
|
1852
|
-
case_sensitive: z.boolean().optional().describe('Case-sensitive search (default false).')
|
|
1887
|
+
.describe('Lines after around for trace (default 20).')
|
|
1853
1888
|
}
|
|
1854
|
-
}, async ({
|
|
1889
|
+
}, async ({ run_id, scenario_id, source, around, before, after }) => {
|
|
1855
1890
|
return withToolHandling(async () => {
|
|
1856
|
-
const base = resolveRunsDir(
|
|
1891
|
+
const base = resolveRunsDir();
|
|
1857
1892
|
const readBase = resolveExistingRunReadDir(base, run_id === 'LATEST' ? undefined : run_id);
|
|
1858
1893
|
const resolvedRunId = run_id === 'LATEST' ? latestRunId(readBase) : run_id;
|
|
1859
1894
|
if (!resolvedRunId)
|
|
1860
1895
|
throw new Error(`No runs found in ${base}`);
|
|
1861
|
-
const
|
|
1862
|
-
|
|
1863
|
-
|
|
1864
|
-
|
|
1865
|
-
|
|
1866
|
-
|
|
1867
|
-
|
|
1868
|
-
|
|
1869
|
-
const limit = max_matches ?? 10;
|
|
1870
|
-
const matchIndices = [];
|
|
1871
|
-
for (let i = 0; i < lines.length; i++) {
|
|
1872
|
-
const hay = case_sensitive ? lines[i] : lines[i].toLowerCase();
|
|
1873
|
-
if (hay.includes(q)) {
|
|
1874
|
-
matchIndices.push(i);
|
|
1875
|
-
if (matchIndices.length >= limit)
|
|
1876
|
-
break;
|
|
1877
|
-
}
|
|
1878
|
-
}
|
|
1879
|
-
const matches = matchIndices.map((matchIdx) => {
|
|
1880
|
-
const start = Math.max(0, matchIdx - ctx);
|
|
1881
|
-
const end = Math.min(lines.length - 1, matchIdx + ctx);
|
|
1882
|
-
return {
|
|
1883
|
-
match_line: matchIdx + 1,
|
|
1884
|
-
context_start_line: start + 1,
|
|
1885
|
-
context_end_line: end + 1,
|
|
1886
|
-
lines: lines.slice(start, end + 1).map((text, offset) => ({
|
|
1887
|
-
line: start + offset + 1,
|
|
1888
|
-
text,
|
|
1889
|
-
is_match: start + offset === matchIdx
|
|
1890
|
-
}))
|
|
1891
|
-
};
|
|
1896
|
+
const result = getContext({
|
|
1897
|
+
runsDir: readBase,
|
|
1898
|
+
runId: resolvedRunId,
|
|
1899
|
+
scenarioId: scenario_id,
|
|
1900
|
+
source,
|
|
1901
|
+
around,
|
|
1902
|
+
before: before ?? 20,
|
|
1903
|
+
after: after ?? 20
|
|
1892
1904
|
});
|
|
1893
|
-
return ok(`
|
|
1894
|
-
|
|
1895
|
-
artifact,
|
|
1896
|
-
query,
|
|
1897
|
-
total_lines: lines.length,
|
|
1898
|
-
match_count: matches.length,
|
|
1899
|
-
truncated_at_limit: matchIndices.length >= limit,
|
|
1900
|
-
matches
|
|
1905
|
+
return ok(`Loaded context for run ${resolvedRunId} scenario ${scenario_id}.`, {
|
|
1906
|
+
...result
|
|
1901
1907
|
});
|
|
1902
1908
|
});
|
|
1903
1909
|
});
|
|
@@ -1907,22 +1913,17 @@ export function registerPrompts(server) {
|
|
|
1907
1913
|
description: 'Guide an LLM to author or refine MCPLab scenarios, prioritizing reusable scenario library files and deterministic eval rules.',
|
|
1908
1914
|
argsSchema: {
|
|
1909
1915
|
task: z.string().describe('What the scenario should test.'),
|
|
1910
|
-
bundle_root: z
|
|
1911
|
-
.string()
|
|
1912
|
-
.optional()
|
|
1913
|
-
.describe('Optional MCPLab library bundle root to inspect.'),
|
|
1914
1916
|
server_ids: z
|
|
1915
1917
|
.string()
|
|
1916
1918
|
.optional()
|
|
1917
1919
|
.describe('Comma-separated server ids to target if already known.'),
|
|
1918
1920
|
agent_id: z.string().optional().describe('Optional pinned agent id.')
|
|
1919
1921
|
}
|
|
1920
|
-
}, async ({ task,
|
|
1922
|
+
}, async ({ task, server_ids, agent_id }) => {
|
|
1921
1923
|
const maybeServers = server_ids
|
|
1922
1924
|
? `Target servers (if valid): ${server_ids}\n`
|
|
1923
1925
|
: 'First inspect available servers with mcplab_list_library.\n';
|
|
1924
1926
|
const maybeAgent = agent_id ? `Pinned agent (optional): ${agent_id}\n` : '';
|
|
1925
|
-
const maybeBundle = bundle_root ? `Bundle root hint: ${bundle_root}\n` : '';
|
|
1926
1927
|
return {
|
|
1927
1928
|
messages: [
|
|
1928
1929
|
{
|
|
@@ -1930,7 +1931,7 @@ export function registerPrompts(server) {
|
|
|
1930
1931
|
content: {
|
|
1931
1932
|
type: 'text',
|
|
1932
1933
|
text: `Help me author a MCPLab scenario for this testing task:\n\n${task}\n\n` +
|
|
1933
|
-
`${
|
|
1934
|
+
`${maybeServers}${maybeAgent}` +
|
|
1934
1935
|
`Workflow:\n` +
|
|
1935
1936
|
`1. Inspect library entries (servers/agents/scenarios) if needed.\n` +
|
|
1936
1937
|
`2. Draft a scenario with mcplab_generate_scenario_entry.\n` +
|
|
@@ -1980,10 +1981,8 @@ const PREFERRED_TOOL_TITLES = {
|
|
|
1980
1981
|
mcplab_search_markdown_reports: 'Search Markdown Reports',
|
|
1981
1982
|
mcplab_list_library: 'Search Library Entries',
|
|
1982
1983
|
mcplab_generate_agent_entry: 'Generate MCPLab agents.yaml Entry',
|
|
1983
|
-
mcplab_search_runs: 'Search Evaluation Runs',
|
|
1984
1984
|
mcplab_search_tool_analysis_results: 'Search Tool Analysis Results',
|
|
1985
|
-
mcplab_trace_search: 'Search Trace Events'
|
|
1986
|
-
mcplab_grep_run_artifact: 'Search Run Artifact Text'
|
|
1985
|
+
mcplab_trace_search: 'Search Trace Events'
|
|
1987
1986
|
};
|
|
1988
1987
|
function normalizeOptionalNonEmpty(value) {
|
|
1989
1988
|
if (typeof value !== 'string')
|
|
@@ -2037,17 +2036,31 @@ function humanizeToolName(toolName) {
|
|
|
2037
2036
|
.map((word) => word.charAt(0).toUpperCase() + word.slice(1))
|
|
2038
2037
|
.join(' ');
|
|
2039
2038
|
}
|
|
2040
|
-
function
|
|
2041
|
-
if (bundleRoot?.trim())
|
|
2042
|
-
return resolve(bundleRoot);
|
|
2039
|
+
function resolveServerOwnedRoots() {
|
|
2043
2040
|
const cwd = process.cwd();
|
|
2041
|
+
const reportsDir = resolvePathInsideWorkspace(process.env.MCPLAB_REPORTS_DIR || 'mcplab/reports');
|
|
2042
|
+
const runsDir = resolvePathInsideWorkspace(process.env.MCPLAB_RUNS_DIR || 'mcplab/results/evaluation-runs');
|
|
2043
|
+
const toolAnalysisDir = resolvePathInsideWorkspace(process.env.MCPLAB_TOOL_ANALYSIS_DIR || 'mcplab/results/tool-analysis');
|
|
2044
|
+
const configuredBundleRoot = process.env.MCPLAB_BUNDLE_ROOT?.trim();
|
|
2045
|
+
if (configuredBundleRoot) {
|
|
2046
|
+
return {
|
|
2047
|
+
reportsDir,
|
|
2048
|
+
runsDir,
|
|
2049
|
+
toolAnalysisDir,
|
|
2050
|
+
bundleRoot: resolve(configuredBundleRoot)
|
|
2051
|
+
};
|
|
2052
|
+
}
|
|
2044
2053
|
const candidates = ['mcplab', 'examples/libraries'];
|
|
2045
2054
|
for (const candidate of candidates) {
|
|
2046
2055
|
const abs = resolve(cwd, candidate);
|
|
2047
|
-
if (existsSync(abs))
|
|
2048
|
-
return abs;
|
|
2056
|
+
if (existsSync(abs)) {
|
|
2057
|
+
return { reportsDir, runsDir, toolAnalysisDir, bundleRoot: abs };
|
|
2058
|
+
}
|
|
2049
2059
|
}
|
|
2050
|
-
return resolve(cwd, 'mcplab');
|
|
2060
|
+
return { reportsDir, runsDir, toolAnalysisDir, bundleRoot: resolve(cwd, 'mcplab') };
|
|
2061
|
+
}
|
|
2062
|
+
function resolveBundleRoot() {
|
|
2063
|
+
return SERVER_OWNED_ROOTS.bundleRoot;
|
|
2051
2064
|
}
|
|
2052
2065
|
function readLibrary(bundleRoot, includeContent) {
|
|
2053
2066
|
const serversPath = join(bundleRoot, 'servers.yaml');
|
|
@@ -2077,16 +2090,18 @@ function readLibrary(bundleRoot, includeContent) {
|
|
|
2077
2090
|
}
|
|
2078
2091
|
const out = {
|
|
2079
2092
|
bundleRoot,
|
|
2080
|
-
servers:
|
|
2081
|
-
|
|
2082
|
-
|
|
2083
|
-
|
|
2084
|
-
|
|
2085
|
-
|
|
2086
|
-
|
|
2087
|
-
|
|
2088
|
-
|
|
2089
|
-
|
|
2093
|
+
servers: Object.keys(servers)
|
|
2094
|
+
.sort()
|
|
2095
|
+
.map((id) => ({
|
|
2096
|
+
id,
|
|
2097
|
+
...(includeContent ? { entry: servers[id] ?? {} } : {})
|
|
2098
|
+
})),
|
|
2099
|
+
agents: Object.keys(agents)
|
|
2100
|
+
.sort()
|
|
2101
|
+
.map((id) => ({
|
|
2102
|
+
id,
|
|
2103
|
+
...(includeContent ? { entry: agents[id] } : {})
|
|
2104
|
+
})),
|
|
2090
2105
|
scenarios: scenarioEntries
|
|
2091
2106
|
};
|
|
2092
2107
|
return out;
|
|
@@ -2267,15 +2282,15 @@ function filterScenarios(scenarios, scenarioFilter, agentFilter) {
|
|
|
2267
2282
|
});
|
|
2268
2283
|
}
|
|
2269
2284
|
function loadRunsForAnalysis(params) {
|
|
2270
|
-
const base = resolveRunsDir(
|
|
2285
|
+
const base = resolveRunsDir();
|
|
2271
2286
|
const ids = selectRunIdsForAnalysis(base, params.runIds, params.latestN);
|
|
2272
2287
|
return ids.map((id) => loadSingleRunForAnalysis(base, id));
|
|
2273
2288
|
}
|
|
2274
2289
|
function loadSingleRunForAnalysis(primaryRunsDir, runIdInput) {
|
|
2275
2290
|
const resolvedRunId = resolveRunIdToken(primaryRunsDir, runIdInput);
|
|
2276
2291
|
const readBase = resolveExistingRunReadDir(primaryRunsDir, resolvedRunId);
|
|
2277
|
-
const runPath =
|
|
2278
|
-
const resultsPath =
|
|
2292
|
+
const runPath = resolve(readBase, resolvedRunId);
|
|
2293
|
+
const resultsPath = resolveRunArtifactPath(readBase, resolvedRunId, 'results.json');
|
|
2279
2294
|
if (!existsSync(resultsPath)) {
|
|
2280
2295
|
throw new Error(`results.json not found for run '${resolvedRunId}' at ${resultsPath}`);
|
|
2281
2296
|
}
|
|
@@ -2357,13 +2372,13 @@ function listRuns(runsDir, limit, includeSummary) {
|
|
|
2357
2372
|
});
|
|
2358
2373
|
}
|
|
2359
2374
|
function defaultRunsDirPath() {
|
|
2360
|
-
return
|
|
2375
|
+
return SERVER_OWNED_ROOTS.runsDir;
|
|
2361
2376
|
}
|
|
2362
2377
|
function legacyRunsDirPath() {
|
|
2363
2378
|
return resolvePathInsideWorkspace('mcplab/runs');
|
|
2364
2379
|
}
|
|
2365
|
-
function resolveRunsDir(
|
|
2366
|
-
return
|
|
2380
|
+
function resolveRunsDir() {
|
|
2381
|
+
return SERVER_OWNED_ROOTS.runsDir;
|
|
2367
2382
|
}
|
|
2368
2383
|
function runReadDirs(primaryRunsDir) {
|
|
2369
2384
|
const dirs = [primaryRunsDir];
|
|
@@ -2423,11 +2438,11 @@ function detectLikelyBundleRoot(configPath) {
|
|
|
2423
2438
|
const fallback = resolveBundleRoot();
|
|
2424
2439
|
return existsSync(fallback) ? fallback : null;
|
|
2425
2440
|
}
|
|
2426
|
-
function resolveToolAnalysisResultsDir(
|
|
2427
|
-
return
|
|
2441
|
+
function resolveToolAnalysisResultsDir() {
|
|
2442
|
+
return SERVER_OWNED_ROOTS.toolAnalysisDir;
|
|
2428
2443
|
}
|
|
2429
|
-
function resolveMarkdownReportsDir(
|
|
2430
|
-
return
|
|
2444
|
+
function resolveMarkdownReportsDir() {
|
|
2445
|
+
return SERVER_OWNED_ROOTS.reportsDir;
|
|
2431
2446
|
}
|
|
2432
2447
|
function isMarkdownReportExt(path) {
|
|
2433
2448
|
const ext = extname(path).toLowerCase();
|
|
@@ -2484,11 +2499,11 @@ function resolveMarkdownReportPath(root, pathInput) {
|
|
|
2484
2499
|
const trimmed = pathInput.trim();
|
|
2485
2500
|
if (!trimmed)
|
|
2486
2501
|
throw new Error('path is required');
|
|
2487
|
-
const workspaceRelativePrefix = `mcplab${sep}reports${sep}`;
|
|
2488
2502
|
const normalized = trimmed.replaceAll('/', sep);
|
|
2489
|
-
|
|
2490
|
-
|
|
2491
|
-
|
|
2503
|
+
if (normalized === `mcplab${sep}reports` || normalized.startsWith(`mcplab${sep}reports${sep}`)) {
|
|
2504
|
+
throw new Error('path must be relative to reports root; do not prefix with mcplab/reports/');
|
|
2505
|
+
}
|
|
2506
|
+
const candidate = resolve(root, normalized);
|
|
2492
2507
|
const withinRoot = candidate === root || candidate.startsWith(`${root}${sep}`);
|
|
2493
2508
|
if (!withinRoot)
|
|
2494
2509
|
throw new Error('path escapes markdown reports root');
|
|
@@ -2663,13 +2678,13 @@ function isScenarioRunTraceRecord(value) {
|
|
|
2663
2678
|
Array.isArray(v.messages) &&
|
|
2664
2679
|
v.messages.every(isTraceMessage));
|
|
2665
2680
|
}
|
|
2666
|
-
function readScenarioRunTraceRecordsForRun(
|
|
2667
|
-
const base = resolveRunsDir(
|
|
2681
|
+
function readScenarioRunTraceRecordsForRun(runIdInput) {
|
|
2682
|
+
const base = resolveRunsDir();
|
|
2668
2683
|
const readBase = resolveExistingRunReadDir(base, runIdInput === 'LATEST' ? undefined : runIdInput);
|
|
2669
2684
|
const runId = runIdInput === 'LATEST' ? latestRunId(readBase) : runIdInput;
|
|
2670
2685
|
if (!runId)
|
|
2671
2686
|
throw new Error(`No runs found in ${base}`);
|
|
2672
|
-
const tracePath =
|
|
2687
|
+
const tracePath = resolveRunArtifactPath(readBase, runId, 'trace.jsonl');
|
|
2673
2688
|
if (!existsSync(tracePath))
|
|
2674
2689
|
throw new Error(`Artifact not found: ${tracePath}`);
|
|
2675
2690
|
const raw = readFileSync(tracePath, 'utf8');
|
|
@@ -2842,6 +2857,14 @@ function resolvePathInsideWorkspace(pathInput) {
|
|
|
2842
2857
|
}
|
|
2843
2858
|
return target;
|
|
2844
2859
|
}
|
|
2860
|
+
function validateWorkspaceRelativePath(value, fieldName) {
|
|
2861
|
+
if (value.startsWith('/') || /^[A-Za-z]:/.test(value)) {
|
|
2862
|
+
throw new Error(`${fieldName} must be relative (absolute paths are not allowed)`);
|
|
2863
|
+
}
|
|
2864
|
+
if (value.split(/[\\/]/).some((part) => part === '..')) {
|
|
2865
|
+
throw new Error(`${fieldName} must not contain ".." path segments`);
|
|
2866
|
+
}
|
|
2867
|
+
}
|
|
2845
2868
|
function ok(summary, structuredContent) {
|
|
2846
2869
|
const payload = structuredContent ?? {};
|
|
2847
2870
|
return {
|
|
@@ -2858,8 +2881,7 @@ function err(error) {
|
|
|
2858
2881
|
const message = error instanceof Error ? error.message : String(error);
|
|
2859
2882
|
return {
|
|
2860
2883
|
isError: true,
|
|
2861
|
-
content: [{ type: 'text', text: `Error: ${message}` }]
|
|
2862
|
-
structuredContent: { error: message }
|
|
2884
|
+
content: [{ type: 'text', text: `Error: ${message}` }]
|
|
2863
2885
|
};
|
|
2864
2886
|
}
|
|
2865
2887
|
async function withToolHandling(fn) {
|