@inspectr/mcplab-mcp-server 1.2.2 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/runtime.js CHANGED
@@ -5,7 +5,7 @@ import { basename, dirname, extname, resolve, join, sep, relative } from 'node:p
5
5
  import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
6
6
  import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js';
7
7
  import { isInitializeRequest } from '@modelcontextprotocol/sdk/types.js';
8
- import { loadConfig, runAll, selectScenarios } from '@inspectr/mcplab-core';
8
+ import { loadConfig, runAll, selectScenarios, loadOrBuildSearchIndex, indexNeedsRefresh, getResultsIndexPaths, resolveRunArtifactPath, searchDocs, getContext } from '@inspectr/mcplab-core';
9
9
  import { renderReport } from '@inspectr/mcplab-reporting';
10
10
  import { parse as parseYaml, stringify as stringifyYaml } from 'yaml';
11
11
  import { z } from 'zod';
@@ -15,6 +15,7 @@ const SERVER_VERSION = typeof PACKAGE_JSON.version === 'string' && PACKAGE_JSON.
15
15
  ? PACKAGE_JSON.version
16
16
  : '0.0.0';
17
17
  const SERVER_ICON_URL = 'https://mcplab.inspectr.dev/favicon.svg';
18
+ const SERVER_OWNED_ROOTS = resolveServerOwnedRoots();
18
19
  const DEFAULT_MCP_PATH = '/mcp';
19
20
  const DEFAULT_MCP_PORT = 3011;
20
21
  const DEFAULT_MCP_HOST = '127.0.0.1';
@@ -36,6 +37,8 @@ const ResultsMetadataSchema = z
36
37
  mcp_server_versions: z.record(z.string())
37
38
  })
38
39
  .passthrough();
40
+ const ResultsQueryStatusSchema = z.enum(['passed', 'failed', 'all']);
41
+ const ResultsQuerySourceSchema = z.enum(['results', 'trace', 'summary']);
39
42
  const MetricSummarySchema = z.object({
40
43
  total_runs: z.number().int().nonnegative(),
41
44
  passed_runs: z.number().int().nonnegative(),
@@ -97,10 +100,28 @@ const AgentEntrySchema = z.object({
97
100
  max_tokens: z.number().int().positive().optional(),
98
101
  system: z.string().optional()
99
102
  });
103
+ const LibraryServerEntryContentSchema = z
104
+ .object({
105
+ transport: z.string().optional(),
106
+ url: z.string().optional(),
107
+ auth: GenericObjectSchema.optional(),
108
+ name: z.string().optional(),
109
+ description: z.string().optional(),
110
+ tags: z.array(z.string()).optional()
111
+ })
112
+ .passthrough();
113
+ const LibraryServerEntrySchema = z.object({
114
+ id: z.string(),
115
+ entry: LibraryServerEntryContentSchema.optional()
116
+ });
117
+ const LibraryAgentEntrySchema = z.object({
118
+ id: z.string(),
119
+ entry: AgentEntrySchema.optional()
120
+ });
100
121
  const LibraryEntrySchema = z.object({
101
122
  bundleRoot: z.string(),
102
- servers: z.union([z.array(z.object({ id: z.string() })), z.record(GenericObjectSchema)]),
103
- agents: z.union([z.array(z.object({ id: z.string() })), z.record(AgentEntrySchema)]),
123
+ servers: z.array(LibraryServerEntrySchema),
124
+ agents: z.array(LibraryAgentEntrySchema),
104
125
  scenarios: z.array(LibraryScenarioEntrySchema)
105
126
  });
106
127
  const ServerAuthSchema = z.union([
@@ -155,6 +176,41 @@ const ConfigSummarySchema = z.object({
155
176
  extract_count: z.number().int().nonnegative()
156
177
  }))
157
178
  });
179
+ const ResolvedScenarioSchema = z.object({
180
+ id: z.string(),
181
+ servers: z.array(z.string()),
182
+ agent: z.string().optional(),
183
+ prompt: z.string().optional(),
184
+ eval: z
185
+ .object({
186
+ type: z.string(),
187
+ assertions: z.array(z.string()).optional(),
188
+ rubric: GenericObjectSchema.optional()
189
+ })
190
+ .passthrough()
191
+ .optional(),
192
+ extract: z
193
+ .array(z
194
+ .object({
195
+ name: z.string(),
196
+ from: z.string().optional(),
197
+ regex: z.string().optional(),
198
+ path: z.string().optional(),
199
+ expression: z.string().optional(),
200
+ transform: z.string().optional()
201
+ })
202
+ .passthrough())
203
+ .optional()
204
+ });
205
+ const RunDefaultsSchema = z
206
+ .object({
207
+ selected_agents: z.array(z.string()).optional(),
208
+ runs_per_scenario: z.number().int().positive().optional(),
209
+ timeout_ms: z.number().int().positive().optional(),
210
+ retries: z.number().int().nonnegative().optional(),
211
+ concurrency: z.number().int().positive().optional()
212
+ })
213
+ .passthrough();
158
214
  const ToolAnalysisListItemSchema = z.object({
159
215
  report_id: z.string(),
160
216
  path: z.string().optional(),
@@ -280,37 +336,98 @@ const WriteMarkdownReportSuccessSchema = z.object({
280
336
  overwritten: z.boolean().describe('True when an existing file was replaced.'),
281
337
  workspace_root: z.string().describe('Workspace root used for path safety validation.')
282
338
  });
339
+ const WriteMarkdownReportErrorCodeSchema = z.enum([
340
+ 'PATH_ESCAPE',
341
+ 'PERMISSION_DENIED',
342
+ 'FILE_EXISTS',
343
+ 'INVALID_EXTENSION',
344
+ 'PARENT_DIR_MISSING',
345
+ 'IO_ERROR'
346
+ ]);
283
347
  const WriteMarkdownReportErrorSchema = z.object({
284
348
  ok: z.literal(false),
285
- error_code: z.enum([
286
- 'PATH_ESCAPE',
287
- 'PERMISSION_DENIED',
288
- 'FILE_EXISTS',
289
- 'INVALID_EXTENSION',
290
- 'PARENT_DIR_MISSING',
291
- 'IO_ERROR'
292
- ]),
349
+ error_code: WriteMarkdownReportErrorCodeSchema,
293
350
  error_message: z.string(),
294
351
  attempted_path: z.string().optional(),
295
352
  violated_constraint: z.string().optional()
296
353
  });
297
- const GenerateServerEntryInputSchema = z
298
- .object({
299
- id: z.string(),
300
- url: z.string(),
301
- transport: z.enum(['http']).optional(),
302
- auth_type: z.enum(['none', 'bearer', 'api_key', 'oauth_client_credentials']).optional(),
303
- bearer_token: z.string().optional(),
304
- bearer_env: z.string().optional(),
305
- api_key_header_name: z.string().optional(),
306
- api_key_value: z.string().optional(),
307
- oauth_token_url: z.string().optional(),
308
- oauth_client_id_env: z.string().optional(),
309
- oauth_client_secret_env: z.string().optional(),
310
- oauth_scope: z.string().optional(),
311
- oauth_audience: z.string().optional()
312
- })
313
- .superRefine((value, ctx) => {
354
+ const WriteMarkdownReportOutputSchema = z.object({
355
+ ok: z.boolean(),
356
+ path: z.string().optional(),
357
+ bytes: z.number().int().nonnegative().optional(),
358
+ chars: z.number().int().nonnegative().optional(),
359
+ overwritten: z.boolean().optional(),
360
+ workspace_root: z.string().optional(),
361
+ error_code: WriteMarkdownReportErrorCodeSchema.optional(),
362
+ error_message: z.string().optional(),
363
+ attempted_path: z.string().optional(),
364
+ violated_constraint: z.string().optional()
365
+ });
366
+ const SafeRelativePathSchema = z
367
+ .string()
368
+ .max(200)
369
+ .regex(/^(?!\/)(?![A-Za-z]:)(?!.*(?:^|\/)\.\.(?:\/|$))[\w./-]+$/, 'Must be a relative workspace path without ".." segments or absolute prefixes.');
370
+ const GenerateServerEntryInputCoreSchema = z.object({
371
+ id: z.string().describe('Server id key (kebab-case recommended).'),
372
+ url: z.string().describe('MCP server URL (Streamable HTTP endpoint).'),
373
+ transport: z.enum(['http']).optional().describe('MCPLab transport type (currently http).')
374
+ });
375
+ const GenerateServerEntryInputBaseSchema = GenerateServerEntryInputCoreSchema.extend({
376
+ auth_type: z
377
+ .enum(['none', 'bearer', 'api_key', 'oauth_client_credentials'])
378
+ .optional()
379
+ .describe('Authentication mode.'),
380
+ bearer_token: z
381
+ .string()
382
+ .optional()
383
+ .describe('Direct bearer token value or ${VAR} env reference when auth_type=bearer.'),
384
+ bearer_env: z.string().optional().describe('Env var for bearer token when auth_type=bearer.'),
385
+ api_key_header_name: z
386
+ .string()
387
+ .optional()
388
+ .describe('Header name for API key auth (default: X-API-Key).'),
389
+ api_key_value: z
390
+ .string()
391
+ .optional()
392
+ .describe('API key value or ${VAR} env reference when auth_type=api_key.'),
393
+ oauth_token_url: z
394
+ .string()
395
+ .optional()
396
+ .describe('OAuth token URL when auth_type=oauth_client_credentials.'),
397
+ oauth_client_id_env: z.string().optional().describe('OAuth client id env var.'),
398
+ oauth_client_secret_env: z.string().optional().describe('OAuth client secret env var.'),
399
+ oauth_scope: z.string().optional().describe('Optional OAuth scope.'),
400
+ oauth_audience: z.string().optional().describe('Optional OAuth audience.')
401
+ });
402
+ const GenerateServerEntryPublicInputSchema = z.union([
403
+ GenerateServerEntryInputCoreSchema.extend({
404
+ auth_type: z.enum(['none']).optional()
405
+ }),
406
+ GenerateServerEntryInputCoreSchema.extend({
407
+ auth_type: z.literal('bearer'),
408
+ bearer_token: z.string(),
409
+ bearer_env: z.string().optional()
410
+ }),
411
+ GenerateServerEntryInputCoreSchema.extend({
412
+ auth_type: z.literal('bearer'),
413
+ bearer_token: z.string().optional(),
414
+ bearer_env: z.string()
415
+ }),
416
+ GenerateServerEntryInputCoreSchema.extend({
417
+ auth_type: z.literal('api_key'),
418
+ api_key_header_name: z.string().optional(),
419
+ api_key_value: z.string()
420
+ }),
421
+ GenerateServerEntryInputCoreSchema.extend({
422
+ auth_type: z.literal('oauth_client_credentials'),
423
+ oauth_token_url: z.string(),
424
+ oauth_client_id_env: z.string(),
425
+ oauth_client_secret_env: z.string(),
426
+ oauth_scope: z.string().optional(),
427
+ oauth_audience: z.string().optional()
428
+ })
429
+ ]);
430
+ const GenerateServerEntryInputSchema = GenerateServerEntryInputBaseSchema.superRefine((value, ctx) => {
314
431
  const authType = value.auth_type ?? 'none';
315
432
  if (authType === 'bearer') {
316
433
  if (!value.bearer_token && !value.bearer_env) {
@@ -433,10 +550,12 @@ export function registerTools(server) {
433
550
  };
434
551
  registerTool('mcplab_write_markdown_report', {
435
552
  description: 'Write a Markdown (.md or .markdown) file to a path within the current workspace. Returns structured output with ok:true and the resolved path on success. On failure, returns ok:false with an error_code from: PATH_ESCAPE (path traversal attempt), FILE_EXISTS (file already exists and overwrite is false), PERMISSION_DENIED, PARENT_DIR_MISSING (create_dirs is false and parent does not exist), INVALID_EXTENSION (not .md or .markdown), IO_ERROR.',
436
- outputSchema: z.union([WriteMarkdownReportSuccessSchema, WriteMarkdownReportErrorSchema]),
553
+ outputSchema: WriteMarkdownReportOutputSchema,
437
554
  inputSchema: {
438
555
  output_path: z
439
556
  .string()
557
+ .max(200)
558
+ .regex(/^[^\0]+\.(?:md|markdown)$/i, 'output_path must end with .md or .markdown.')
440
559
  .describe("Target .md/.markdown path. Use a relative path (e.g. mcplab/reports/my-report.md) — relative paths are always safe and resolve against the server's working directory (process.cwd()). Absolute paths are accepted only if they stay inside that directory; any path that escapes it is rejected with error_code PATH_ESCAPE."),
441
560
  markdown: z
442
561
  .string()
@@ -575,7 +694,11 @@ export function registerTools(server) {
575
694
  reports_dir: z.string(),
576
695
  run_id_filter: z.string().optional(),
577
696
  query: z.string().optional(),
697
+ offset: z.number().int().min(0),
698
+ limit: z.number().int().positive().max(200),
699
+ returned: z.number().int().nonnegative(),
578
700
  total_matching: z.number().int().nonnegative(),
701
+ next_offset: z.number().int().min(0).nullable(),
579
702
  items: z.array(z.object({
580
703
  path: z.string(),
581
704
  relativePath: z.string(),
@@ -585,10 +708,6 @@ export function registerTools(server) {
585
708
  }))
586
709
  },
587
710
  inputSchema: {
588
- reports_dir: z
589
- .string()
590
- .optional()
591
- .describe('Markdown reports root (default mcplab/reports).'),
592
711
  run_id: z
593
712
  .string()
594
713
  .optional()
@@ -602,12 +721,18 @@ export function registerTools(server) {
602
721
  .int()
603
722
  .positive()
604
723
  .max(200)
605
- .optional()
606
- .describe('Max reports to return (default 20).')
724
+ .default(20)
725
+ .describe('Max reports to return (default 20).'),
726
+ offset: z
727
+ .number()
728
+ .int()
729
+ .min(0)
730
+ .default(0)
731
+ .describe('Pagination offset into matching results (default 0).')
607
732
  }
608
- }, async ({ reports_dir, run_id, query, limit }) => {
733
+ }, async ({ run_id, query, limit, offset }) => {
609
734
  return withToolHandling(async () => {
610
- const root = resolveMarkdownReportsDir(reports_dir);
735
+ const root = resolveMarkdownReportsDir();
611
736
  const all = listMarkdownReportsFromDisk(root);
612
737
  const runFilter = String(run_id ?? '').trim();
613
738
  const searchQuery = String(query ?? '')
@@ -624,18 +749,25 @@ export function registerTools(server) {
624
749
  const hay = `${item.path}\n${item.relativePath}\n${item.name}`.toLowerCase();
625
750
  return hay.includes(searchQuery);
626
751
  });
627
- const capped = filtered.slice(0, limit ?? 20);
752
+ const start = Math.max(0, offset ?? 0);
753
+ const pageSize = limit ?? 20;
754
+ const capped = filtered.slice(start, start + pageSize);
755
+ const nextOffset = start + capped.length < filtered.length ? start + capped.length : null;
628
756
  return ok(`Found ${capped.length}/${filtered.length} markdown report(s) in ${root}`, {
629
757
  reports_dir: root,
630
758
  run_id_filter: runFilter || undefined,
631
759
  query: searchQuery || undefined,
760
+ offset: start,
761
+ limit: pageSize,
762
+ returned: capped.length,
632
763
  total_matching: filtered.length,
764
+ next_offset: nextOffset,
633
765
  items: capped
634
766
  });
635
767
  });
636
768
  });
637
769
  registerTool('mcplab_read_markdown_report', {
638
- description: 'Read a saved markdown report by relative path (under mcplab/reports by default) or by workspace-relative path, with optional truncation.',
770
+ description: 'Read a saved markdown report by reports-root-relative path (under mcplab/reports), with optional truncation.',
639
771
  outputSchema: {
640
772
  reports_dir: z.string(),
641
773
  path: z.string(),
@@ -647,13 +779,7 @@ export function registerTools(server) {
647
779
  content: z.string()
648
780
  },
649
781
  inputSchema: {
650
- path: z
651
- .string()
652
- .describe('Report path (relative to reports root or workspace-relative, e.g. mcplab/reports/... ).'),
653
- reports_dir: z
654
- .string()
655
- .optional()
656
- .describe('Markdown reports root (default mcplab/reports).'),
782
+ path: SafeRelativePathSchema.describe('Report path relative to the reports root (e.g. team/run-2026-05-03.md). Do not prefix with mcplab/reports/.'),
657
783
  max_chars: z
658
784
  .number()
659
785
  .int()
@@ -661,9 +787,10 @@ export function registerTools(server) {
661
787
  .optional()
662
788
  .describe('Optional truncation for markdown content preview (default 20000).')
663
789
  }
664
- }, async ({ path, reports_dir, max_chars }) => {
790
+ }, async ({ path, max_chars }) => {
665
791
  return withToolHandling(async () => {
666
- const root = resolveMarkdownReportsDir(reports_dir);
792
+ validateWorkspaceRelativePath(path, 'path');
793
+ const root = resolveMarkdownReportsDir();
667
794
  const targetPath = resolveMarkdownReportPath(root, path);
668
795
  if (!isMarkdownReportExt(targetPath)) {
669
796
  throw new Error('path must point to a .md or .markdown file');
@@ -692,10 +819,6 @@ export function registerTools(server) {
692
819
  description: 'List reusable MCPLab library entries (servers, agents, scenarios) from a bundle root such as mcplab/ or examples/libraries/.',
693
820
  outputSchema: LibraryEntrySchema,
694
821
  inputSchema: {
695
- bundleRoot: z
696
- .string()
697
- .optional()
698
- .describe('Optional library bundle root. Defaults to mcplab/ or examples/libraries/ if present.'),
699
822
  kind: z
700
823
  .enum(['all', 'servers', 'agents', 'scenarios'])
701
824
  .optional()
@@ -705,16 +828,18 @@ export function registerTools(server) {
705
828
  .optional()
706
829
  .describe('Include parsed YAML content for each item (larger output).')
707
830
  }
708
- }, async ({ bundleRoot, kind, includeContent }) => {
831
+ }, async ({ kind, includeContent }) => {
709
832
  return withToolHandling(async () => {
710
- const root = resolveBundleRoot(bundleRoot);
833
+ const root = resolveBundleRoot();
711
834
  const data = readLibrary(root, Boolean(includeContent));
712
835
  const selectedKind = kind ?? 'all';
713
836
  const structured = selectedKind === 'all'
714
837
  ? data
715
838
  : {
716
839
  bundleRoot: data.bundleRoot,
717
- [selectedKind]: data[selectedKind]
840
+ servers: selectedKind === 'servers' ? data.servers : [],
841
+ agents: selectedKind === 'agents' ? data.agents : [],
842
+ scenarios: selectedKind === 'scenarios' ? data.scenarios : []
718
843
  };
719
844
  return ok(`Loaded MCPLab library from ${root}`, structured);
720
845
  });
@@ -730,13 +855,12 @@ export function registerTools(server) {
730
855
  content: GenericObjectSchema
731
856
  },
732
857
  inputSchema: {
733
- bundleRoot: z.string().optional().describe('Optional library bundle root path.'),
734
858
  kind: z.enum(['servers', 'agents', 'scenarios']).describe('Library category.'),
735
859
  id: z.string().describe('Entry id (for scenarios this is scenario.id, not filename).')
736
860
  }
737
- }, async ({ bundleRoot, kind, id }) => {
861
+ }, async ({ kind, id }) => {
738
862
  return withToolHandling(async () => {
739
- const root = resolveBundleRoot(bundleRoot);
863
+ const root = resolveBundleRoot();
740
864
  const item = getLibraryItem(root, kind, id);
741
865
  return ok(`Loaded ${kind.slice(0, -1)} '${id}' from ${root}`, item);
742
866
  });
@@ -748,39 +872,7 @@ export function registerTools(server) {
748
872
  entry: ServerEntrySchema,
749
873
  yaml: z.string()
750
874
  },
751
- inputSchema: {
752
- id: z.string().describe('Server id key (kebab-case recommended).'),
753
- url: z.string().describe('MCP server URL (Streamable HTTP endpoint).'),
754
- transport: z.enum(['http']).optional().describe('MCPLab transport type (currently http).'),
755
- auth_type: z
756
- .enum(['none', 'bearer', 'api_key', 'oauth_client_credentials'])
757
- .optional()
758
- .describe('Authentication mode.'),
759
- bearer_token: z
760
- .string()
761
- .optional()
762
- .describe('Direct bearer token value or ${VAR} env reference when auth_type=bearer.'),
763
- bearer_env: z
764
- .string()
765
- .optional()
766
- .describe('Env var for bearer token when auth_type=bearer.'),
767
- api_key_header_name: z
768
- .string()
769
- .optional()
770
- .describe('Header name for API key auth (default: X-API-Key).'),
771
- api_key_value: z
772
- .string()
773
- .optional()
774
- .describe('API key value or ${VAR} env reference when auth_type=api_key.'),
775
- oauth_token_url: z
776
- .string()
777
- .optional()
778
- .describe('OAuth token URL when auth_type=oauth_client_credentials.'),
779
- oauth_client_id_env: z.string().optional().describe('OAuth client id env var.'),
780
- oauth_client_secret_env: z.string().optional().describe('OAuth client secret env var.'),
781
- oauth_scope: z.string().optional().describe('Optional OAuth scope.'),
782
- oauth_audience: z.string().optional().describe('Optional OAuth audience.')
783
- }
875
+ inputSchema: GenerateServerEntryPublicInputSchema
784
876
  }, async (input) => {
785
877
  return withToolHandling(async () => {
786
878
  const parsed = GenerateServerEntryInputSchema.parse(input);
@@ -904,33 +996,27 @@ export function registerTools(server) {
904
996
  resolved_config: z.object({
905
997
  servers: z.record(GenericObjectSchema),
906
998
  agents: z.record(AgentEntrySchema),
907
- scenarios: z.array(GenericObjectSchema),
908
- run_defaults: GenericObjectSchema.optional()
999
+ scenarios: z.array(ResolvedScenarioSchema),
1000
+ run_defaults: RunDefaultsSchema.optional()
909
1001
  })
910
1002
  },
911
1003
  inputSchema: {
912
1004
  config_path: z.string().describe('Path to MCPLab eval YAML config.'),
913
- bundle_root: z
914
- .string()
915
- .optional()
916
- .describe('Optional bundle root override for refs resolution.'),
917
1005
  scenario_id: z
918
1006
  .string()
919
1007
  .optional()
920
1008
  .describe('Optional single scenario id to validate selection.')
921
1009
  }
922
- }, async ({ config_path, bundle_root, scenario_id }) => {
1010
+ }, async ({ config_path, scenario_id }) => {
923
1011
  return withToolHandling(async () => {
924
1012
  const loaded = loadConfig(resolve(config_path), {
925
- bundleRoot: bundle_root ? resolve(bundle_root) : undefined
1013
+ bundleRoot: resolveBundleRoot()
926
1014
  });
927
1015
  const selected = selectScenarios(loaded.config, scenario_id);
928
1016
  const summary = summarizeConfig(selected);
929
1017
  return ok(`Validated config ${config_path}`, {
930
1018
  configPath: resolve(config_path),
931
- bundleRoot: bundle_root
932
- ? resolve(bundle_root)
933
- : detectLikelyBundleRoot(resolve(config_path)),
1019
+ bundleRoot: detectLikelyBundleRoot(resolve(config_path)),
934
1020
  hash: loaded.hash,
935
1021
  summary,
936
1022
  resolved_config: selected
@@ -959,26 +1045,18 @@ export function registerTools(server) {
959
1045
  },
960
1046
  inputSchema: {
961
1047
  config_path: z.string().describe('Path to MCPLab eval YAML config.'),
962
- bundle_root: z
963
- .string()
964
- .optional()
965
- .describe('Optional bundle root override for library refs.'),
966
1048
  scenario_id: z.string().optional().describe('Optional scenario id to run.'),
967
1049
  runs_per_scenario: z
968
1050
  .number()
969
1051
  .int()
970
1052
  .positive()
971
1053
  .optional()
972
- .describe('Runs per scenario (default 1).'),
973
- runs_dir: z
974
- .string()
975
- .optional()
976
- .describe('Output directory for run artifacts (default mcplab/results/evaluation-runs).')
1054
+ .describe('Runs per scenario (default 1).')
977
1055
  }
978
- }, async ({ config_path, bundle_root, scenario_id, runs_per_scenario, runs_dir }) => {
1056
+ }, async ({ config_path, scenario_id, runs_per_scenario }) => {
979
1057
  return withToolHandling(async () => {
980
1058
  const loaded = loadConfig(resolve(config_path), {
981
- bundleRoot: bundle_root ? resolve(bundle_root) : undefined
1059
+ bundleRoot: resolveBundleRoot()
982
1060
  });
983
1061
  const selected = selectScenarios(loaded.config, scenario_id);
984
1062
  const executable = expandConfigForAgents(selected, selected.run_defaults?.selected_agents);
@@ -987,7 +1065,7 @@ export function registerTools(server) {
987
1065
  scenarioId: scenario_id,
988
1066
  configHash: loaded.hash,
989
1067
  cliVersion: `mcplab-mcp-server/${SERVER_VERSION}`,
990
- runsDir: runs_dir ?? 'mcplab/results/evaluation-runs'
1068
+ runsDir: resolveRunsDir()
991
1069
  });
992
1070
  const reportHtml = renderReport(results);
993
1071
  const allRuns = results.scenarios.flatMap((scenario) => scenario.runs);
@@ -1024,54 +1102,6 @@ export function registerTools(server) {
1024
1102
  });
1025
1103
  });
1026
1104
  });
1027
- registerTool('mcplab_search_runs', {
1028
- description: 'Search MCPLab run artifact directories with results.json summary metrics. Use the optional query parameter to filter by run_id, path, or summary fields (case-insensitive substring). Set include_summary=false to skip reading results.json for faster listing.',
1029
- outputSchema: {
1030
- runsDir: z.string(),
1031
- query: z.string().optional(),
1032
- total_matching: z.number().int().nonnegative(),
1033
- runs: z.array(RunListEntrySchema)
1034
- },
1035
- inputSchema: {
1036
- runs_dir: z
1037
- .string()
1038
- .optional()
1039
- .describe('Runs directory (default mcplab/results/evaluation-runs).'),
1040
- query: z
1041
- .string()
1042
- .optional()
1043
- .describe('Optional case-insensitive search query across run id/path and summary fields.'),
1044
- limit: z
1045
- .number()
1046
- .int()
1047
- .positive()
1048
- .max(100)
1049
- .default(10)
1050
- .describe('Max runs to return. Defaults to 10.'),
1051
- include_summary: z
1052
- .boolean()
1053
- .default(true)
1054
- .describe('Read results.json summary for each run when available. Defaults to true.')
1055
- }
1056
- }, async ({ runs_dir, query, limit, include_summary }) => {
1057
- return withToolHandling(async () => {
1058
- const base = resolveRunsDir(runs_dir);
1059
- const entries = listRunsWithFallback(base, undefined, include_summary);
1060
- const searchQuery = String(query ?? '')
1061
- .trim()
1062
- .toLowerCase();
1063
- const filtered = searchQuery
1064
- ? entries.filter((entry) => searchableText(entry).includes(searchQuery))
1065
- : entries;
1066
- const capped = filtered.slice(0, limit);
1067
- return ok(`Found ${filtered.length} run(s) in ${base}`, {
1068
- runsDir: base,
1069
- query: searchQuery || undefined,
1070
- total_matching: filtered.length,
1071
- runs: capped
1072
- });
1073
- });
1074
- });
1075
1105
  registerTool('mcplab_aggregate_runs', {
1076
1106
  description: 'Aggregate metrics across historical MCPLab runs with compact summary-first output.',
1077
1107
  outputSchema: z.object({
@@ -1095,10 +1125,6 @@ export function registerTools(server) {
1095
1125
  details: z.array(AggregateRowSchema).optional()
1096
1126
  }),
1097
1127
  inputSchema: {
1098
- runs_dir: z
1099
- .string()
1100
- .optional()
1101
- .describe('Runs directory (default mcplab/results/evaluation-runs).'),
1102
1128
  run_ids: z
1103
1129
  .array(z.string())
1104
1130
  .optional()
@@ -1128,10 +1154,9 @@ export function registerTools(server) {
1128
1154
  .optional()
1129
1155
  .describe('Include full grouped rows. Defaults to false (summary-first).')
1130
1156
  }
1131
- }, async ({ runs_dir, run_ids, latest_n, scenario_ids, agents, group_by, top_n, include_details }) => {
1157
+ }, async ({ run_ids, latest_n, scenario_ids, agents, group_by, top_n, include_details }) => {
1132
1158
  return withToolHandling(async () => {
1133
1159
  const loaded = loadRunsForAnalysis({
1134
- runsDirInput: runs_dir,
1135
1160
  runIds: run_ids,
1136
1161
  latestN: latest_n ?? 20
1137
1162
  });
@@ -1189,10 +1214,6 @@ export function registerTools(server) {
1189
1214
  details: z.array(CompareRowSchema).optional()
1190
1215
  }),
1191
1216
  inputSchema: {
1192
- runs_dir: z
1193
- .string()
1194
- .optional()
1195
- .describe('Runs directory (default mcplab/results/evaluation-runs).'),
1196
1217
  left_run_id: z.string().describe("Left run id or 'LATEST'."),
1197
1218
  right_run_id: z.string().describe("Right run id or 'LATEST'."),
1198
1219
  scenario_ids: z.array(z.string()).optional().describe('Optional scenario id filter.'),
@@ -1209,9 +1230,9 @@ export function registerTools(server) {
1209
1230
  .optional()
1210
1231
  .describe('Include full classification rows. Defaults to false (summary-first).')
1211
1232
  }
1212
- }, async ({ runs_dir, left_run_id, right_run_id, scenario_ids, agents, top_n, include_details }) => {
1233
+ }, async ({ left_run_id, right_run_id, scenario_ids, agents, top_n, include_details }) => {
1213
1234
  return withToolHandling(async () => {
1214
- const base = resolveRunsDir(runs_dir);
1235
+ const base = resolveRunsDir();
1215
1236
  const left = loadSingleRunForAnalysis(base, left_run_id);
1216
1237
  const right = loadSingleRunForAnalysis(base, right_run_id);
1217
1238
  const report = buildCompareRunsReport({
@@ -1233,10 +1254,6 @@ export function registerTools(server) {
1233
1254
  items: z.array(ToolAnalysisListItemSchema)
1234
1255
  },
1235
1256
  inputSchema: {
1236
- tool_analysis_results_dir: z
1237
- .string()
1238
- .optional()
1239
- .describe('Directory containing saved tool analysis report folders.'),
1240
1257
  query: z
1241
1258
  .string()
1242
1259
  .optional()
@@ -1249,9 +1266,9 @@ export function registerTools(server) {
1249
1266
  .default(20)
1250
1267
  .describe('Max reports to return. Defaults to 20.')
1251
1268
  }
1252
- }, async ({ tool_analysis_results_dir, query, limit }) => {
1269
+ }, async ({ query, limit }) => {
1253
1270
  return withToolHandling(async () => {
1254
- const baseDir = resolveToolAnalysisResultsDir(tool_analysis_results_dir);
1271
+ const baseDir = resolveToolAnalysisResultsDir();
1255
1272
  const reports = listToolAnalysisReportsFromDiskWithFallback(baseDir, undefined);
1256
1273
  const searchQuery = String(query ?? '')
1257
1274
  .trim()
@@ -1280,10 +1297,6 @@ export function registerTools(server) {
1280
1297
  }),
1281
1298
  inputSchema: {
1282
1299
  report_id: z.string().describe("Report id directory name (or 'LATEST')."),
1283
- tool_analysis_results_dir: z
1284
- .string()
1285
- .optional()
1286
- .describe('Directory containing saved tool analysis reports (default mcplab/results/tool-analysis).'),
1287
1300
  max_chars: z
1288
1301
  .number()
1289
1302
  .int()
@@ -1295,9 +1308,9 @@ export function registerTools(server) {
1295
1308
  .optional()
1296
1309
  .describe('Include the full parsed record in structured content. Defaults to true.')
1297
1310
  }
1298
- }, async ({ report_id, tool_analysis_results_dir, max_chars, include_record }) => {
1311
+ }, async ({ report_id, max_chars, include_record }) => {
1299
1312
  return withToolHandling(async () => {
1300
- const baseDir = resolveToolAnalysisResultsDir(tool_analysis_results_dir);
1313
+ const baseDir = resolveToolAnalysisResultsDir();
1301
1314
  const resolvedReportId = report_id === 'LATEST'
1302
1315
  ? latestToolAnalysisReportIdWithFallback(baseDir)
1303
1316
  : report_id.trim();
@@ -1336,22 +1349,18 @@ export function registerTools(server) {
1336
1349
  },
1337
1350
  inputSchema: {
1338
1351
  report_id: z.string().describe('Report id directory name to delete.'),
1339
- tool_analysis_results_dir: z
1340
- .string()
1341
- .optional()
1342
- .describe('Directory containing saved tool analysis reports (default mcplab/results/tool-analysis).'),
1343
1352
  dry_run: z
1344
1353
  .boolean()
1345
- .optional()
1346
- .describe('If true, return what would be deleted without deleting anything.'),
1354
+ .default(false)
1355
+ .describe('If true, return what would be deleted without deleting anything. Defaults to false.'),
1347
1356
  confirm: z
1348
1357
  .boolean()
1349
- .optional()
1350
- .describe('Must be true to execute deletion when dry_run is false.')
1358
+ .default(false)
1359
+ .describe('Must be true to execute deletion when dry_run is false. If confirm is false, deletion is rejected with an error.')
1351
1360
  }
1352
- }, async ({ report_id, tool_analysis_results_dir, dry_run, confirm }) => {
1361
+ }, async ({ report_id, dry_run, confirm }) => {
1353
1362
  return withToolHandling(async () => {
1354
- const baseDir = resolveToolAnalysisResultsDir(tool_analysis_results_dir);
1363
+ const baseDir = resolveToolAnalysisResultsDir();
1355
1364
  const dirPath = toolAnalysisReportDirPathWithFallback(baseDir, report_id.trim());
1356
1365
  const existed = existsSync(dirPath);
1357
1366
  const isDryRun = Boolean(dry_run);
@@ -1401,10 +1410,6 @@ export function registerTools(server) {
1401
1410
  items: z.array(FlattenedTraceItemSchema)
1402
1411
  },
1403
1412
  inputSchema: {
1404
- runs_dir: z
1405
- .string()
1406
- .optional()
1407
- .describe('Runs directory (default mcplab/results/evaluation-runs).'),
1408
1413
  run_id: z.string().describe("Run id directory name or 'LATEST'."),
1409
1414
  event_types: z
1410
1415
  .array(z.string())
@@ -1420,9 +1425,9 @@ export function registerTools(server) {
1420
1425
  .optional()
1421
1426
  .describe('Max items to return (default 200).')
1422
1427
  }
1423
- }, async ({ runs_dir, run_id, event_types, scenario_id, agent, limit }) => {
1428
+ }, async ({ run_id, event_types, scenario_id, agent, limit }) => {
1424
1429
  return withToolHandling(async () => {
1425
- const { runId, records, legacyDetected } = readScenarioRunTraceRecordsForRun(runs_dir, run_id);
1430
+ const { runId, records, legacyDetected } = readScenarioRunTraceRecordsForRun(run_id);
1426
1431
  const typeSet = event_types?.length
1427
1432
  ? new Set(event_types)
1428
1433
  : null;
@@ -1464,10 +1469,6 @@ export function registerTools(server) {
1464
1469
  }))
1465
1470
  },
1466
1471
  inputSchema: {
1467
- runs_dir: z
1468
- .string()
1469
- .optional()
1470
- .describe('Runs directory (default mcplab/results/evaluation-runs).'),
1471
1472
  run_id: z.string().describe("Run id directory name or 'LATEST'."),
1472
1473
  scenario_id: z.string().optional().describe('Optional scenario id filter.'),
1473
1474
  agent: z.string().optional().describe('Optional agent filter.'),
@@ -1479,9 +1480,9 @@ export function registerTools(server) {
1479
1480
  .optional()
1480
1481
  .describe('Optional truncation per final answer text (default 8000).')
1481
1482
  }
1482
- }, async ({ runs_dir, run_id, scenario_id, agent, max_chars_per_answer }) => {
1483
+ }, async ({ run_id, scenario_id, agent, max_chars_per_answer }) => {
1483
1484
  return withToolHandling(async () => {
1484
- const { runId, records, legacyDetected } = readScenarioRunTraceRecordsForRun(runs_dir, run_id);
1485
+ const { runId, records, legacyDetected } = readScenarioRunTraceRecordsForRun(run_id);
1485
1486
  const maxChars = max_chars_per_answer ?? 8000;
1486
1487
  const items = records
1487
1488
  .filter((record) => (!scenario_id || record.scenario_id === scenario_id) &&
@@ -1518,10 +1519,6 @@ export function registerTools(server) {
1518
1519
  timeline: z.array(ConversationTimelineItemSchema)
1519
1520
  },
1520
1521
  inputSchema: {
1521
- runs_dir: z
1522
- .string()
1523
- .optional()
1524
- .describe('Runs directory (default mcplab/results/evaluation-runs).'),
1525
1522
  run_id: z.string().describe("Run id directory name or 'LATEST'."),
1526
1523
  scenario_id: z.string().describe('Scenario id to filter.'),
1527
1524
  agent: z.string().describe('Agent name to filter.'),
@@ -1540,9 +1537,9 @@ export function registerTools(server) {
1540
1537
  .optional()
1541
1538
  .describe('Max chars for text fields (default 4000).')
1542
1539
  }
1543
- }, async ({ runs_dir, run_id, scenario_id, agent, max_items, max_text_chars }) => {
1540
+ }, async ({ run_id, scenario_id, agent, max_items, max_text_chars }) => {
1544
1541
  return withToolHandling(async () => {
1545
- const { runId, records, legacyDetected } = readScenarioRunTraceRecordsForRun(runs_dir, run_id);
1542
+ const { runId, records, legacyDetected } = readScenarioRunTraceRecordsForRun(run_id);
1546
1543
  const textMax = max_text_chars ?? 4000;
1547
1544
  const record = records.find((r) => r.scenario_id === scenario_id && r.agent === agent);
1548
1545
  const timeline = record
@@ -1561,15 +1558,11 @@ export function registerTools(server) {
1561
1558
  description: 'Search scenario_run trace content for a text query and return matching message/block items.',
1562
1559
  outputSchema: {
1563
1560
  run_id: z.string(),
1564
- query: z.string(),
1561
+ query: z.string().trim().min(1),
1565
1562
  legacy_trace_detected: z.boolean().optional(),
1566
1563
  matches: z.array(FlattenedTraceItemSchema)
1567
1564
  },
1568
1565
  inputSchema: {
1569
- runs_dir: z
1570
- .string()
1571
- .optional()
1572
- .describe('Runs directory (default mcplab/results/evaluation-runs).'),
1573
1566
  run_id: z.string().describe("Run id directory name or 'LATEST'."),
1574
1567
  query: z.string().describe('Case-insensitive text query.'),
1575
1568
  event_types: z
@@ -1584,12 +1577,12 @@ export function registerTools(server) {
1584
1577
  .optional()
1585
1578
  .describe('Max matches to return (default 50).')
1586
1579
  }
1587
- }, async ({ runs_dir, run_id, query, event_types, limit }) => {
1580
+ }, async ({ run_id, query, event_types, limit }) => {
1588
1581
  return withToolHandling(async () => {
1589
1582
  const q = query.trim().toLowerCase();
1590
1583
  if (!q)
1591
1584
  throw new Error('query is required');
1592
- const { runId, records, legacyDetected } = readScenarioRunTraceRecordsForRun(runs_dir, run_id);
1585
+ const { runId, records, legacyDetected } = readScenarioRunTraceRecordsForRun(run_id);
1593
1586
  const typeSet = event_types?.length
1594
1587
  ? new Set(event_types)
1595
1588
  : null;
@@ -1632,15 +1625,11 @@ export function registerTools(server) {
1632
1625
  }))
1633
1626
  },
1634
1627
  inputSchema: {
1635
- runs_dir: z
1636
- .string()
1637
- .optional()
1638
- .describe('Runs directory (default mcplab/results/evaluation-runs).'),
1639
1628
  run_id: z.string().describe("Run id directory name or 'LATEST'.")
1640
1629
  }
1641
- }, async ({ runs_dir, run_id }) => {
1630
+ }, async ({ run_id }) => {
1642
1631
  return withToolHandling(async () => {
1643
- const { runId, records, legacyDetected } = readScenarioRunTraceRecordsForRun(runs_dir, run_id);
1632
+ const { runId, records, legacyDetected } = readScenarioRunTraceRecordsForRun(run_id);
1644
1633
  const messageRoleCounts = {};
1645
1634
  const blockTypeCounts = {};
1646
1635
  const toolUsage = {};
@@ -1711,10 +1700,6 @@ export function registerTools(server) {
1711
1700
  .optional()
1712
1701
  },
1713
1702
  inputSchema: {
1714
- runs_dir: z
1715
- .string()
1716
- .optional()
1717
- .describe('Runs directory (default mcplab/results/evaluation-runs).'),
1718
1703
  run_id: z.string().describe('Run id directory name or LATEST.'),
1719
1704
  artifact: z
1720
1705
  .enum([
@@ -1744,15 +1729,15 @@ export function registerTools(server) {
1744
1729
  .optional()
1745
1730
  .describe('1-indexed line to stop reading at (inclusive).')
1746
1731
  }
1747
- }, async ({ runs_dir, run_id, artifact, max_chars, line_start, line_end }) => {
1732
+ }, async ({ run_id, artifact, max_chars, line_start, line_end }) => {
1748
1733
  return withToolHandling(async () => {
1749
- const base = resolveRunsDir(runs_dir);
1734
+ const base = resolveRunsDir();
1750
1735
  const readBase = resolveExistingRunReadDir(base, run_id === 'LATEST' ? undefined : run_id);
1751
1736
  const resolvedRunId = run_id === 'LATEST' ? latestRunId(readBase) : run_id;
1752
1737
  if (!resolvedRunId) {
1753
1738
  throw new Error(`No runs found in ${base}`);
1754
1739
  }
1755
- const fullPath = join(readBase, resolvedRunId, artifact);
1740
+ const fullPath = resolveRunArtifactPath(readBase, resolvedRunId, artifact);
1756
1741
  if (!existsSync(fullPath)) {
1757
1742
  throw new Error(`Artifact not found: ${fullPath}`);
1758
1743
  }
@@ -1793,111 +1778,132 @@ export function registerTools(server) {
1793
1778
  return ok(`Read ${artifact} from run ${resolvedRunId}`, structured);
1794
1779
  });
1795
1780
  });
1796
- registerTool('mcplab_grep_run_artifact', {
1797
- description: 'Search for text within a MCPLab run artifact and return matching lines with surrounding context. Use this to find specific sections in large files (e.g. a tool name in report.html) without reading the full file. Returns line numbers so you can follow up with mcplab_read_run_artifact line_start/line_end to read the full section.',
1781
+ registerTool('mcplab_results_index', {
1782
+ description: 'Build or refresh local MCPLab results search index under mcplab/results/.index for LLM-first querying.',
1783
+ outputSchema: {
1784
+ runs_dir: z.string(),
1785
+ rebuilt: z.boolean(),
1786
+ doc_count: z.number().int().nonnegative(),
1787
+ index_path: z.string(),
1788
+ manifest_path: z.string()
1789
+ },
1790
+ inputSchema: {
1791
+ rebuild: z.boolean().optional().describe('Force full index rebuild.')
1792
+ }
1793
+ }, async ({ rebuild }) => {
1794
+ return withToolHandling(async () => {
1795
+ const runsDir = resolveRunsDir();
1796
+ const wasStale = indexNeedsRefresh(runsDir);
1797
+ const docs = loadOrBuildSearchIndex(runsDir, Boolean(rebuild));
1798
+ const paths = getResultsIndexPaths(runsDir);
1799
+ return ok(`Results index ready (${docs.length} docs).`, {
1800
+ runs_dir: runsDir,
1801
+ rebuilt: Boolean(rebuild) || wasStale,
1802
+ doc_count: docs.length,
1803
+ index_path: paths.indexPath,
1804
+ manifest_path: paths.manifestPath
1805
+ });
1806
+ });
1807
+ });
1808
+ registerTool('mcplab_results_search', {
1809
+ description: 'Search MCPLab run results in compact LLM-first format. Auto-refreshes index when artifacts changed.',
1798
1810
  outputSchema: {
1799
- run_id: z.string(),
1800
- artifact: z.enum([
1801
- 'results.json',
1802
- 'summary.md',
1803
- 'trace.jsonl',
1804
- 'resolved-config.yaml',
1805
- 'report.html'
1806
- ]),
1807
1811
  query: z.string(),
1808
- total_lines: z.number().int().nonnegative(),
1809
- match_count: z.number().int().nonnegative(),
1810
- truncated_at_limit: z.boolean(),
1811
- matches: z.array(z.object({
1812
- match_line: z.number().int().positive(),
1813
- context_start_line: z.number().int().positive(),
1814
- context_end_line: z.number().int().positive(),
1815
- lines: z.array(z.object({
1816
- line: z.number().int().positive(),
1817
- text: z.string(),
1818
- is_match: z.boolean()
1819
- }))
1812
+ runs_dir: z.string(),
1813
+ total_hits: z.number().int().nonnegative(),
1814
+ hits: z.array(z.object({
1815
+ run_id: z.string(),
1816
+ scenario_id: z.string().optional(),
1817
+ agent: z.string().optional(),
1818
+ status: z.enum(['passed', 'failed']).optional(),
1819
+ source: ResultsQuerySourceSchema,
1820
+ file: z.string(),
1821
+ line_start: z.number().int().positive().optional(),
1822
+ line_end: z.number().int().positive().optional(),
1823
+ snippet: z.string(),
1824
+ score: z.number(),
1825
+ context_command: z.string().optional()
1820
1826
  }))
1821
1827
  },
1822
1828
  inputSchema: {
1823
- runs_dir: z
1824
- .string()
1829
+ query: z.string().trim().min(1).describe('Search query.'),
1830
+ status: ResultsQueryStatusSchema.optional().describe('Filter by status (default all).'),
1831
+ source: z
1832
+ .array(ResultsQuerySourceSchema)
1825
1833
  .optional()
1826
- .describe('Runs directory (default mcplab/results/evaluation-runs).'),
1834
+ .describe('Sources to search (default results,trace,summary).'),
1835
+ scenario: z.string().optional().describe('Filter by scenario id.'),
1836
+ agent: z.string().optional().describe('Filter by agent id.'),
1837
+ limit: z.number().int().positive().max(100).optional().describe('Max hits (default 10).')
1838
+ }
1839
+ }, async ({ query, status, source, scenario, agent, limit }) => {
1840
+ return withToolHandling(async () => {
1841
+ const runsDir = resolveRunsDir();
1842
+ const docs = loadOrBuildSearchIndex(runsDir, false);
1843
+ const hits = searchDocs(docs, {
1844
+ query,
1845
+ status: status ?? 'all',
1846
+ source: source && source.length > 0 ? source : ['results', 'trace', 'summary'],
1847
+ scenario,
1848
+ agent,
1849
+ limit: limit ?? 10
1850
+ });
1851
+ return ok(`Found ${hits.length} result hit(s).`, {
1852
+ query,
1853
+ runs_dir: runsDir,
1854
+ total_hits: hits.length,
1855
+ hits
1856
+ });
1857
+ });
1858
+ });
1859
+ registerTool('mcplab_results_context', {
1860
+ description: 'Fetch focused context for a scenario/run from results, trace, or summary. Returns bounded excerpts only. around is trace-line context and is only valid with source=trace (or when source is omitted).',
1861
+ outputSchema: {
1862
+ run_id: z.string(),
1863
+ scenario_id: z.string(),
1864
+ source: z.enum(['results', 'trace', 'summary', 'mixed']),
1865
+ line_start: z.number().int().positive().optional(),
1866
+ line_end: z.number().int().positive().optional(),
1867
+ excerpt: z.string()
1868
+ },
1869
+ inputSchema: {
1827
1870
  run_id: z.string().describe("Run id directory name or 'LATEST'."),
1828
- artifact: z
1829
- .enum([
1830
- 'results.json',
1831
- 'summary.md',
1832
- 'trace.jsonl',
1833
- 'resolved-config.yaml',
1834
- 'report.html'
1835
- ])
1836
- .describe('Artifact filename to search.'),
1837
- query: z.string().describe('Text to search for (case-insensitive by default).'),
1838
- context_lines: z
1871
+ scenario_id: z.string().describe('Scenario id to focus.'),
1872
+ source: ResultsQuerySourceSchema.optional().describe('Context source; default mixed.'),
1873
+ around: z.number().int().positive().optional().describe('Trace line center.'),
1874
+ before: z
1839
1875
  .number()
1840
1876
  .int()
1841
1877
  .min(0)
1842
- .max(50)
1878
+ .max(200)
1843
1879
  .optional()
1844
- .describe('Lines of context before and after each match (default 5).'),
1845
- max_matches: z
1880
+ .describe('Lines before around for trace (default 20).'),
1881
+ after: z
1846
1882
  .number()
1847
1883
  .int()
1848
- .positive()
1849
- .max(50)
1884
+ .min(0)
1885
+ .max(200)
1850
1886
  .optional()
1851
- .describe('Maximum number of matches to return (default 10).'),
1852
- case_sensitive: z.boolean().optional().describe('Case-sensitive search (default false).')
1887
+ .describe('Lines after around for trace (default 20).')
1853
1888
  }
1854
- }, async ({ runs_dir, run_id, artifact, query, context_lines, max_matches, case_sensitive }) => {
1889
+ }, async ({ run_id, scenario_id, source, around, before, after }) => {
1855
1890
  return withToolHandling(async () => {
1856
- const base = resolveRunsDir(runs_dir);
1891
+ const base = resolveRunsDir();
1857
1892
  const readBase = resolveExistingRunReadDir(base, run_id === 'LATEST' ? undefined : run_id);
1858
1893
  const resolvedRunId = run_id === 'LATEST' ? latestRunId(readBase) : run_id;
1859
1894
  if (!resolvedRunId)
1860
1895
  throw new Error(`No runs found in ${base}`);
1861
- const fullPath = join(readBase, resolvedRunId, artifact);
1862
- if (!existsSync(fullPath))
1863
- throw new Error(`Artifact not found: ${fullPath}`);
1864
- const lines = readFileSync(fullPath, 'utf8').split('\n');
1865
- const q = case_sensitive ? query.trim() : query.trim().toLowerCase();
1866
- if (!q)
1867
- throw new Error('query must not be empty');
1868
- const ctx = context_lines ?? 5;
1869
- const limit = max_matches ?? 10;
1870
- const matchIndices = [];
1871
- for (let i = 0; i < lines.length; i++) {
1872
- const hay = case_sensitive ? lines[i] : lines[i].toLowerCase();
1873
- if (hay.includes(q)) {
1874
- matchIndices.push(i);
1875
- if (matchIndices.length >= limit)
1876
- break;
1877
- }
1878
- }
1879
- const matches = matchIndices.map((matchIdx) => {
1880
- const start = Math.max(0, matchIdx - ctx);
1881
- const end = Math.min(lines.length - 1, matchIdx + ctx);
1882
- return {
1883
- match_line: matchIdx + 1,
1884
- context_start_line: start + 1,
1885
- context_end_line: end + 1,
1886
- lines: lines.slice(start, end + 1).map((text, offset) => ({
1887
- line: start + offset + 1,
1888
- text,
1889
- is_match: start + offset === matchIdx
1890
- }))
1891
- };
1896
+ const result = getContext({
1897
+ runsDir: readBase,
1898
+ runId: resolvedRunId,
1899
+ scenarioId: scenario_id,
1900
+ source,
1901
+ around,
1902
+ before: before ?? 20,
1903
+ after: after ?? 20
1892
1904
  });
1893
- return ok(`Found ${matchIndices.length} match(es) for "${query}" in ${artifact} (run ${resolvedRunId})`, {
1894
- run_id: resolvedRunId,
1895
- artifact,
1896
- query,
1897
- total_lines: lines.length,
1898
- match_count: matches.length,
1899
- truncated_at_limit: matchIndices.length >= limit,
1900
- matches
1905
+ return ok(`Loaded context for run ${resolvedRunId} scenario ${scenario_id}.`, {
1906
+ ...result
1901
1907
  });
1902
1908
  });
1903
1909
  });
@@ -1907,22 +1913,17 @@ export function registerPrompts(server) {
1907
1913
  description: 'Guide an LLM to author or refine MCPLab scenarios, prioritizing reusable scenario library files and deterministic eval rules.',
1908
1914
  argsSchema: {
1909
1915
  task: z.string().describe('What the scenario should test.'),
1910
- bundle_root: z
1911
- .string()
1912
- .optional()
1913
- .describe('Optional MCPLab library bundle root to inspect.'),
1914
1916
  server_ids: z
1915
1917
  .string()
1916
1918
  .optional()
1917
1919
  .describe('Comma-separated server ids to target if already known.'),
1918
1920
  agent_id: z.string().optional().describe('Optional pinned agent id.')
1919
1921
  }
1920
- }, async ({ task, bundle_root, server_ids, agent_id }) => {
1922
+ }, async ({ task, server_ids, agent_id }) => {
1921
1923
  const maybeServers = server_ids
1922
1924
  ? `Target servers (if valid): ${server_ids}\n`
1923
1925
  : 'First inspect available servers with mcplab_list_library.\n';
1924
1926
  const maybeAgent = agent_id ? `Pinned agent (optional): ${agent_id}\n` : '';
1925
- const maybeBundle = bundle_root ? `Bundle root hint: ${bundle_root}\n` : '';
1926
1927
  return {
1927
1928
  messages: [
1928
1929
  {
@@ -1930,7 +1931,7 @@ export function registerPrompts(server) {
1930
1931
  content: {
1931
1932
  type: 'text',
1932
1933
  text: `Help me author a MCPLab scenario for this testing task:\n\n${task}\n\n` +
1933
- `${maybeBundle}${maybeServers}${maybeAgent}` +
1934
+ `${maybeServers}${maybeAgent}` +
1934
1935
  `Workflow:\n` +
1935
1936
  `1. Inspect library entries (servers/agents/scenarios) if needed.\n` +
1936
1937
  `2. Draft a scenario with mcplab_generate_scenario_entry.\n` +
@@ -1980,10 +1981,8 @@ const PREFERRED_TOOL_TITLES = {
1980
1981
  mcplab_search_markdown_reports: 'Search Markdown Reports',
1981
1982
  mcplab_list_library: 'Search Library Entries',
1982
1983
  mcplab_generate_agent_entry: 'Generate MCPLab agents.yaml Entry',
1983
- mcplab_search_runs: 'Search Evaluation Runs',
1984
1984
  mcplab_search_tool_analysis_results: 'Search Tool Analysis Results',
1985
- mcplab_trace_search: 'Search Trace Events',
1986
- mcplab_grep_run_artifact: 'Search Run Artifact Text'
1985
+ mcplab_trace_search: 'Search Trace Events'
1987
1986
  };
1988
1987
  function normalizeOptionalNonEmpty(value) {
1989
1988
  if (typeof value !== 'string')
@@ -2037,17 +2036,31 @@ function humanizeToolName(toolName) {
2037
2036
  .map((word) => word.charAt(0).toUpperCase() + word.slice(1))
2038
2037
  .join(' ');
2039
2038
  }
2040
- function resolveBundleRoot(bundleRoot) {
2041
- if (bundleRoot?.trim())
2042
- return resolve(bundleRoot);
2039
+ function resolveServerOwnedRoots() {
2043
2040
  const cwd = process.cwd();
2041
+ const reportsDir = resolvePathInsideWorkspace(process.env.MCPLAB_REPORTS_DIR || 'mcplab/reports');
2042
+ const runsDir = resolvePathInsideWorkspace(process.env.MCPLAB_RUNS_DIR || 'mcplab/results/evaluation-runs');
2043
+ const toolAnalysisDir = resolvePathInsideWorkspace(process.env.MCPLAB_TOOL_ANALYSIS_DIR || 'mcplab/results/tool-analysis');
2044
+ const configuredBundleRoot = process.env.MCPLAB_BUNDLE_ROOT?.trim();
2045
+ if (configuredBundleRoot) {
2046
+ return {
2047
+ reportsDir,
2048
+ runsDir,
2049
+ toolAnalysisDir,
2050
+ bundleRoot: resolve(configuredBundleRoot)
2051
+ };
2052
+ }
2044
2053
  const candidates = ['mcplab', 'examples/libraries'];
2045
2054
  for (const candidate of candidates) {
2046
2055
  const abs = resolve(cwd, candidate);
2047
- if (existsSync(abs))
2048
- return abs;
2056
+ if (existsSync(abs)) {
2057
+ return { reportsDir, runsDir, toolAnalysisDir, bundleRoot: abs };
2058
+ }
2049
2059
  }
2050
- return resolve(cwd, 'mcplab');
2060
+ return { reportsDir, runsDir, toolAnalysisDir, bundleRoot: resolve(cwd, 'mcplab') };
2061
+ }
2062
+ function resolveBundleRoot() {
2063
+ return SERVER_OWNED_ROOTS.bundleRoot;
2051
2064
  }
2052
2065
  function readLibrary(bundleRoot, includeContent) {
2053
2066
  const serversPath = join(bundleRoot, 'servers.yaml');
@@ -2077,16 +2090,18 @@ function readLibrary(bundleRoot, includeContent) {
2077
2090
  }
2078
2091
  const out = {
2079
2092
  bundleRoot,
2080
- servers: includeContent
2081
- ? servers
2082
- : Object.keys(servers)
2083
- .sort()
2084
- .map((id) => ({ id })),
2085
- agents: includeContent
2086
- ? agents
2087
- : Object.keys(agents)
2088
- .sort()
2089
- .map((id) => ({ id })),
2093
+ servers: Object.keys(servers)
2094
+ .sort()
2095
+ .map((id) => ({
2096
+ id,
2097
+ ...(includeContent ? { entry: servers[id] ?? {} } : {})
2098
+ })),
2099
+ agents: Object.keys(agents)
2100
+ .sort()
2101
+ .map((id) => ({
2102
+ id,
2103
+ ...(includeContent ? { entry: agents[id] } : {})
2104
+ })),
2090
2105
  scenarios: scenarioEntries
2091
2106
  };
2092
2107
  return out;
@@ -2267,15 +2282,15 @@ function filterScenarios(scenarios, scenarioFilter, agentFilter) {
2267
2282
  });
2268
2283
  }
2269
2284
  function loadRunsForAnalysis(params) {
2270
- const base = resolveRunsDir(params.runsDirInput);
2285
+ const base = resolveRunsDir();
2271
2286
  const ids = selectRunIdsForAnalysis(base, params.runIds, params.latestN);
2272
2287
  return ids.map((id) => loadSingleRunForAnalysis(base, id));
2273
2288
  }
2274
2289
  function loadSingleRunForAnalysis(primaryRunsDir, runIdInput) {
2275
2290
  const resolvedRunId = resolveRunIdToken(primaryRunsDir, runIdInput);
2276
2291
  const readBase = resolveExistingRunReadDir(primaryRunsDir, resolvedRunId);
2277
- const runPath = join(readBase, resolvedRunId);
2278
- const resultsPath = join(runPath, 'results.json');
2292
+ const runPath = resolve(readBase, resolvedRunId);
2293
+ const resultsPath = resolveRunArtifactPath(readBase, resolvedRunId, 'results.json');
2279
2294
  if (!existsSync(resultsPath)) {
2280
2295
  throw new Error(`results.json not found for run '${resolvedRunId}' at ${resultsPath}`);
2281
2296
  }
@@ -2357,13 +2372,13 @@ function listRuns(runsDir, limit, includeSummary) {
2357
2372
  });
2358
2373
  }
2359
2374
  function defaultRunsDirPath() {
2360
- return resolvePathInsideWorkspace('mcplab/results/evaluation-runs');
2375
+ return SERVER_OWNED_ROOTS.runsDir;
2361
2376
  }
2362
2377
  function legacyRunsDirPath() {
2363
2378
  return resolvePathInsideWorkspace('mcplab/runs');
2364
2379
  }
2365
- function resolveRunsDir(input) {
2366
- return resolve(input?.trim() ? input : defaultRunsDirPath());
2380
+ function resolveRunsDir() {
2381
+ return SERVER_OWNED_ROOTS.runsDir;
2367
2382
  }
2368
2383
  function runReadDirs(primaryRunsDir) {
2369
2384
  const dirs = [primaryRunsDir];
@@ -2423,11 +2438,11 @@ function detectLikelyBundleRoot(configPath) {
2423
2438
  const fallback = resolveBundleRoot();
2424
2439
  return existsSync(fallback) ? fallback : null;
2425
2440
  }
2426
- function resolveToolAnalysisResultsDir(input) {
2427
- return resolvePathInsideWorkspace(input?.trim() ? input : 'mcplab/results/tool-analysis');
2441
+ function resolveToolAnalysisResultsDir() {
2442
+ return SERVER_OWNED_ROOTS.toolAnalysisDir;
2428
2443
  }
2429
- function resolveMarkdownReportsDir(input) {
2430
- return resolvePathInsideWorkspace(input?.trim() ? input : 'mcplab/reports');
2444
+ function resolveMarkdownReportsDir() {
2445
+ return SERVER_OWNED_ROOTS.reportsDir;
2431
2446
  }
2432
2447
  function isMarkdownReportExt(path) {
2433
2448
  const ext = extname(path).toLowerCase();
@@ -2484,11 +2499,11 @@ function resolveMarkdownReportPath(root, pathInput) {
2484
2499
  const trimmed = pathInput.trim();
2485
2500
  if (!trimmed)
2486
2501
  throw new Error('path is required');
2487
- const workspaceRelativePrefix = `mcplab${sep}reports${sep}`;
2488
2502
  const normalized = trimmed.replaceAll('/', sep);
2489
- const candidate = normalized === `mcplab${sep}reports` || normalized.startsWith(workspaceRelativePrefix)
2490
- ? resolvePathInsideWorkspace(normalized)
2491
- : resolve(root, normalized);
2503
+ if (normalized === `mcplab${sep}reports` || normalized.startsWith(`mcplab${sep}reports${sep}`)) {
2504
+ throw new Error('path must be relative to reports root; do not prefix with mcplab/reports/');
2505
+ }
2506
+ const candidate = resolve(root, normalized);
2492
2507
  const withinRoot = candidate === root || candidate.startsWith(`${root}${sep}`);
2493
2508
  if (!withinRoot)
2494
2509
  throw new Error('path escapes markdown reports root');
@@ -2663,13 +2678,13 @@ function isScenarioRunTraceRecord(value) {
2663
2678
  Array.isArray(v.messages) &&
2664
2679
  v.messages.every(isTraceMessage));
2665
2680
  }
2666
- function readScenarioRunTraceRecordsForRun(runsDirInput, runIdInput) {
2667
- const base = resolveRunsDir(runsDirInput);
2681
+ function readScenarioRunTraceRecordsForRun(runIdInput) {
2682
+ const base = resolveRunsDir();
2668
2683
  const readBase = resolveExistingRunReadDir(base, runIdInput === 'LATEST' ? undefined : runIdInput);
2669
2684
  const runId = runIdInput === 'LATEST' ? latestRunId(readBase) : runIdInput;
2670
2685
  if (!runId)
2671
2686
  throw new Error(`No runs found in ${base}`);
2672
- const tracePath = join(readBase, runId, 'trace.jsonl');
2687
+ const tracePath = resolveRunArtifactPath(readBase, runId, 'trace.jsonl');
2673
2688
  if (!existsSync(tracePath))
2674
2689
  throw new Error(`Artifact not found: ${tracePath}`);
2675
2690
  const raw = readFileSync(tracePath, 'utf8');
@@ -2842,6 +2857,14 @@ function resolvePathInsideWorkspace(pathInput) {
2842
2857
  }
2843
2858
  return target;
2844
2859
  }
2860
+ function validateWorkspaceRelativePath(value, fieldName) {
2861
+ if (value.startsWith('/') || /^[A-Za-z]:/.test(value)) {
2862
+ throw new Error(`${fieldName} must be relative (absolute paths are not allowed)`);
2863
+ }
2864
+ if (value.split(/[\\/]/).some((part) => part === '..')) {
2865
+ throw new Error(`${fieldName} must not contain ".." path segments`);
2866
+ }
2867
+ }
2845
2868
  function ok(summary, structuredContent) {
2846
2869
  const payload = structuredContent ?? {};
2847
2870
  return {
@@ -2858,8 +2881,7 @@ function err(error) {
2858
2881
  const message = error instanceof Error ? error.message : String(error);
2859
2882
  return {
2860
2883
  isError: true,
2861
- content: [{ type: 'text', text: `Error: ${message}` }],
2862
- structuredContent: { error: message }
2884
+ content: [{ type: 'text', text: `Error: ${message}` }]
2863
2885
  };
2864
2886
  }
2865
2887
  async function withToolHandling(fn) {