codex-plugin-doctor 1.50.0 → 1.52.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +17 -6
- package/dist/core/bounded-http-client.d.ts +23 -0
- package/dist/core/bounded-http-client.js +240 -0
- package/dist/core/mcp-conformance.d.ts +2 -0
- package/dist/core/mcp-conformance.js +215 -0
- package/dist/core/output-contract.js +104 -5
- package/dist/core/release-check.d.ts +2 -0
- package/dist/core/release-check.js +2 -0
- package/dist/core/release-evidence.d.ts +2 -0
- package/dist/core/release-evidence.js +111 -7
- package/dist/core/remote-mcp-probe.d.ts +16 -0
- package/dist/core/remote-mcp-probe.js +226 -0
- package/dist/core/remote-network-policy.d.ts +24 -0
- package/dist/core/remote-network-policy.js +125 -0
- package/dist/core/remote-oauth-readiness.d.ts +12 -0
- package/dist/core/remote-oauth-readiness.js +295 -0
- package/dist/core/remote-url-policy.d.ts +7 -0
- package/dist/core/remote-url-policy.js +49 -0
- package/dist/core/runtime-plan.d.ts +3 -0
- package/dist/core/runtime-plan.js +36 -2
- package/dist/core/runtime-policy.js +5 -1
- package/dist/core/runtime-probe.d.ts +11 -2
- package/dist/core/runtime-probe.js +375 -70
- package/dist/core/runtime-transcript.js +13 -3
- package/dist/core/validate-plugin.js +32 -1
- package/dist/domain/types.d.ts +47 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.js +4 -0
- package/dist/mcp/generic-mcp-doctor.d.ts +10 -2
- package/dist/mcp/generic-mcp-doctor.js +44 -13
- package/dist/reporting/render-markdown-report.js +44 -13
- package/dist/reporting/render-text-report.js +39 -22
- package/dist/rules/rule-catalog.js +253 -1
- package/dist/run-cli.js +137 -9
- package/dist/security/security-audit.js +34 -7
- package/examples/README.md +94 -94
- package/examples/codex-doctor-generic-mcp/.mcp.json +10 -10
- package/examples/codex-doctor-generic-mcp/server.js +1 -1
- package/examples/codex-doctor-risky/.codex-plugin/plugin.json +7 -7
- package/examples/codex-doctor-risky/.mcp.json +12 -12
- package/examples/codex-doctor-risky/mock-server.js +1 -1
- package/examples/codex-doctor-runtime/.codex-plugin/plugin.json +8 -8
- package/examples/codex-doctor-runtime/.mcp.json +9 -9
- package/examples/codex-doctor-runtime/mock-server.js +237 -237
- package/examples/codex-doctor-runtime/skills/context-check/SKILL.md +7 -7
- package/examples/codex-doctor-starter/.codex-plugin/plugin.json +7 -7
- package/examples/codex-doctor-starter/skills/repo-scan/SKILL.md +7 -7
- package/package.json +1 -1
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import { execFile, spawn } from "node:child_process";
|
|
2
2
|
import { randomUUID } from "node:crypto";
|
|
3
|
-
import { readFile, stat } from "node:fs/promises";
|
|
3
|
+
import { readFile, realpath, stat } from "node:fs/promises";
|
|
4
4
|
import path from "node:path";
|
|
5
5
|
import readline from "node:readline";
|
|
6
|
+
import { evaluateMcpConformance } from "./mcp-conformance.js";
|
|
7
|
+
import { probeRemoteMcpServer } from "./remote-mcp-probe.js";
|
|
6
8
|
import { buildRuntimeLaunch, DOCKER_RUNTIME_STARTUP_TIMEOUT_MS, RuntimeSandboxError } from "./runtime-sandbox.js";
|
|
7
9
|
import { formatRequestTranscript as formatRequestTranscriptForLog, formatResponseTranscript as formatResponseTranscriptForLog } from "./runtime-transcript.js";
|
|
8
10
|
const MCP_PROTOCOL_VERSION = "2025-11-25";
|
|
@@ -12,6 +14,7 @@ const MAX_TOOL_CALL_CONTENT_LENGTH = 4096;
|
|
|
12
14
|
const MAX_RESOURCE_READ_CONTENT_LENGTH = 4096;
|
|
13
15
|
const MAX_PROMPT_GET_CONTENT_LENGTH = 4096;
|
|
14
16
|
const DOCKER_CLEANUP_TIMEOUT_MS = 5_000;
|
|
17
|
+
const MAX_RUNTIME_PAGINATION_PAGES = 100;
|
|
15
18
|
function buildNativeRuntimeLaunch(input) {
|
|
16
19
|
return {
|
|
17
20
|
...input,
|
|
@@ -33,7 +36,16 @@ function createRuntimeScorecard() {
|
|
|
33
36
|
resourceRead: "unsupported",
|
|
34
37
|
resourceTemplatesList: "unsupported",
|
|
35
38
|
promptsList: "unsupported",
|
|
36
|
-
promptGet: "unsupported"
|
|
39
|
+
promptGet: "unsupported",
|
|
40
|
+
conformance: {
|
|
41
|
+
protocolVersion: null,
|
|
42
|
+
profile: null,
|
|
43
|
+
capabilityConsistency: "skipped",
|
|
44
|
+
taskDeclarations: "skipped",
|
|
45
|
+
tasksList: "skipped",
|
|
46
|
+
schemaDialect: "skipped",
|
|
47
|
+
overall: "skipped"
|
|
48
|
+
}
|
|
37
49
|
};
|
|
38
50
|
}
|
|
39
51
|
function buildFailure(id, message, impact, suggestedFix, evidence) {
|
|
@@ -57,6 +69,16 @@ function buildWarning(id, message, impact, suggestedFix, evidence) {
|
|
|
57
69
|
};
|
|
58
70
|
}
|
|
59
71
|
function methodForRuntimeFinding(id) {
|
|
72
|
+
if (id.startsWith("mcp.conformance.protocol.") ||
|
|
73
|
+
id.startsWith("mcp.conformance.tasks.capability_")) {
|
|
74
|
+
return "initialize";
|
|
75
|
+
}
|
|
76
|
+
if (id.startsWith("mcp.conformance.tasks.task_support_") ||
|
|
77
|
+
id.startsWith("mcp.conformance.schema.")) {
|
|
78
|
+
return "tools/list";
|
|
79
|
+
}
|
|
80
|
+
if (id.startsWith("mcp.conformance.tasks_list."))
|
|
81
|
+
return "tasks/list";
|
|
60
82
|
if (id.includes(".initialize."))
|
|
61
83
|
return "initialize";
|
|
62
84
|
if (id.includes(".tools_list."))
|
|
@@ -81,9 +103,9 @@ function withRuntimeEvidence(finding, serverName) {
|
|
|
81
103
|
return {
|
|
82
104
|
...finding,
|
|
83
105
|
evidence: {
|
|
106
|
+
...(finding.evidence ?? {}),
|
|
84
107
|
serverName,
|
|
85
|
-
method: methodForRuntimeFinding(finding.id)
|
|
86
|
-
...(finding.evidence ?? {})
|
|
108
|
+
method: methodForRuntimeFinding(finding.id)
|
|
87
109
|
}
|
|
88
110
|
};
|
|
89
111
|
}
|
|
@@ -228,15 +250,7 @@ function isPathWithinRoot(rootPath, candidatePath) {
|
|
|
228
250
|
return (relativePath === "" ||
|
|
229
251
|
(!relativePath.startsWith("..") && !path.isAbsolute(relativePath)));
|
|
230
252
|
}
|
|
231
|
-
async function loadMcpServers(
|
|
232
|
-
const { manifest, rootPath } = discoveredPackage;
|
|
233
|
-
if (!manifest.mcpServers) {
|
|
234
|
-
return null;
|
|
235
|
-
}
|
|
236
|
-
const mcpConfigPath = path.resolve(rootPath, manifest.mcpServers);
|
|
237
|
-
if (!isPathWithinRoot(rootPath, mcpConfigPath)) {
|
|
238
|
-
return null;
|
|
239
|
-
}
|
|
253
|
+
async function loadMcpServers(mcpConfigPath) {
|
|
240
254
|
const exists = await fileExists(mcpConfigPath);
|
|
241
255
|
if (!exists) {
|
|
242
256
|
return null;
|
|
@@ -276,6 +290,80 @@ function hasPromptsCapability(message) {
|
|
|
276
290
|
const capabilities = getCapabilities(message);
|
|
277
291
|
return capabilities !== null && isPlainObject(capabilities.prompts);
|
|
278
292
|
}
|
|
293
|
+
function worstRuntimeStatus(left, right) {
|
|
294
|
+
const severity = {
|
|
295
|
+
fail: 4,
|
|
296
|
+
warn: 3,
|
|
297
|
+
pass: 2,
|
|
298
|
+
skipped: 1,
|
|
299
|
+
unsupported: 0
|
|
300
|
+
};
|
|
301
|
+
return severity[left] >= severity[right] ? left : right;
|
|
302
|
+
}
|
|
303
|
+
function mergeRuntimeScorecards(left, right) {
|
|
304
|
+
const leftConformance = left.conformance;
|
|
305
|
+
const rightConformance = right.conformance;
|
|
306
|
+
const conformance = leftConformance && rightConformance
|
|
307
|
+
? {
|
|
308
|
+
protocolVersion: leftConformance.protocolVersion === null
|
|
309
|
+
? rightConformance.protocolVersion
|
|
310
|
+
: rightConformance.protocolVersion === null ||
|
|
311
|
+
leftConformance.protocolVersion === rightConformance.protocolVersion
|
|
312
|
+
? leftConformance.protocolVersion
|
|
313
|
+
: null,
|
|
314
|
+
profile: leftConformance.profile === null
|
|
315
|
+
? rightConformance.profile
|
|
316
|
+
: rightConformance.profile === null || leftConformance.profile === rightConformance.profile
|
|
317
|
+
? leftConformance.profile
|
|
318
|
+
: null,
|
|
319
|
+
capabilityConsistency: worstRuntimeStatus(leftConformance.capabilityConsistency, rightConformance.capabilityConsistency),
|
|
320
|
+
taskDeclarations: worstRuntimeStatus(leftConformance.taskDeclarations, rightConformance.taskDeclarations),
|
|
321
|
+
tasksList: worstRuntimeStatus(leftConformance.tasksList, rightConformance.tasksList),
|
|
322
|
+
schemaDialect: worstRuntimeStatus(leftConformance.schemaDialect, rightConformance.schemaDialect),
|
|
323
|
+
overall: worstRuntimeStatus(leftConformance.overall, rightConformance.overall)
|
|
324
|
+
}
|
|
325
|
+
: leftConformance ?? rightConformance;
|
|
326
|
+
const remote = left.remote && right.remote
|
|
327
|
+
? mergeRemoteRuntimeScorecards(left.remote, right.remote)
|
|
328
|
+
: left.remote ?? right.remote;
|
|
329
|
+
return {
|
|
330
|
+
initialize: worstRuntimeStatus(left.initialize, right.initialize),
|
|
331
|
+
toolsList: worstRuntimeStatus(left.toolsList, right.toolsList),
|
|
332
|
+
toolsCall: worstRuntimeStatus(left.toolsCall, right.toolsCall),
|
|
333
|
+
resourcesList: worstRuntimeStatus(left.resourcesList, right.resourcesList),
|
|
334
|
+
resourceRead: worstRuntimeStatus(left.resourceRead, right.resourceRead),
|
|
335
|
+
resourceTemplatesList: worstRuntimeStatus(left.resourceTemplatesList, right.resourceTemplatesList),
|
|
336
|
+
promptsList: worstRuntimeStatus(left.promptsList, right.promptsList),
|
|
337
|
+
promptGet: worstRuntimeStatus(left.promptGet, right.promptGet),
|
|
338
|
+
...(conformance ? { conformance } : {}),
|
|
339
|
+
...(remote ? { remote } : {})
|
|
340
|
+
};
|
|
341
|
+
}
|
|
342
|
+
function mergeRemoteRuntimeScorecards(left, right) {
|
|
343
|
+
const sessionSeverity = {
|
|
344
|
+
"present-invalid": 2,
|
|
345
|
+
"present-valid": 1,
|
|
346
|
+
absent: 0
|
|
347
|
+
};
|
|
348
|
+
return {
|
|
349
|
+
transport: worstRuntimeStatus(left.transport, right.transport),
|
|
350
|
+
networkSafety: worstRuntimeStatus(left.networkSafety, right.networkSafety),
|
|
351
|
+
initialize: worstRuntimeStatus(left.initialize, right.initialize),
|
|
352
|
+
contentType: worstRuntimeStatus(left.contentType, right.contentType),
|
|
353
|
+
session: sessionSeverity[left.session] >= sessionSeverity[right.session]
|
|
354
|
+
? left.session
|
|
355
|
+
: right.session,
|
|
356
|
+
protocolHeaders: worstRuntimeStatus(left.protocolHeaders, right.protocolHeaders),
|
|
357
|
+
authorization: worstRuntimeStatus(left.authorization, right.authorization),
|
|
358
|
+
overall: worstRuntimeStatus(left.overall, right.overall)
|
|
359
|
+
};
|
|
360
|
+
}
|
|
361
|
+
function supportsTasksListProbe(protocolVersion, capabilities) {
|
|
362
|
+
if (protocolVersion < MCP_PROTOCOL_VERSION || !isPlainObject(capabilities.tasks)) {
|
|
363
|
+
return false;
|
|
364
|
+
}
|
|
365
|
+
return isPlainObject(capabilities.tasks.list);
|
|
366
|
+
}
|
|
279
367
|
function getNextCursor(result) {
|
|
280
368
|
if (result.nextCursor === undefined) {
|
|
281
369
|
return null;
|
|
@@ -294,13 +382,14 @@ function extractToolsPage(message) {
|
|
|
294
382
|
for (const tool of tools) {
|
|
295
383
|
if (!isPlainObject(tool) ||
|
|
296
384
|
typeof tool.name !== "string" ||
|
|
297
|
-
!
|
|
298
|
-
tool.inputSchema.type !== "object") {
|
|
385
|
+
!("inputSchema" in tool)) {
|
|
299
386
|
return null;
|
|
300
387
|
}
|
|
301
388
|
parsedTools.push({
|
|
302
389
|
name: tool.name,
|
|
303
|
-
inputSchema: tool.inputSchema
|
|
390
|
+
inputSchema: tool.inputSchema,
|
|
391
|
+
...(tool.outputSchema === undefined ? {} : { outputSchema: tool.outputSchema }),
|
|
392
|
+
...(tool.execution === undefined ? {} : { execution: tool.execution })
|
|
304
393
|
});
|
|
305
394
|
}
|
|
306
395
|
const nextCursor = getNextCursor(message.result);
|
|
@@ -472,16 +561,55 @@ function findCallableTool(tools) {
|
|
|
472
561
|
if (isDestructiveTool(tool)) {
|
|
473
562
|
continue;
|
|
474
563
|
}
|
|
475
|
-
|
|
564
|
+
if (!isPlainObject(tool.inputSchema) || tool.inputSchema.type !== "object") {
|
|
565
|
+
continue;
|
|
566
|
+
}
|
|
567
|
+
const callableTool = {
|
|
568
|
+
name: tool.name,
|
|
569
|
+
inputSchema: tool.inputSchema
|
|
570
|
+
};
|
|
571
|
+
const args = buildToolArguments(callableTool);
|
|
476
572
|
if (args !== undefined) {
|
|
477
573
|
return {
|
|
478
|
-
tool,
|
|
574
|
+
tool: callableTool,
|
|
479
575
|
args
|
|
480
576
|
};
|
|
481
577
|
}
|
|
482
578
|
}
|
|
483
579
|
return null;
|
|
484
580
|
}
|
|
581
|
+
function hasValidToolInputSchemas(tools) {
|
|
582
|
+
return tools.every((tool) => isPlainObject(tool.inputSchema) && tool.inputSchema.type === "object");
|
|
583
|
+
}
|
|
584
|
+
async function resolveRuntimeCwd(rootPath, configuredCwd) {
|
|
585
|
+
const resolvedCwd = typeof configuredCwd === "string"
|
|
586
|
+
? path.resolve(rootPath, configuredCwd)
|
|
587
|
+
: rootPath;
|
|
588
|
+
try {
|
|
589
|
+
const [canonicalCwd, details] = await Promise.all([
|
|
590
|
+
realpath(resolvedCwd),
|
|
591
|
+
stat(resolvedCwd)
|
|
592
|
+
]);
|
|
593
|
+
return details.isDirectory() && isPathWithinRoot(rootPath, canonicalCwd)
|
|
594
|
+
? canonicalCwd
|
|
595
|
+
: null;
|
|
596
|
+
}
|
|
597
|
+
catch {
|
|
598
|
+
return null;
|
|
599
|
+
}
|
|
600
|
+
}
|
|
601
|
+
function isProtocolVersion(value) {
|
|
602
|
+
const match = /^(\d{4})-(\d{2})-(\d{2})$/.exec(value);
|
|
603
|
+
if (!match) {
|
|
604
|
+
return false;
|
|
605
|
+
}
|
|
606
|
+
const [year, month, day] = match.slice(1).map(Number);
|
|
607
|
+
const date = new Date(0);
|
|
608
|
+
date.setUTCFullYear(year, month - 1, day);
|
|
609
|
+
return (date.getUTCFullYear() === year &&
|
|
610
|
+
date.getUTCMonth() === month - 1 &&
|
|
611
|
+
date.getUTCDate() === day);
|
|
612
|
+
}
|
|
485
613
|
function buildPromptArguments(prompt) {
|
|
486
614
|
if (!prompt.arguments || prompt.arguments.length === 0) {
|
|
487
615
|
return undefined;
|
|
@@ -744,7 +872,12 @@ async function probeCommandServer(input) {
|
|
|
744
872
|
const fetchPaginated = async (input) => {
|
|
745
873
|
let cursor = null;
|
|
746
874
|
const items = [];
|
|
875
|
+
const seenCursors = new Set();
|
|
876
|
+
let pageCount = 0;
|
|
747
877
|
do {
|
|
878
|
+
if (pageCount >= MAX_RUNTIME_PAGINATION_PAGES) {
|
|
879
|
+
return null;
|
|
880
|
+
}
|
|
748
881
|
const response = await sendRequest(input.method, cursor ? { cursor } : undefined, input.timeoutFinding);
|
|
749
882
|
transcript?.(formatResponseTranscriptForLog(input.method, response));
|
|
750
883
|
if (isErrorResponse(response)) {
|
|
@@ -759,10 +892,81 @@ async function probeCommandServer(input) {
|
|
|
759
892
|
return null;
|
|
760
893
|
}
|
|
761
894
|
items.push(...page.items);
|
|
895
|
+
pageCount += 1;
|
|
896
|
+
if (page.nextCursor && seenCursors.has(page.nextCursor)) {
|
|
897
|
+
return null;
|
|
898
|
+
}
|
|
899
|
+
if (page.nextCursor) {
|
|
900
|
+
seenCursors.add(page.nextCursor);
|
|
901
|
+
}
|
|
762
902
|
cursor = page.nextCursor;
|
|
763
903
|
} while (cursor);
|
|
764
904
|
return items;
|
|
765
905
|
};
|
|
906
|
+
const observeTasksList = async () => {
|
|
907
|
+
let cursor = null;
|
|
908
|
+
const seenCursors = new Set();
|
|
909
|
+
let itemCount = 0;
|
|
910
|
+
let pageCount = 0;
|
|
911
|
+
try {
|
|
912
|
+
do {
|
|
913
|
+
if (pageCount >= MAX_RUNTIME_PAGINATION_PAGES) {
|
|
914
|
+
return {
|
|
915
|
+
status: "fail",
|
|
916
|
+
itemCount,
|
|
917
|
+
pageCount,
|
|
918
|
+
failure: "invalid"
|
|
919
|
+
};
|
|
920
|
+
}
|
|
921
|
+
const response = await sendRequest("tasks/list", cursor ? { cursor } : undefined, buildFailure("mcp.conformance.tasks_list.timeout", `The MCP server \`${serverName}\` did not answer the tasks/list request in time.`, "Codex cannot safely inspect task discovery when the server does not complete tasks/list.", "Reduce tasks/list latency and verify pagination completes."));
|
|
922
|
+
if (isErrorResponse(response) || !isPlainObject(response.result)) {
|
|
923
|
+
return {
|
|
924
|
+
status: "fail",
|
|
925
|
+
itemCount,
|
|
926
|
+
pageCount,
|
|
927
|
+
failure: "invalid"
|
|
928
|
+
};
|
|
929
|
+
}
|
|
930
|
+
const tasks = response.result.tasks;
|
|
931
|
+
const nextCursor = getNextCursor(response.result);
|
|
932
|
+
if (!Array.isArray(tasks) ||
|
|
933
|
+
!tasks.every(isPlainObject) ||
|
|
934
|
+
nextCursor === "__invalid__") {
|
|
935
|
+
return {
|
|
936
|
+
status: "fail",
|
|
937
|
+
itemCount,
|
|
938
|
+
pageCount,
|
|
939
|
+
failure: "invalid"
|
|
940
|
+
};
|
|
941
|
+
}
|
|
942
|
+
itemCount += tasks.length;
|
|
943
|
+
pageCount += 1;
|
|
944
|
+
if (nextCursor && seenCursors.has(nextCursor)) {
|
|
945
|
+
return {
|
|
946
|
+
status: "fail",
|
|
947
|
+
itemCount,
|
|
948
|
+
pageCount,
|
|
949
|
+
failure: "invalid"
|
|
950
|
+
};
|
|
951
|
+
}
|
|
952
|
+
if (nextCursor) {
|
|
953
|
+
seenCursors.add(nextCursor);
|
|
954
|
+
}
|
|
955
|
+
cursor = nextCursor;
|
|
956
|
+
} while (cursor);
|
|
957
|
+
}
|
|
958
|
+
catch (error) {
|
|
959
|
+
return {
|
|
960
|
+
status: "fail",
|
|
961
|
+
itemCount,
|
|
962
|
+
pageCount,
|
|
963
|
+
failure: isFinding(error) && error.id === "mcp.conformance.tasks_list.timeout"
|
|
964
|
+
? "timeout"
|
|
965
|
+
: "invalid"
|
|
966
|
+
};
|
|
967
|
+
}
|
|
968
|
+
return { status: "pass", itemCount, pageCount };
|
|
969
|
+
};
|
|
766
970
|
child.stderr?.on("data", (chunk) => {
|
|
767
971
|
if (stderrPreview.length >= 160) {
|
|
768
972
|
return;
|
|
@@ -835,6 +1039,7 @@ async function probeCommandServer(input) {
|
|
|
835
1039
|
}
|
|
836
1040
|
const result = initializeResponse.result;
|
|
837
1041
|
if (typeof result.protocolVersion !== "string" ||
|
|
1042
|
+
!isProtocolVersion(result.protocolVersion) ||
|
|
838
1043
|
!isPlainObject(result.capabilities) ||
|
|
839
1044
|
!isPlainObject(result.serverInfo) ||
|
|
840
1045
|
typeof result.serverInfo.name !== "string" ||
|
|
@@ -845,43 +1050,79 @@ async function probeCommandServer(input) {
|
|
|
845
1050
|
}
|
|
846
1051
|
scorecard.initialize = "pass";
|
|
847
1052
|
sendNotification("notifications/initialized");
|
|
1053
|
+
let tools = [];
|
|
1054
|
+
let toolTerminalFinding = null;
|
|
1055
|
+
let canEvaluateConformance = true;
|
|
848
1056
|
if (!hasToolsCapability(initializeResponse)) {
|
|
849
1057
|
scorecard.toolsList = "unsupported";
|
|
850
1058
|
scorecard.toolsCall = "unsupported";
|
|
851
|
-
|
|
852
|
-
return;
|
|
853
|
-
}
|
|
854
|
-
const tools = await fetchPaginated({
|
|
855
|
-
method: "tools/list",
|
|
856
|
-
timeoutFinding: buildFailure("plugin.runtime.tools_list.timeout", `The MCP server \`${serverName}\` did not answer the tools/list request in time.`, "A server that cannot return its tool catalog in time will feel broken or invisible in Codex.", "Inspect the tool discovery path and reduce latency before returning the tool list."),
|
|
857
|
-
extractPage: extractToolsPage
|
|
858
|
-
});
|
|
859
|
-
if (!tools) {
|
|
860
|
-
scorecard.toolsList = "fail";
|
|
861
|
-
settle(buildFailure("plugin.runtime.tools_list.invalid", `The MCP server \`${serverName}\` returned an invalid tools/list result.`, "Codex cannot safely consume malformed tool definitions from `tools/list`.", "Return a `tools` array where every tool has a string `name` and an object-shaped `inputSchema` with `type: \"object\"`."));
|
|
862
|
-
return;
|
|
863
|
-
}
|
|
864
|
-
scorecard.toolsList = "pass";
|
|
865
|
-
const callableTool = findCallableTool(tools);
|
|
866
|
-
if (!callableTool) {
|
|
867
|
-
scorecard.toolsCall = "skipped";
|
|
868
|
-
settle(buildWarning("plugin.runtime.tool_call.skipped", `The MCP server \`${serverName}\` does not expose a safely callable tool for probing.`, "The validator confirmed tool discovery but could not safely perform a non-destructive `tools/call` probe.", "Expose at least one non-destructive tool with a JSON schema the validator can generate arguments for."));
|
|
869
|
-
return;
|
|
1059
|
+
warnings.push(buildWarning("plugin.runtime.tools.unsupported", `The MCP server \`${serverName}\` does not advertise tools capability.`, "Codex cannot use `tools/list` for deeper validation if the server does not declare tools support.", "Expose `capabilities.tools` during initialize if this server is expected to provide tools."));
|
|
870
1060
|
}
|
|
871
1061
|
else {
|
|
872
|
-
const
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
1062
|
+
const listedTools = await fetchPaginated({
|
|
1063
|
+
method: "tools/list",
|
|
1064
|
+
timeoutFinding: buildFailure("plugin.runtime.tools_list.timeout", `The MCP server \`${serverName}\` did not answer the tools/list request in time.`, "A server that cannot return its tool catalog in time will feel broken or invisible in Codex.", "Inspect the tool discovery path and reduce latency before returning the tool list."),
|
|
1065
|
+
extractPage: extractToolsPage
|
|
1066
|
+
});
|
|
1067
|
+
if (!listedTools) {
|
|
1068
|
+
canEvaluateConformance = false;
|
|
1069
|
+
scorecard.toolsList = "fail";
|
|
1070
|
+
toolTerminalFinding = buildFailure("plugin.runtime.tools_list.invalid", `The MCP server \`${serverName}\` returned an invalid tools/list result.`, "Codex cannot safely consume malformed tool definitions from `tools/list`.", "Return a `tools` array where every tool has a string `name` and an object-shaped `inputSchema` with `type: \"object\"`.");
|
|
1071
|
+
}
|
|
1072
|
+
else {
|
|
1073
|
+
tools = listedTools;
|
|
1074
|
+
if (!hasValidToolInputSchemas(tools)) {
|
|
1075
|
+
canEvaluateConformance = false;
|
|
1076
|
+
scorecard.toolsList = "fail";
|
|
1077
|
+
toolTerminalFinding = buildFailure("plugin.runtime.tools_list.invalid", `The MCP server \`${serverName}\` returned an invalid tools/list result.`, "Codex cannot safely consume malformed tool definitions from `tools/list`.", "Return a `tools` array where every tool has a string `name` and an object-shaped `inputSchema` with `type: \"object\"`.");
|
|
1078
|
+
}
|
|
1079
|
+
else {
|
|
1080
|
+
scorecard.toolsList = "pass";
|
|
1081
|
+
}
|
|
1082
|
+
}
|
|
1083
|
+
const callableTool = toolTerminalFinding ? null : findCallableTool(tools);
|
|
1084
|
+
if (!toolTerminalFinding && !callableTool) {
|
|
1085
|
+
scorecard.toolsCall = "skipped";
|
|
1086
|
+
toolTerminalFinding = buildWarning("plugin.runtime.tool_call.skipped", `The MCP server \`${serverName}\` does not expose a safely callable tool for probing.`, "The validator confirmed tool discovery but could not safely perform a non-destructive `tools/call` probe.", "Expose at least one non-destructive tool with a JSON schema the validator can generate arguments for.");
|
|
880
1087
|
}
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
1088
|
+
else if (callableTool) {
|
|
1089
|
+
const toolCallResponse = await sendRequest("tools/call", {
|
|
1090
|
+
name: callableTool.tool.name,
|
|
1091
|
+
arguments: callableTool.args
|
|
1092
|
+
}, buildFailure("plugin.runtime.tool_call.timeout", `The MCP server \`${serverName}\` did not answer the tools/call request in time.`, "A server that cannot complete a basic tool invocation in time will feel broken in real Codex usage.", "Inspect the selected tool implementation and reduce its response latency.", { toolName: callableTool.tool.name }));
|
|
1093
|
+
if (isErrorResponse(toolCallResponse) || !isValidCallToolResult(toolCallResponse)) {
|
|
1094
|
+
scorecard.toolsCall = "fail";
|
|
1095
|
+
toolTerminalFinding = buildFailure("plugin.runtime.tool_call.invalid", `The MCP server \`${serverName}\` returned an invalid tools/call result.`, "Codex cannot safely consume malformed tool call results from the server.", "Return a CallToolResult with a `content` array containing valid MCP content blocks.", { toolName: callableTool.tool.name });
|
|
1096
|
+
}
|
|
1097
|
+
if (!toolTerminalFinding) {
|
|
1098
|
+
scorecard.toolsCall = "pass";
|
|
1099
|
+
warnings.push(...collectOversizedToolCallWarnings(toolCallResponse, {
|
|
1100
|
+
toolName: callableTool.tool.name
|
|
1101
|
+
}));
|
|
1102
|
+
}
|
|
1103
|
+
}
|
|
1104
|
+
}
|
|
1105
|
+
const tasksList = supportsTasksListProbe(result.protocolVersion, result.capabilities)
|
|
1106
|
+
? await observeTasksList()
|
|
1107
|
+
: {
|
|
1108
|
+
status: "skipped",
|
|
1109
|
+
itemCount: 0,
|
|
1110
|
+
pageCount: 0
|
|
1111
|
+
};
|
|
1112
|
+
if (canEvaluateConformance || tasksList.status !== "skipped") {
|
|
1113
|
+
const conformance = evaluateMcpConformance({
|
|
1114
|
+
protocolVersion: result.protocolVersion,
|
|
1115
|
+
capabilities: result.capabilities,
|
|
1116
|
+
tools: tools,
|
|
1117
|
+
tasksList,
|
|
1118
|
+
skipToolDerivedChecks: !canEvaluateConformance
|
|
1119
|
+
});
|
|
1120
|
+
scorecard.conformance = conformance.scorecard;
|
|
1121
|
+
warnings.push(...conformance.findings);
|
|
1122
|
+
}
|
|
1123
|
+
if (toolTerminalFinding) {
|
|
1124
|
+
settle(toolTerminalFinding);
|
|
1125
|
+
return;
|
|
885
1126
|
}
|
|
886
1127
|
if (!hasResourcesCapability(initializeResponse)) {
|
|
887
1128
|
scorecard.resourcesList = "unsupported";
|
|
@@ -1001,8 +1242,40 @@ async function probeCommandServer(input) {
|
|
|
1001
1242
|
});
|
|
1002
1243
|
}
|
|
1003
1244
|
export async function probeRuntime(discoveredPackage, options = {}) {
|
|
1245
|
+
const { manifest, rootPath } = discoveredPackage;
|
|
1246
|
+
if (!manifest.mcpServers) {
|
|
1247
|
+
return {
|
|
1248
|
+
findings: [],
|
|
1249
|
+
scorecard: createRuntimeScorecard()
|
|
1250
|
+
};
|
|
1251
|
+
}
|
|
1252
|
+
return probeRuntimeConfig(rootPath, manifest.mcpServers, options);
|
|
1253
|
+
}
|
|
1254
|
+
export async function probeRuntimeConfig(rootPath, mcpConfigPath, options = {}) {
|
|
1255
|
+
const resolvedRootPath = path.resolve(rootPath);
|
|
1256
|
+
const resolvedMcpConfigPath = path.resolve(resolvedRootPath, mcpConfigPath);
|
|
1004
1257
|
const startupTimeoutMs = options.startupTimeoutMs ?? 400;
|
|
1005
|
-
|
|
1258
|
+
let canonicalRootPath;
|
|
1259
|
+
let canonicalMcpConfigPath;
|
|
1260
|
+
try {
|
|
1261
|
+
[canonicalRootPath, canonicalMcpConfigPath] = await Promise.all([
|
|
1262
|
+
realpath(resolvedRootPath),
|
|
1263
|
+
realpath(resolvedMcpConfigPath)
|
|
1264
|
+
]);
|
|
1265
|
+
}
|
|
1266
|
+
catch {
|
|
1267
|
+
return {
|
|
1268
|
+
findings: [],
|
|
1269
|
+
scorecard: createRuntimeScorecard()
|
|
1270
|
+
};
|
|
1271
|
+
}
|
|
1272
|
+
if (!isPathWithinRoot(canonicalRootPath, canonicalMcpConfigPath)) {
|
|
1273
|
+
return {
|
|
1274
|
+
findings: [],
|
|
1275
|
+
scorecard: createRuntimeScorecard()
|
|
1276
|
+
};
|
|
1277
|
+
}
|
|
1278
|
+
const servers = await loadMcpServers(canonicalMcpConfigPath);
|
|
1006
1279
|
if (!servers) {
|
|
1007
1280
|
return {
|
|
1008
1281
|
findings: [],
|
|
@@ -1012,31 +1285,63 @@ export async function probeRuntime(discoveredPackage, options = {}) {
|
|
|
1012
1285
|
const findings = [];
|
|
1013
1286
|
let scorecard = createRuntimeScorecard();
|
|
1014
1287
|
let execution;
|
|
1288
|
+
let hasProbedServer = false;
|
|
1015
1289
|
for (const [serverName, config] of Object.entries(servers)) {
|
|
1016
1290
|
if (!isPlainObject(config)) {
|
|
1017
1291
|
continue;
|
|
1018
1292
|
}
|
|
1019
|
-
const
|
|
1020
|
-
|
|
1021
|
-
|
|
1293
|
+
const url = config.url;
|
|
1294
|
+
let result;
|
|
1295
|
+
if (typeof url === "string") {
|
|
1296
|
+
const remote = await probeRemoteMcpServer(serverName, url, {
|
|
1297
|
+
allowNetwork: options.allowNetwork,
|
|
1298
|
+
allowLocalNetwork: options.allowLocalNetwork,
|
|
1299
|
+
requestTimeoutMs: options.remoteRequestTimeoutMs,
|
|
1300
|
+
lookup: options.remoteLookup,
|
|
1301
|
+
request: options.remoteRequest
|
|
1302
|
+
});
|
|
1303
|
+
const remoteScorecard = createRuntimeScorecard();
|
|
1304
|
+
remoteScorecard.remote = remote.scorecard;
|
|
1305
|
+
result = {
|
|
1306
|
+
findings: remote.findings.map((finding) => withRuntimeEvidence(finding, serverName)),
|
|
1307
|
+
scorecard: remoteScorecard
|
|
1308
|
+
};
|
|
1022
1309
|
}
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1310
|
+
else {
|
|
1311
|
+
const command = config.command;
|
|
1312
|
+
if (typeof command !== "string") {
|
|
1313
|
+
continue;
|
|
1314
|
+
}
|
|
1315
|
+
const args = Array.isArray(config.args)
|
|
1316
|
+
? config.args.filter((value) => typeof value === "string")
|
|
1317
|
+
: [];
|
|
1318
|
+
const cwd = await resolveRuntimeCwd(canonicalRootPath, config.cwd);
|
|
1319
|
+
result = cwd === null
|
|
1320
|
+
? (() => {
|
|
1321
|
+
const invalidCwdScorecard = createRuntimeScorecard();
|
|
1322
|
+
invalidCwdScorecard.initialize = "fail";
|
|
1323
|
+
return {
|
|
1324
|
+
findings: [
|
|
1325
|
+
withRuntimeEvidence(buildFailure("plugin.runtime.startup.invalid_cwd", `The MCP server \`${serverName}\` has an invalid runtime working directory.`, "Runtime validation must not start a server from a missing, non-directory, or out-of-package working directory.", "Set the MCP server cwd to an existing directory inside the plugin package root, or remove it."), serverName)
|
|
1326
|
+
],
|
|
1327
|
+
scorecard: invalidCwdScorecard
|
|
1328
|
+
};
|
|
1329
|
+
})()
|
|
1330
|
+
: await probeCommandServer({
|
|
1331
|
+
serverName,
|
|
1332
|
+
packageRoot: canonicalRootPath,
|
|
1333
|
+
command,
|
|
1334
|
+
args,
|
|
1335
|
+
cwd,
|
|
1336
|
+
startupTimeoutMs,
|
|
1337
|
+
sandbox: options.sandbox,
|
|
1338
|
+
transcript: options.transcript
|
|
1339
|
+
});
|
|
1340
|
+
}
|
|
1341
|
+
scorecard = hasProbedServer
|
|
1342
|
+
? mergeRuntimeScorecards(scorecard, result.scorecard)
|
|
1343
|
+
: result.scorecard;
|
|
1344
|
+
hasProbedServer = true;
|
|
1040
1345
|
execution = result.execution ?? execution;
|
|
1041
1346
|
if (result.findings.length > 0) {
|
|
1042
1347
|
findings.push(...result.findings);
|
|
@@ -73,15 +73,20 @@ export function formatRequestTranscript(method, params) {
|
|
|
73
73
|
if (!params) {
|
|
74
74
|
return `-> ${method}`;
|
|
75
75
|
}
|
|
76
|
+
if (method === "tasks/list" && params.cursor !== undefined) {
|
|
77
|
+
return '-> tasks/list {"cursor":"[CURSOR]"}';
|
|
78
|
+
}
|
|
76
79
|
return `-> ${method} ${JSON.stringify(sanitizeTranscriptValue(params))}`;
|
|
77
80
|
}
|
|
78
81
|
export function formatResponseTranscript(method, message) {
|
|
79
82
|
const error = getErrorObject(message);
|
|
80
83
|
if (error) {
|
|
81
84
|
const code = typeof error.code === "number" ? error.code : "?";
|
|
82
|
-
const messageText =
|
|
83
|
-
?
|
|
84
|
-
:
|
|
85
|
+
const messageText = method === "tasks/list"
|
|
86
|
+
? "[REDACTED]"
|
|
87
|
+
: typeof error.message === "string"
|
|
88
|
+
? sanitizeTranscriptValue(error.message, ["error", "message"])
|
|
89
|
+
: "error";
|
|
85
90
|
return `<- ${method} error ${JSON.stringify({
|
|
86
91
|
code,
|
|
87
92
|
message: messageText
|
|
@@ -129,6 +134,11 @@ export function formatResponseTranscript(method, message) {
|
|
|
129
134
|
prompts: Array.isArray(result.prompts) ? result.prompts.length : 0,
|
|
130
135
|
nextCursor: typeof result.nextCursor === "string" ? "[CURSOR]" : undefined
|
|
131
136
|
})}`;
|
|
137
|
+
case "tasks/list":
|
|
138
|
+
return `<- tasks/list ${JSON.stringify({
|
|
139
|
+
tasks: Array.isArray(result.tasks) ? result.tasks.length : 0,
|
|
140
|
+
nextCursor: typeof result.nextCursor === "string" ? "[CURSOR]" : undefined
|
|
141
|
+
})}`;
|
|
132
142
|
case "prompts/get":
|
|
133
143
|
return `<- prompts/get ${JSON.stringify({
|
|
134
144
|
messages: Array.isArray(result.messages) ? result.messages.length : 0
|
|
@@ -2,6 +2,7 @@ import { readFile, readdir, stat } from "node:fs/promises";
|
|
|
2
2
|
import path from "node:path";
|
|
3
3
|
import { withFindingFingerprints } from "../reporting/finding-fingerprint.js";
|
|
4
4
|
import { discoverPackage } from "./discover-package.js";
|
|
5
|
+
import { inspectRemoteMcpUrl } from "./remote-url-policy.js";
|
|
5
6
|
import { probeRuntime } from "./runtime-probe.js";
|
|
6
7
|
function buildFailure(id, message, impact, suggestedFix, evidence) {
|
|
7
8
|
return {
|
|
@@ -23,6 +24,16 @@ function buildWarning(id, message, impact, suggestedFix, evidence) {
|
|
|
23
24
|
...(evidence ? { evidence } : {})
|
|
24
25
|
};
|
|
25
26
|
}
|
|
27
|
+
function remoteUrlIssueFindingId(issue) {
|
|
28
|
+
return issue === "insecure_non_loopback"
|
|
29
|
+
? "plugin.security.insecure_http_url"
|
|
30
|
+
: `plugin.security.remote_mcp_url.${issue}`;
|
|
31
|
+
}
|
|
32
|
+
function remoteUrlIssueMessage(issue) {
|
|
33
|
+
return issue === "insecure_non_loopback"
|
|
34
|
+
? "uses an insecure public HTTP URL"
|
|
35
|
+
: `uses a remote MCP URL with ${issue.replaceAll("_", " ")}`;
|
|
36
|
+
}
|
|
26
37
|
async function directoryExists(targetPath) {
|
|
27
38
|
try {
|
|
28
39
|
const details = await stat(targetPath);
|
|
@@ -376,6 +387,24 @@ async function validateMcpConfig(discoveredPackage) {
|
|
|
376
387
|
field: "transport"
|
|
377
388
|
}));
|
|
378
389
|
}
|
|
390
|
+
if (typeof command === "string" && typeof url === "string") {
|
|
391
|
+
findings.push(buildFailure("plugin.mcp.server.transport.conflict", `The MCP server \`${serverName}\` must not define both \`command\` and \`url\`.`, "A server with two transports cannot be selected deterministically by MCP clients.", `Keep either \`command\` or \`url\` for the \`${serverName}\` entry in \`${mcpConfigPath}\`.`, {
|
|
392
|
+
configPath: relativePackagePath(rootPath, mcpConfigPath),
|
|
393
|
+
serverName,
|
|
394
|
+
field: "transport"
|
|
395
|
+
}));
|
|
396
|
+
}
|
|
397
|
+
if (typeof url === "string") {
|
|
398
|
+
const inspection = inspectRemoteMcpUrl(url);
|
|
399
|
+
for (const issue of inspection.issues) {
|
|
400
|
+
findings.push(buildFailure(remoteUrlIssueFindingId(issue), `The MCP server \`${serverName}\` ${remoteUrlIssueMessage(issue)}.`, "Unsafe or ambiguous remote transport configuration can expose credentials or prevent reliable MCP connectivity.", "Use an absolute HTTPS URL without credentials, query parameters, fragments, or numeric IP literals; HTTP is only supported for localhost development.", {
|
|
401
|
+
configPath: relativePackagePath(rootPath, mcpConfigPath),
|
|
402
|
+
serverName,
|
|
403
|
+
field: "url",
|
|
404
|
+
url: inspection.sanitizedUrl
|
|
405
|
+
}));
|
|
406
|
+
}
|
|
407
|
+
}
|
|
379
408
|
if (isPlainObject(env)) {
|
|
380
409
|
for (const [envKey, envValue] of Object.entries(env)) {
|
|
381
410
|
if (isSensitiveKey(envKey) &&
|
|
@@ -415,7 +444,9 @@ export async function validatePlugin(targetPath, options = {}) {
|
|
|
415
444
|
? await probeRuntime(discoveredPackage, {
|
|
416
445
|
startupTimeoutMs: options.runtimeStartupTimeoutMs,
|
|
417
446
|
sandbox: options.runtimeSandbox,
|
|
418
|
-
transcript: options.runtimeTranscript
|
|
447
|
+
transcript: options.runtimeTranscript,
|
|
448
|
+
allowNetwork: options.allowNetwork,
|
|
449
|
+
allowLocalNetwork: options.allowLocalNetwork
|
|
419
450
|
})
|
|
420
451
|
: null;
|
|
421
452
|
const findings = [
|