@bryan-thompson/inspector-assessment-cli 1.36.1 → 1.36.3

package/build/__tests__/assessment-runner/tools-with-hints.test.js

@@ -0,0 +1,224 @@
+/**
+ * Tests for tools-with-hints module
+ *
+ * Issue #155: Verifies that hint properties (readOnlyHint, etc.) are preserved
+ * from raw MCP transport responses, even when the SDK's Zod validation would
+ * normally strip them.
+ */
+import { jest, describe, it, expect, beforeEach, afterEach } from "@jest/globals";
+import { getToolsWithPreservedHints, } from "../../lib/assessment-runner/tools-with-hints.js";
+describe("getToolsWithPreservedHints", () => {
+    let mockClient;
+    let mockTransport;
+    beforeEach(() => {
+        mockTransport = {
+            onmessage: undefined,
+        };
+        mockClient = {
+            transport: mockTransport,
+            listTools: jest.fn(),
+        };
+    });
+    afterEach(() => {
+        jest.clearAllMocks();
+    });
+    it("should preserve readOnlyHint from direct property in raw response", async () => {
+        // SDK returns tool without readOnlyHint (stripped by Zod)
+        mockClient.listTools.mockImplementation(async () => {
+            // Simulate raw response before SDK processes it
+            if (mockTransport.onmessage) {
+                mockTransport.onmessage({
+                    result: {
+                        tools: [
+                            {
+                                name: "browse_subreddit",
+                                description: "Browse posts",
+                                readOnlyHint: true, // Direct property in raw response
+                            },
+                        ],
+                    },
+                });
+            }
+            // Return SDK-processed tools (without direct readOnlyHint)
+            return {
+                tools: [
+                    {
+                        name: "browse_subreddit",
+                        description: "Browse posts",
+                        // readOnlyHint stripped by SDK Zod validation
+                    },
+                ],
+            };
+        });
+        const tools = await getToolsWithPreservedHints(mockClient);
+        expect(tools).toHaveLength(1);
+        expect(tools[0].name).toBe("browse_subreddit");
+        expect(tools[0].readOnlyHint).toBe(true);
+    });
+    it("should preserve all hint properties from raw response", async () => {
+        mockClient.listTools.mockImplementation(async () => {
+            if (mockTransport.onmessage) {
+                mockTransport.onmessage({
+                    result: {
+                        tools: [
+                            {
+                                name: "delete_item",
+                                readOnlyHint: false,
+                                destructiveHint: true,
+                                idempotentHint: false,
+                                openWorldHint: true,
+                            },
+                        ],
+                    },
+                });
+            }
+            return { tools: [{ name: "delete_item" }] };
+        });
+        const tools = await getToolsWithPreservedHints(mockClient);
+        expect(tools[0].readOnlyHint).toBe(false);
+        expect(tools[0].destructiveHint).toBe(true);
+        expect(tools[0].idempotentHint).toBe(false);
+        expect(tools[0].openWorldHint).toBe(true);
+    });
+    it("should not override SDK annotations with raw direct properties", async () => {
+        // If SDK has annotations, they take precedence
+        mockClient.listTools.mockImplementation(async () => {
+            if (mockTransport.onmessage) {
+                mockTransport.onmessage({
+                    result: {
+                        tools: [
+                            {
+                                name: "read_file",
+                                readOnlyHint: false, // Raw says false
+                            },
+                        ],
+                    },
+                });
+            }
+            return {
+                tools: [
+                    {
+                        name: "read_file",
+                        annotations: {
+                            readOnlyHint: true, // SDK annotations say true
+                        },
+                    },
+                ],
+            };
+        });
+        const tools = await getToolsWithPreservedHints(mockClient);
+        // SDK annotations should take precedence
+        expect(tools[0].annotations?.readOnlyHint).toBe(true);
+        // Direct property should NOT override
+        expect(tools[0].readOnlyHint).toBeUndefined();
+    });
+    it("should handle tools with no raw match gracefully", async () => {
+        mockClient.listTools.mockImplementation(async () => {
+            if (mockTransport.onmessage) {
+                mockTransport.onmessage({
+                    result: {
+                        tools: [
+                            { name: "tool_a", readOnlyHint: true },
+                            // tool_b missing from raw
+                        ],
+                    },
+                });
+            }
+            return {
+                tools: [
+                    { name: "tool_a" },
+                    { name: "tool_b" }, // Not in raw response
+                ],
+            };
+        });
+        const tools = await getToolsWithPreservedHints(mockClient);
+        expect(tools).toHaveLength(2);
+        expect(tools[0].readOnlyHint).toBe(true);
+        expect(tools[1].readOnlyHint).toBeUndefined();
+    });
+    it("should fallback to SDK tools when transport not available", async () => {
+        mockClient.transport = null;
+        mockClient.listTools.mockImplementation(async () => ({
+            tools: [{ name: "test_tool", annotations: { readOnlyHint: true } }],
+        }));
+        const tools = await getToolsWithPreservedHints(mockClient);
+        expect(tools).toHaveLength(1);
+        expect(tools[0].name).toBe("test_tool");
+    });
+    it("should fallback to SDK tools when raw capture fails", async () => {
+        mockClient.listTools.mockImplementation(async () => {
+            // Don't trigger onmessage (simulating capture failure)
+            return {
+                tools: [{ name: "fallback_tool" }],
+            };
+        });
+        const tools = await getToolsWithPreservedHints(mockClient);
+        expect(tools).toHaveLength(1);
+        expect(tools[0].name).toBe("fallback_tool");
+    });
+    it("should preserve hints from metadata object", async () => {
+        mockClient.listTools.mockImplementation(async () => {
+            if (mockTransport.onmessage) {
+                mockTransport.onmessage({
+                    result: {
+                        tools: [
+                            {
+                                name: "query_db",
+                                metadata: {
+                                    readOnlyHint: true,
+                                },
+                            },
+                        ],
+                    },
+                });
+            }
+            return { tools: [{ name: "query_db" }] };
+        });
+        const tools = await getToolsWithPreservedHints(mockClient);
+        expect(tools[0].readOnlyHint).toBe(true);
+    });
+    it("should preserve hints from _meta object", async () => {
+        mockClient.listTools.mockImplementation(async () => {
+            if (mockTransport.onmessage) {
+                mockTransport.onmessage({
+                    result: {
+                        tools: [
+                            {
+                                name: "custom_tool",
+                                _meta: {
+                                    destructiveHint: true,
+                                },
+                            },
+                        ],
+                    },
+                });
+            }
+            return { tools: [{ name: "custom_tool" }] };
+        });
+        const tools = await getToolsWithPreservedHints(mockClient);
+        expect(tools[0].destructiveHint).toBe(true);
+    });
+    it("should restore original onmessage handler after call", async () => {
+        const originalHandler = jest.fn();
+        mockTransport.onmessage = originalHandler;
+        mockClient.listTools.mockImplementation(async () => {
+            if (mockTransport.onmessage) {
+                mockTransport.onmessage({ result: { tools: [] } });
+            }
+            return { tools: [] };
+        });
+        await getToolsWithPreservedHints(mockClient);
+        // Original handler should be restored
+        expect(mockTransport.onmessage).toBe(originalHandler);
+    });
+    it("should restore handler even on error", async () => {
+        const originalHandler = jest.fn();
+        mockTransport.onmessage = originalHandler;
+        mockClient.listTools.mockImplementation(async () => {
+            throw new Error("Test error");
+        });
+        await expect(getToolsWithPreservedHints(mockClient)).rejects.toThrow("Test error");
+        // Original handler should be restored despite error
+        expect(mockTransport.onmessage).toBe(originalHandler);
+    });
+});

package/build/__tests__/jsonl-events.test.js

@@ -142,6 +142,71 @@ describe("JSONL Event Emission", () => {
             emitToolDiscovered(tool);
             expect(emittedEvents[0].annotations).toBeNull();
         });
+        // Issue #155: Test that direct properties are detected
+        it("should detect annotations from direct properties (Issue #155)", () => {
+            // Simulate a tool with readOnlyHint as a direct property (not in annotations object)
+            // This is how some MCP SDKs serialize annotations
+            const tool = {
+                name: "browse_subreddit",
+                description: "Browse a subreddit",
+                inputSchema: { type: "object" },
+                readOnlyHint: true, // Direct property, not in annotations object
+            };
+            emitToolDiscovered(tool);
+            const annotations = emittedEvents[0].annotations;
+            expect(annotations).not.toBeNull();
+            expect(annotations?.readOnlyHint).toBe(true);
+        });
+        it("should detect annotations from metadata object (Issue #155)", () => {
+            // Simulate a tool with annotations in metadata object
+            const tool = {
+                name: "get_data",
+                description: "Get data",
+                inputSchema: { type: "object" },
+                metadata: {
+                    readOnlyHint: true,
+                    destructiveHint: false,
+                },
+            };
+            emitToolDiscovered(tool);
+            const annotations = emittedEvents[0].annotations;
+            expect(annotations).not.toBeNull();
+            expect(annotations?.readOnlyHint).toBe(true);
+            expect(annotations?.destructiveHint).toBe(false);
+        });
+        it("should detect annotations from _meta object (Issue #155)", () => {
+            // Simulate a tool with annotations in _meta object
+            const tool = {
+                name: "search_docs",
+                description: "Search documents",
+                inputSchema: { type: "object" },
+                _meta: {
+                    idempotentHint: true,
+                    openWorldHint: false,
+                },
+            };
+            emitToolDiscovered(tool);
+            const annotations = emittedEvents[0].annotations;
+            expect(annotations).not.toBeNull();
+            expect(annotations?.idempotentHint).toBe(true);
+            expect(annotations?.openWorldHint).toBe(false);
+        });
+        it("should prioritize tool.annotations over direct properties (Issue #155)", () => {
+            // When both exist, tool.annotations should take precedence
+            const tool = {
+                name: "conflicting_tool",
+                description: "Test priority",
+                inputSchema: { type: "object" },
+                annotations: {
+                    readOnlyHint: false, // In annotations object
+                },
+                readOnlyHint: true, // Direct property (should be ignored)
+            };
+            emitToolDiscovered(tool);
+            const annotations = emittedEvents[0].annotations;
+            expect(annotations).not.toBeNull();
+            expect(annotations?.readOnlyHint).toBe(false); // Should use annotations object value
+        });
     });
     describe("emitToolsDiscoveryComplete", () => {
         it("should emit tools_discovery_complete with count", () => {

package/build/lib/assessment-runner/assessment-executor.js

@@ -18,6 +18,8 @@ import { createCallToolWrapper } from "./tool-wrapper.js";
 import { buildConfig } from "./config-builder.js";
 // Issue #155: Import annotation debug mode setter
 import { setAnnotationDebugMode } from "../../../../client/lib/services/assessment/modules/annotations/AlignmentChecker.js";
+// Issue #155: Import helper to preserve hint properties stripped by SDK
+import { getToolsWithPreservedHints } from "./tools-with-hints.js";
 /**
  * Run full assessment against an MCP server
  *
@@ -65,9 +67,12 @@ export async function runFullAssessment(options) {
     if (!serverInfo && !options.jsonOnly) {
         console.log("⚠️  Server did not provide serverInfo during initialization");
     }
-    const response = await client.listTools();
-    const tools = response.tools || [];
+    // Issue #155: Use helper that preserves hint properties stripped by SDK Zod validation
+    // The SDK's listTools() validates against a schema that strips direct properties
+    // like readOnlyHint. This helper intercepts the raw transport response to preserve them.
+    const tools = await getToolsWithPreservedHints(client);
     // Emit JSONL tool discovery events for audit-worker parsing
+    // Tools now have hint properties preserved from raw response
     for (const tool of tools) {
         emitToolDiscovered(tool);
     }
@@ -283,9 +288,9 @@ export async function runFullAssessment(options) {
         serverName: options.serverName,
         tools,
         callTool: createCallToolWrapper(client),
+        // Issue #155: Use helper to preserve hint properties in refreshed tool lists
         listTools: async () => {
-            const response = await client.listTools();
-            return response.tools;
+            return getToolsWithPreservedHints(client);
         },
         config,
         sourceCodePath: options.sourceCodePath,

package/build/lib/assessment-runner/tools-with-hints.js

@@ -0,0 +1,106 @@
+/**
+ * Tools With Preserved Hints
+ *
+ * Issue #155: The MCP SDK's listTools() method validates responses against
+ * a Zod schema that strips properties not explicitly defined (like readOnlyHint
+ * as a direct property on tools). This module intercepts the raw transport
+ * response to preserve these properties.
+ *
+ * @module cli/lib/assessment-runner/tools-with-hints
+ */
+// Hint property names we want to preserve
+const HINT_PROPERTIES = [
+    "readOnlyHint",
+    "destructiveHint",
+    "idempotentHint",
+    "openWorldHint",
+];
+/**
+ * Get tools from MCP server with hint properties preserved.
+ *
+ * The MCP SDK's listTools() validates against a Zod schema that may strip
+ * direct hint properties (readOnlyHint, etc.) that aren't in the schema.
+ * This function intercepts the raw transport response to preserve them.
+ *
+ * @param client - Connected MCP client
+ * @returns Tools with hint properties preserved from raw response
+ */
+export async function getToolsWithPreservedHints(client) {
+    let rawTools = [];
+    // Get the underlying transport
+    const transport = client.transport;
+    if (!transport) {
+        // Fallback: just use SDK listTools if transport not accessible
+        const response = await client.listTools();
+        return response.tools || [];
+    }
+    // Store original message handler
+    const originalOnMessage = transport.onmessage;
+    // Intercept messages to capture raw tools/list response
+    transport.onmessage = (message, extra) => {
+        // Check if this is a tools/list response
+        const msg = message;
+        if (msg?.result?.tools && Array.isArray(msg.result.tools)) {
+            rawTools = msg.result.tools;
+        }
+        // Call original handler (cast to any to preserve original typing)
+        if (originalOnMessage) {
+            originalOnMessage(message, extra);
+        }
+    };
+    try {
+        // Make SDK call (triggers our interceptor)
+        const sdkResponse = await client.listTools();
+        const sdkTools = sdkResponse.tools || [];
+        // Restore original handler
+        transport.onmessage = originalOnMessage;
+        // If we didn't capture raw tools, just return SDK tools
+        if (rawTools.length === 0) {
+            return sdkTools;
+        }
+        // Merge preserved hint properties from raw into SDK tools
+        return sdkTools.map((sdkTool) => {
+            // Find matching raw tool by name
+            const rawTool = rawTools.find((rt) => rt.name === sdkTool.name);
+            if (!rawTool) {
+                return sdkTool;
+            }
+            // Start with SDK tool
+            const enrichedTool = { ...sdkTool };
+            // Preserve hint properties from raw response (priority order)
+            for (const hint of HINT_PROPERTIES) {
+                // Skip if SDK already has it via annotations
+                if (sdkTool.annotations?.[hint] !== undefined) {
+                    continue;
+                }
+                // Check raw tool locations in priority order:
+                // 1. Direct property on raw tool
+                // 2. annotations object
+                // 3. metadata object
+                // 4. _meta object
+                let value;
+                if (typeof rawTool[hint] === "boolean") {
+                    value = rawTool[hint];
+                }
+                else if (typeof rawTool.annotations?.[hint] === "boolean") {
+                    value = rawTool.annotations[hint];
+                }
+                else if (typeof rawTool.metadata?.[hint] === "boolean") {
+                    value = rawTool.metadata[hint];
+                }
+                else if (typeof rawTool._meta?.[hint] === "boolean") {
+                    value = rawTool._meta[hint];
+                }
+                if (value !== undefined) {
+                    enrichedTool[hint] = value;
+                }
+            }
+            return enrichedTool;
+        });
+    }
+    catch (error) {
+        // Restore original handler on error
+        transport.onmessage = originalOnMessage;
+        throw error;
+    }
+}

package/build/lib/jsonl-events.js

@@ -60,16 +60,99 @@ export function extractToolParams(schema) {
 /**
  * Emit tool_discovered event for each tool found.
  * Includes annotations if the server provides them.
+ *
+ * Issue #155: Check multiple locations for annotations:
+ * 1. tool.annotations object (MCP 2024-11 spec)
+ * 2. Direct properties (tool.readOnlyHint, etc.)
+ * 3. tool.metadata object
+ * 4. tool._meta object
  */
 export function emitToolDiscovered(tool) {
     const params = extractToolParams(tool.inputSchema);
-    // Extract annotations, null if not present
-    const annotations = tool.annotations
+    // Issue #155: Extract annotations from multiple sources (priority order)
+    // NOTE: This is a simplified version of AlignmentChecker.extractAnnotations()
+    // that only checks *Hint-suffixed properties (readOnlyHint, destructiveHint, etc.),
+    // not non-suffixed variants like readOnly, destructive, idempotent, openWorld.
+    // See AlignmentChecker.resolveAnnotationValue() for full implementation with fallbacks.
+    const toolAny = tool;
+    // Priority 1: Check tool.annotations object (MCP spec)
+    let readOnlyHint;
+    let destructiveHint;
+    let idempotentHint;
+    let openWorldHint;
+    if (tool.annotations) {
+        readOnlyHint = tool.annotations.readOnlyHint;
+        destructiveHint = tool.annotations.destructiveHint;
+        idempotentHint = tool.annotations.idempotentHint;
+        openWorldHint = tool.annotations.openWorldHint;
+    }
+    // Priority 2: Check direct properties on tool object
+    // Only use if not already found in annotations
+    if (readOnlyHint === undefined && typeof toolAny.readOnlyHint === "boolean") {
+        readOnlyHint = toolAny.readOnlyHint;
+    }
+    if (destructiveHint === undefined &&
+        typeof toolAny.destructiveHint === "boolean") {
+        destructiveHint = toolAny.destructiveHint;
+    }
+    if (idempotentHint === undefined &&
+        typeof toolAny.idempotentHint === "boolean") {
+        idempotentHint = toolAny.idempotentHint;
+    }
+    if (openWorldHint === undefined &&
+        typeof toolAny.openWorldHint === "boolean") {
+        openWorldHint = toolAny.openWorldHint;
+    }
+    // Priority 3: Check tool.metadata object
+    const metadata = toolAny.metadata;
+    if (metadata) {
+        if (readOnlyHint === undefined &&
+            typeof metadata.readOnlyHint === "boolean") {
+            readOnlyHint = metadata.readOnlyHint;
+        }
+        if (destructiveHint === undefined &&
+            typeof metadata.destructiveHint === "boolean") {
+            destructiveHint = metadata.destructiveHint;
+        }
+        if (idempotentHint === undefined &&
+            typeof metadata.idempotentHint === "boolean") {
+            idempotentHint = metadata.idempotentHint;
+        }
+        if (openWorldHint === undefined &&
+            typeof metadata.openWorldHint === "boolean") {
+            openWorldHint = metadata.openWorldHint;
+        }
+    }
+    // Priority 4: Check tool._meta object
+    const _meta = toolAny._meta;
+    if (_meta) {
+        if (readOnlyHint === undefined && typeof _meta.readOnlyHint === "boolean") {
+            readOnlyHint = _meta.readOnlyHint;
+        }
+        if (destructiveHint === undefined &&
+            typeof _meta.destructiveHint === "boolean") {
+            destructiveHint = _meta.destructiveHint;
+        }
+        if (idempotentHint === undefined &&
+            typeof _meta.idempotentHint === "boolean") {
+            idempotentHint = _meta.idempotentHint;
+        }
+        if (openWorldHint === undefined &&
+            typeof _meta.openWorldHint === "boolean") {
+            openWorldHint = _meta.openWorldHint;
+        }
+    }
+    // Build annotations object if any hints were found
+    const hasAnnotations = readOnlyHint !== undefined ||
+        destructiveHint !== undefined ||
+        idempotentHint !== undefined ||
+        openWorldHint !== undefined;
+    const annotations = hasAnnotations
         ? {
-            readOnlyHint: tool.annotations.readOnlyHint,
-            destructiveHint: tool.annotations.destructiveHint,
-            idempotentHint: tool.annotations.idempotentHint,
-            openWorldHint: tool.annotations.openWorldHint,
+            readOnlyHint,
+            destructiveHint,
+            idempotentHint,
+            openWorldHint,
         }
         : null;
     emitJSONL({

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bryan-thompson/inspector-assessment-cli",
-  "version": "1.36.1",
+  "version": "1.36.3",
   "description": "CLI for the Enhanced MCP Inspector with assessment capabilities",
   "license": "MIT",
   "author": "Bryan Thompson <bryan@triepod.ai>",