@bryan-thompson/inspector-assessment-cli 1.30.1 → 1.32.0

package/build/lib/__tests__/zodErrorFormatter.test.js

@@ -0,0 +1,282 @@
+/**
+ * Tests for Zod Error Formatting Utilities
+ *
+ * Validates the error formatting functions for CLI-friendly output.
+ *
+ * @module cli/lib/__tests__/zodErrorFormatter
+ */
+// Uses Jest globals (describe, test, expect)
+import { jest } from "@jest/globals";
+import { z } from "zod";
+import { formatZodIssue, formatZodError, formatZodErrorIndented, printZodErrorForCli, zodErrorToArray, formatZodErrorForJson, formatUserFriendlyError, } from "../zodErrorFormatter.js";
+// Helper to create a ZodError with specific issues
+function createZodError(issues) {
+    // Create a schema that will fail and then manipulate the error
+    const schema = z.object({ dummy: z.string() });
+    const result = schema.safeParse({ dummy: 123 });
+    if (!result.success) {
+        // Replace the errors with our custom issues
+        result.error.issues = issues.map((issue) => ({
+            code: issue.code || "custom",
+            path: issue.path,
+            message: issue.message,
+        }));
+        return result.error;
+    }
+    throw new Error("Failed to create ZodError");
+}
+describe("zodErrorFormatter", () => {
+    describe("formatZodIssue", () => {
+        test("formats issue with empty path (message only)", () => {
+            const issue = { code: "custom", path: [], message: "Required" };
+            const result = formatZodIssue(issue);
+            expect(result).toBe("Required");
+        });
+        test("formats issue with single-level path", () => {
+            const issue = {
+                code: "custom",
+                path: ["field"],
+                message: "Invalid value",
+            };
+            const result = formatZodIssue(issue);
+            expect(result).toBe("field: Invalid value");
+        });
+        test("formats issue with nested path", () => {
+            const issue = {
+                code: "custom",
+                path: ["config", "nested", "field"],
+                message: "Must be a string",
+            };
+            const result = formatZodIssue(issue);
+            expect(result).toBe("config.nested.field: Must be a string");
+        });
+        test("formats issue with array index in path", () => {
+            const issue = {
+                code: "custom",
+                path: ["items", 0, "name"],
+                message: "Required",
+            };
+            const result = formatZodIssue(issue);
+            expect(result).toBe("items.0.name: Required");
+        });
+    });
+    describe("formatZodError", () => {
+        test("formats single error", () => {
+            const error = createZodError([
+                { path: ["field"], message: "Invalid value" },
+            ]);
+            const result = formatZodError(error);
+            expect(result).toBe("field: Invalid value");
+        });
+        test("formats multiple errors with newlines", () => {
+            const error = createZodError([
+                { path: ["field1"], message: "Error 1" },
+                { path: ["field2"], message: "Error 2" },
+                { path: ["field3"], message: "Error 3" },
+            ]);
+            const result = formatZodError(error);
+            expect(result).toBe("field1: Error 1\nfield2: Error 2\nfield3: Error 3");
+        });
+        test("handles error with empty path", () => {
+            const error = createZodError([{ path: [], message: "Global error" }]);
+            const result = formatZodError(error);
+            expect(result).toBe("Global error");
+        });
+    });
+    describe("formatZodErrorIndented", () => {
+        test("uses default indentation (two spaces)", () => {
+            const error = createZodError([
+                { path: ["field"], message: "Invalid value" },
+            ]);
+            const result = formatZodErrorIndented(error);
+            expect(result).toBe("  field: Invalid value");
+        });
+        test("uses custom indentation", () => {
+            const error = createZodError([
+                { path: ["field"], message: "Invalid value" },
+            ]);
+            const result = formatZodErrorIndented(error, "\t");
+            expect(result).toBe("\tfield: Invalid value");
+        });
+        test("uses four spaces indentation", () => {
+            const error = createZodError([
+                { path: ["field"], message: "Invalid value" },
+            ]);
+            const result = formatZodErrorIndented(error, "    ");
+            expect(result).toBe("    field: Invalid value");
+        });
+        test("applies indent to each line for multiple errors", () => {
+            const error = createZodError([
+                { path: ["field1"], message: "Error 1" },
+                { path: ["field2"], message: "Error 2" },
+            ]);
+            const result = formatZodErrorIndented(error, ">> ");
+            expect(result).toBe(">> field1: Error 1\n>> field2: Error 2");
+        });
+        test("uses empty indentation", () => {
+            const error = createZodError([
+                { path: ["field"], message: "Invalid value" },
+            ]);
+            const result = formatZodErrorIndented(error, "");
+            expect(result).toBe("field: Invalid value");
+        });
+    });
+    describe("printZodErrorForCli", () => {
+        let consoleSpy;
+        beforeEach(() => {
+            consoleSpy = jest.spyOn(console, "error").mockImplementation(() => { });
+        });
+        afterEach(() => {
+            consoleSpy.mockRestore();
+        });
+        test("prints with context prefix", () => {
+            const error = createZodError([
+                { path: ["field"], message: "Invalid value" },
+            ]);
+            printZodErrorForCli(error, "config file");
+            expect(consoleSpy).toHaveBeenCalledWith("Error in config file:\n  field: Invalid value");
+        });
+        test("prints without context (default prefix)", () => {
+            const error = createZodError([
+                { path: ["field"], message: "Invalid value" },
+            ]);
+            printZodErrorForCli(error);
+            expect(consoleSpy).toHaveBeenCalledWith("Validation error:\n  field: Invalid value");
+        });
+        test("outputs multiple errors with indentation", () => {
+            const error = createZodError([
+                { path: ["field1"], message: "Error 1" },
+                { path: ["field2"], message: "Error 2" },
+            ]);
+            printZodErrorForCli(error, "CLI arguments");
+            expect(consoleSpy).toHaveBeenCalledWith("Error in CLI arguments:\n  field1: Error 1\n  field2: Error 2");
+        });
+    });
+    describe("zodErrorToArray", () => {
+        test("converts single error to array", () => {
+            const error = createZodError([
+                { path: ["field"], message: "Invalid value" },
+            ]);
+            const result = zodErrorToArray(error);
+            expect(result).toEqual(["field: Invalid value"]);
+        });
+        test("converts multiple errors to array", () => {
+            const error = createZodError([
+                { path: ["field1"], message: "Error 1" },
+                { path: ["field2"], message: "Error 2" },
+                { path: ["field3"], message: "Error 3" },
+            ]);
+            const result = zodErrorToArray(error);
+            expect(result).toEqual([
+                "field1: Error 1",
+                "field2: Error 2",
+                "field3: Error 3",
+            ]);
+        });
+        test("returns array preserving order", () => {
+            const error = createZodError([
+                { path: ["z"], message: "Z error" },
+                { path: ["a"], message: "A error" },
+                { path: ["m"], message: "M error" },
+            ]);
+            const result = zodErrorToArray(error);
+            expect(result).toEqual(["z: Z error", "a: A error", "m: M error"]);
+        });
+    });
+    describe("formatZodErrorForJson", () => {
+        test("includes message field", () => {
+            const error = createZodError([
+                { path: ["field"], message: "Invalid value" },
+            ]);
+            const result = formatZodErrorForJson(error);
+            expect(result.message).toBe("Validation failed");
+        });
+        test("includes errors array with path, message, code", () => {
+            const error = createZodError([
+                { path: ["field"], message: "Invalid value", code: "invalid_type" },
+            ]);
+            const result = formatZodErrorForJson(error);
+            expect(result.errors).toHaveLength(1);
+            expect(result.errors[0]).toEqual({
+                path: ["field"],
+                message: "Invalid value",
+                code: "invalid_type",
+            });
+        });
+        test("preserves path as array with numbers and strings", () => {
+            const error = createZodError([
+                { path: ["items", 0, "nested", 1, "field"], message: "Error" },
+            ]);
+            const result = formatZodErrorForJson(error);
+            expect(result.errors[0].path).toEqual(["items", 0, "nested", 1, "field"]);
+        });
+        test("handles multiple errors", () => {
+            const error = createZodError([
+                { path: ["field1"], message: "Error 1", code: "too_small" },
+                { path: ["field2"], message: "Error 2", code: "invalid_type" },
+            ]);
+            const result = formatZodErrorForJson(error);
+            expect(result.errors).toHaveLength(2);
+            expect(result.errors[0].code).toBe("too_small");
+            expect(result.errors[1].code).toBe("invalid_type");
+        });
+    });
+    describe("formatUserFriendlyError", () => {
+        test("single error returns plain message", () => {
+            const error = createZodError([
+                { path: ["username"], message: "Required" },
+            ]);
+            const result = formatUserFriendlyError(error);
+            expect(result).toBe("username: Required");
+        });
+        test("single error with empty path returns message only", () => {
+            const error = createZodError([{ path: [], message: "Invalid input" }]);
+            const result = formatUserFriendlyError(error);
+            expect(result).toBe("Invalid input");
+        });
+        test("multiple errors returns bulleted list", () => {
+            const error = createZodError([
+                { path: ["field1"], message: "Error 1" },
+                { path: ["field2"], message: "Error 2" },
+            ]);
+            const result = formatUserFriendlyError(error);
+            expect(result).toBe("Multiple validation errors:\n  - field1: Error 1\n  - field2: Error 2");
+        });
+        test("applies field label mapping", () => {
+            const error = createZodError([
+                { path: ["serverName"], message: "Required" },
+            ]);
+            const labels = { serverName: "Server Name" };
+            const result = formatUserFriendlyError(error, labels);
+            expect(result).toBe("Server Name: Required");
+        });
+        test("applies labels for multiple errors", () => {
+            const error = createZodError([
+                { path: ["serverName"], message: "Required" },
+                { path: ["configPath"], message: "Invalid path" },
+            ]);
+            const labels = {
+                serverName: "Server Name",
+                configPath: "Configuration Path",
+            };
+            const result = formatUserFriendlyError(error, labels);
+            expect(result).toBe("Multiple validation errors:\n  - Server Name: Required\n  - Configuration Path: Invalid path");
+        });
+        test("falls back to path when no matching label", () => {
+            const error = createZodError([
+                { path: ["unknownField"], message: "Error" },
+            ]);
+            const labels = { otherField: "Other Field" };
+            const result = formatUserFriendlyError(error, labels);
+            expect(result).toBe("unknownField: Error");
+        });
+        test("handles nested path with labels", () => {
+            const error = createZodError([
+                { path: ["config", "nested"], message: "Invalid" },
+            ]);
+            const labels = { "config.nested": "Nested Config" };
+            const result = formatUserFriendlyError(error, labels);
+            expect(result).toBe("Nested Config: Invalid");
+        });
+    });
+});

package/build/lib/assessment-runner/__tests__/server-configSchemas.test.js

@@ -0,0 +1,394 @@
+/**
+ * Tests for Server Configuration Zod Schemas
+ *
+ * Validates the schema definitions used for server config file parsing.
+ *
+ * @module cli/lib/assessment-runner/__tests__/server-configSchemas
+ */
+// Uses Jest globals (describe, test, expect)
+import { ZodError } from "zod";
+import { HttpSseServerConfigSchema, StdioServerConfigSchema, ServerEntrySchema, ClaudeDesktopConfigSchema, StandaloneConfigSchema, ConfigFileSchema, parseConfigFile, safeParseConfigFile, validateServerEntry, isHttpSseConfig, isStdioConfig, TransportTypeSchema, } from "../server-configSchemas.js";
+describe("server-configSchemas", () => {
+    describe("Re-exported schemas", () => {
+        test("exports TransportTypeSchema", () => {
+            expect(TransportTypeSchema.safeParse("stdio").success).toBe(true);
+            expect(TransportTypeSchema.safeParse("http").success).toBe(true);
+            expect(TransportTypeSchema.safeParse("sse").success).toBe(true);
+        });
+    });
+    describe("HttpSseServerConfigSchema", () => {
+        describe("valid configs", () => {
+            test("accepts url only", () => {
+                const result = HttpSseServerConfigSchema.safeParse({
+                    url: "http://localhost:3000/mcp",
+                });
+                expect(result.success).toBe(true);
+            });
+            test("accepts url with transport http", () => {
+                const result = HttpSseServerConfigSchema.safeParse({
+                    transport: "http",
+                    url: "http://localhost:3000/mcp",
+                });
+                expect(result.success).toBe(true);
+            });
+            test("accepts url with transport sse", () => {
+                const result = HttpSseServerConfigSchema.safeParse({
+                    transport: "sse",
+                    url: "http://localhost:3000/sse",
+                });
+                expect(result.success).toBe(true);
+            });
+            test("accepts https url", () => {
+                const result = HttpSseServerConfigSchema.safeParse({
+                    url: "https://api.example.com/mcp",
+                });
+                expect(result.success).toBe(true);
+            });
+        });
+        describe("invalid configs", () => {
+            test("rejects missing url", () => {
+                const result = HttpSseServerConfigSchema.safeParse({
+                    transport: "http",
+                });
+                expect(result.success).toBe(false);
+            });
+            test("rejects invalid URL format", () => {
+                const result = HttpSseServerConfigSchema.safeParse({
+                    url: "not-a-valid-url",
+                });
+                expect(result.success).toBe(false);
+            });
+            test("rejects empty url", () => {
+                const result = HttpSseServerConfigSchema.safeParse({
+                    url: "",
+                });
+                expect(result.success).toBe(false);
+            });
+            test("rejects invalid transport value", () => {
+                const result = HttpSseServerConfigSchema.safeParse({
+                    transport: "stdio", // stdio not allowed in this schema
+                    url: "http://localhost:3000",
+                });
+                expect(result.success).toBe(false);
+            });
+        });
+    });
+    describe("StdioServerConfigSchema", () => {
+        describe("valid configs", () => {
+            test("accepts command only (minimum valid)", () => {
+                const result = StdioServerConfigSchema.safeParse({
+                    command: "python3",
+                });
+                expect(result.success).toBe(true);
+            });
+            test("accepts command with args", () => {
+                const result = StdioServerConfigSchema.safeParse({
+                    command: "python3",
+                    args: ["server.py", "--port", "8080"],
+                });
+                expect(result.success).toBe(true);
+            });
+            test("accepts command with env record", () => {
+                const result = StdioServerConfigSchema.safeParse({
+                    command: "python3",
+                    env: { DEBUG: "true", PATH: "/usr/bin" },
+                });
+                expect(result.success).toBe(true);
+            });
+            test("accepts command with cwd", () => {
+                const result = StdioServerConfigSchema.safeParse({
+                    command: "python3",
+                    cwd: "/home/user/project",
+                });
+                expect(result.success).toBe(true);
+            });
+            test("accepts full config with all fields", () => {
+                const result = StdioServerConfigSchema.safeParse({
+                    transport: "stdio",
+                    command: "python3",
+                    args: ["server.py"],
+                    env: { DEBUG: "1" },
+                    cwd: "/home/user/project",
+                });
+                expect(result.success).toBe(true);
+            });
+        });
+        describe("invalid configs", () => {
+            test("rejects missing command", () => {
+                const result = StdioServerConfigSchema.safeParse({
+                    transport: "stdio",
+                });
+                expect(result.success).toBe(false);
+            });
+            test("rejects empty command", () => {
+                const result = StdioServerConfigSchema.safeParse({
+                    command: "",
+                });
+                expect(result.success).toBe(false);
+                if (!result.success) {
+                    expect(result.error.errors[0].message).toContain("command is required");
+                }
+            });
+            test("rejects non-string array for args", () => {
+                const result = StdioServerConfigSchema.safeParse({
+                    command: "python3",
+                    args: [1, 2, 3],
+                });
+                expect(result.success).toBe(false);
+            });
+            test("rejects non-string values in env", () => {
+                const result = StdioServerConfigSchema.safeParse({
+                    command: "python3",
+                    env: { DEBUG: 123 },
+                });
+                expect(result.success).toBe(false);
+            });
+        });
+        describe("default values", () => {
+            test("args defaults to empty array", () => {
+                const result = StdioServerConfigSchema.safeParse({
+                    command: "python3",
+                });
+                expect(result.success).toBe(true);
+                if (result.success) {
+                    expect(result.data.args).toEqual([]);
+                }
+            });
+            test("env defaults to empty object", () => {
+                const result = StdioServerConfigSchema.safeParse({
+                    command: "python3",
+                });
+                expect(result.success).toBe(true);
+                if (result.success) {
+                    expect(result.data.env).toEqual({});
+                }
+            });
+        });
+    });
+    describe("ServerEntrySchema", () => {
+        test("accepts HTTP config", () => {
+            const result = ServerEntrySchema.safeParse({
+                url: "http://localhost:3000/mcp",
+            });
+            expect(result.success).toBe(true);
+        });
+        test("accepts SSE config", () => {
+            const result = ServerEntrySchema.safeParse({
+                transport: "sse",
+                url: "http://localhost:3000/sse",
+            });
+            expect(result.success).toBe(true);
+        });
+        test("accepts stdio config", () => {
+            const result = ServerEntrySchema.safeParse({
+                command: "python3",
+                args: ["server.py"],
+            });
+            expect(result.success).toBe(true);
+        });
+        test("rejects config matching neither schema", () => {
+            const result = ServerEntrySchema.safeParse({
+                transport: "unknown",
+            });
+            expect(result.success).toBe(false);
+        });
+        test("rejects empty object", () => {
+            const result = ServerEntrySchema.safeParse({});
+            expect(result.success).toBe(false);
+        });
+    });
+    describe("ClaudeDesktopConfigSchema", () => {
+        test("accepts {mcpServers: {...}} format", () => {
+            const result = ClaudeDesktopConfigSchema.safeParse({
+                mcpServers: {
+                    "my-server": { url: "http://localhost:3000/mcp" },
+                    "another-server": { command: "python3", args: ["server.py"] },
+                },
+            });
+            expect(result.success).toBe(true);
+        });
+        test("accepts empty mcpServers object", () => {
+            const result = ClaudeDesktopConfigSchema.safeParse({
+                mcpServers: {},
+            });
+            expect(result.success).toBe(true);
+        });
+        test("accepts config without mcpServers (undefined)", () => {
+            const result = ClaudeDesktopConfigSchema.safeParse({});
+            expect(result.success).toBe(true);
+        });
+        test("rejects invalid server entry within mcpServers", () => {
+            const result = ClaudeDesktopConfigSchema.safeParse({
+                mcpServers: {
+                    "invalid-server": { transport: "invalid" },
+                },
+            });
+            expect(result.success).toBe(false);
+        });
+    });
+    describe("StandaloneConfigSchema", () => {
+        test("accepts HTTP config directly", () => {
+            const result = StandaloneConfigSchema.safeParse({
+                url: "http://localhost:3000/mcp",
+            });
+            expect(result.success).toBe(true);
+        });
+        test("accepts stdio config directly", () => {
+            const result = StandaloneConfigSchema.safeParse({
+                command: "python3",
+            });
+            expect(result.success).toBe(true);
+        });
+        test("is equivalent to ServerEntrySchema", () => {
+            expect(StandaloneConfigSchema).toBe(ServerEntrySchema);
+        });
+    });
+    describe("ConfigFileSchema", () => {
+        test("accepts Claude Desktop format", () => {
+            const result = ConfigFileSchema.safeParse({
+                mcpServers: {
+                    "test-server": { url: "http://localhost:3000/mcp" },
+                },
+            });
+            expect(result.success).toBe(true);
+        });
+        test("accepts standalone HTTP config", () => {
+            const result = ConfigFileSchema.safeParse({
+                url: "http://localhost:3000/mcp",
+            });
+            expect(result.success).toBe(true);
+        });
+        test("accepts standalone stdio config", () => {
+            const result = ConfigFileSchema.safeParse({
+                command: "python3",
+                args: ["server.py"],
+            });
+            expect(result.success).toBe(true);
+        });
+        test("accepts unknown keys (Zod passthrough by default)", () => {
+            // ClaudeDesktopConfigSchema has mcpServers as optional
+            // Extra keys are allowed by default in Zod (non-strict mode)
+            const result = ConfigFileSchema.safeParse({
+                invalid: "config",
+            });
+            // This passes because it matches ClaudeDesktopConfigSchema with mcpServers undefined
+            expect(result.success).toBe(true);
+        });
+        test("rejects truly invalid values", () => {
+            // null is not a valid object
+            const result = ConfigFileSchema.safeParse(null);
+            expect(result.success).toBe(false);
+        });
+    });
+    describe("parseConfigFile", () => {
+        test("parses valid Claude Desktop config", () => {
+            const config = parseConfigFile({
+                mcpServers: {
+                    "test-server": { url: "http://localhost:3000/mcp" },
+                },
+            });
+            expect(config).toBeDefined();
+        });
+        test("parses valid standalone config", () => {
+            const config = parseConfigFile({
+                url: "http://localhost:3000/mcp",
+            });
+            expect(config).toBeDefined();
+        });
+        test("throws ZodError for null input", () => {
+            expect(() => parseConfigFile(null)).toThrow(ZodError);
+        });
+        test("throws ZodError for non-object input", () => {
+            expect(() => parseConfigFile("string")).toThrow(ZodError);
+        });
+    });
+    describe("safeParseConfigFile", () => {
+        test("returns success for valid config", () => {
+            const result = safeParseConfigFile({
+                url: "http://localhost:3000/mcp",
+            });
+            expect(result.success).toBe(true);
+        });
+        test("returns error for null input", () => {
+            const result = safeParseConfigFile(null);
+            expect(result.success).toBe(false);
+            if (!result.success) {
+                expect(result.error).toBeInstanceOf(ZodError);
+            }
+        });
+        test("returns error for non-object input", () => {
+            const result = safeParseConfigFile("string");
+            expect(result.success).toBe(false);
+        });
+    });
+    describe("validateServerEntry", () => {
+        test("returns empty array for valid HTTP entry", () => {
+            const errors = validateServerEntry({
+                url: "http://localhost:3000/mcp",
+            });
+            expect(errors).toEqual([]);
+        });
+        test("returns empty array for valid stdio entry", () => {
+            const errors = validateServerEntry({
+                command: "python3",
+            });
+            expect(errors).toEqual([]);
+        });
+        test("returns errors for invalid entry", () => {
+            const errors = validateServerEntry({});
+            expect(errors.length).toBeGreaterThan(0);
+        });
+        test("includes path in error messages", () => {
+            const errors = validateServerEntry({
+                url: "not-a-url",
+            });
+            expect(errors.length).toBeGreaterThan(0);
+            // Should contain error about invalid URL
+            expect(errors.some((e) => e.includes("url"))).toBe(true);
+        });
+    });
+    describe("isHttpSseConfig", () => {
+        test("returns true for config with url", () => {
+            const config = { url: "http://localhost:3000" };
+            expect(isHttpSseConfig(config)).toBe(true);
+        });
+        test("returns true for config with transport http", () => {
+            const config = {
+                transport: "http",
+                url: "http://localhost:3000",
+            };
+            expect(isHttpSseConfig(config)).toBe(true);
+        });
+        test("returns true for config with transport sse", () => {
+            const config = {
+                transport: "sse",
+                url: "http://localhost:3000",
+            };
+            expect(isHttpSseConfig(config)).toBe(true);
+        });
+        test("returns false for stdio config without url", () => {
+            const config = { command: "python3", args: [], env: {} };
+            expect(isHttpSseConfig(config)).toBe(false);
+        });
+    });
+    describe("isStdioConfig", () => {
+        test("returns true for config with command (no url)", () => {
+            const config = { command: "python3", args: [], env: {} };
+            expect(isStdioConfig(config)).toBe(true);
+        });
+        test("returns false for HTTP config", () => {
+            const config = { url: "http://localhost:3000" };
+            expect(isStdioConfig(config)).toBe(false);
+        });
+        test("returns false for config with both url and command", () => {
+            // If both are present, isStdioConfig returns false because url takes precedence
+            const config = {
+                url: "http://localhost:3000",
+                command: "python3",
+                args: [],
+                env: {},
+            };
+            expect(isStdioConfig(config)).toBe(false);
+        });
+    });
+});

package/build/lib/assessment-runner/assessment-executor.js

@@ -154,6 +154,10 @@ export async function runFullAssessment(options) {
         return {};
     }
     const config = buildConfig(options);
+    // Set serverUrl for conformance tests when HTTP/SSE transport is used
+    if (serverConfig.url && !config.serverUrl) {
+        config.serverUrl = serverConfig.url;
+    }
     // Emit modules_configured event for consumer progress tracking
     if (config.assessmentCategories) {
         const enabled = [];