@bryan-thompson/inspector-assessment-cli 1.32.2 → 1.32.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/__tests__/assessment-runner/assessment-executor.test.js +5 -0
- package/build/__tests__/assessment-runner/server-config.test.js +8 -5
- package/build/__tests__/jsonl-events.test.js +195 -1
- package/build/__tests__/lib/server-configSchemas.test.js +314 -0
- package/build/__tests__/lib/zodErrorFormatter.test.js +721 -0
- package/build/__tests__/security/security-pattern-count.test.js +245 -0
- package/build/assess-security.js +11 -77
- package/build/lib/assessment-runner/__tests__/server-config.test.js +116 -0
- package/build/lib/assessment-runner/assessment-executor.js +18 -1
- package/build/lib/assessment-runner/config-builder.js +10 -0
- package/build/lib/assessment-runner/server-config.js +43 -35
- package/build/lib/assessment-runner/server-configSchemas.js +4 -1
- package/build/lib/jsonl-events.js +59 -0
- package/build/lib/zodErrorFormatter.js +31 -0
- package/package.json +1 -1
|
@@ -0,0 +1,721 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Zod Error Formatter Test Suite
|
|
3
|
+
*
|
|
4
|
+
* Tests for formatZodError utility to ensure helpful error messages.
|
|
5
|
+
* Addresses QA requirement: verify Zod error messages are helpful (not just generic "Invalid").
|
|
6
|
+
*/
|
|
7
|
+
import { describe, it, expect } from "@jest/globals";
|
|
8
|
+
import { z } from "zod";
|
|
9
|
+
import { formatZodError, formatZodIssue, formatZodErrorIndented, zodErrorToArray, formatUserFriendlyError, formatZodErrorForJson, } from "../../lib/zodErrorFormatter.js";
|
|
10
|
+
describe("zodErrorFormatter", () => {
|
|
11
|
+
describe("formatZodIssue", () => {
|
|
12
|
+
it("should format issue with path", () => {
|
|
13
|
+
const issue = {
|
|
14
|
+
code: "invalid_type",
|
|
15
|
+
expected: "string",
|
|
16
|
+
received: "number",
|
|
17
|
+
path: ["config", "url"],
|
|
18
|
+
message: "Expected string, received number",
|
|
19
|
+
};
|
|
20
|
+
const result = formatZodIssue(issue);
|
|
21
|
+
expect(result).toBe("config.url: Expected string, received number");
|
|
22
|
+
});
|
|
23
|
+
it("should format issue without path", () => {
|
|
24
|
+
const issue = {
|
|
25
|
+
code: "invalid_type",
|
|
26
|
+
expected: "string",
|
|
27
|
+
received: "number",
|
|
28
|
+
path: [],
|
|
29
|
+
message: "Expected string, received number",
|
|
30
|
+
};
|
|
31
|
+
const result = formatZodIssue(issue);
|
|
32
|
+
expect(result).toBe("Expected string, received number");
|
|
33
|
+
});
|
|
34
|
+
});
|
|
35
|
+
describe("formatZodError - union validation", () => {
|
|
36
|
+
it("should extract specific error messages from union validation (HTTP transport missing url)", () => {
|
|
37
|
+
// Schema for HTTP/SSE transport
|
|
38
|
+
const HttpSseSchema = z.object({
|
|
39
|
+
transport: z.enum(["http", "sse"]).optional(),
|
|
40
|
+
url: z
|
|
41
|
+
.string()
|
|
42
|
+
.min(1, "'url' is required for HTTP/SSE transport")
|
|
43
|
+
.url("url must be a valid URL"),
|
|
44
|
+
});
|
|
45
|
+
// Schema for stdio transport
|
|
46
|
+
const StdioSchema = z.object({
|
|
47
|
+
transport: z.literal("stdio").optional(),
|
|
48
|
+
command: z.string().min(1, "command is required for stdio transport"),
|
|
49
|
+
args: z.array(z.string()).optional(),
|
|
50
|
+
});
|
|
51
|
+
// Union schema (like ServerEntrySchema)
|
|
52
|
+
const ServerEntrySchema = z.union([HttpSseSchema, StdioSchema]);
|
|
53
|
+
// Test case: HTTP transport without url (should fail validation)
|
|
54
|
+
const invalidConfig = { transport: "http" }; // Missing url
|
|
55
|
+
const result = ServerEntrySchema.safeParse(invalidConfig);
|
|
56
|
+
expect(result.success).toBe(false);
|
|
57
|
+
if (!result.success) {
|
|
58
|
+
const formatted = formatZodError(result.error);
|
|
59
|
+
// Should extract the specific error message, not just "Invalid"
|
|
60
|
+
expect(formatted).toContain("url");
|
|
61
|
+
expect(formatted).not.toBe("Invalid input");
|
|
62
|
+
// Verify it's a helpful message
|
|
63
|
+
expect(formatted.includes("required") ||
|
|
64
|
+
formatted.includes("url") ||
|
|
65
|
+
formatted.includes("Expected")).toBe(true);
|
|
66
|
+
}
|
|
67
|
+
});
|
|
68
|
+
it("should extract specific error messages from union validation (SSE transport missing url)", () => {
|
|
69
|
+
const HttpSseSchema = z.object({
|
|
70
|
+
transport: z.enum(["http", "sse"]).optional(),
|
|
71
|
+
url: z
|
|
72
|
+
.string()
|
|
73
|
+
.min(1, "'url' is required for HTTP/SSE transport")
|
|
74
|
+
.url("url must be a valid URL"),
|
|
75
|
+
});
|
|
76
|
+
const StdioSchema = z.object({
|
|
77
|
+
transport: z.literal("stdio").optional(),
|
|
78
|
+
command: z.string().min(1, "command is required for stdio transport"),
|
|
79
|
+
args: z.array(z.string()).optional(),
|
|
80
|
+
});
|
|
81
|
+
const ServerEntrySchema = z.union([HttpSseSchema, StdioSchema]);
|
|
82
|
+
// Test case: SSE transport without url (should fail validation)
|
|
83
|
+
const invalidConfig = { transport: "sse" }; // Missing url
|
|
84
|
+
const result = ServerEntrySchema.safeParse(invalidConfig);
|
|
85
|
+
expect(result.success).toBe(false);
|
|
86
|
+
if (!result.success) {
|
|
87
|
+
const formatted = formatZodError(result.error);
|
|
88
|
+
// Should extract the specific error message
|
|
89
|
+
expect(formatted).toContain("url");
|
|
90
|
+
expect(formatted).not.toBe("Invalid input");
|
|
91
|
+
}
|
|
92
|
+
});
|
|
93
|
+
it("should handle stdio transport missing command", () => {
|
|
94
|
+
const HttpSseSchema = z.object({
|
|
95
|
+
transport: z.enum(["http", "sse"]).optional(),
|
|
96
|
+
url: z.string().min(1),
|
|
97
|
+
});
|
|
98
|
+
const StdioSchema = z.object({
|
|
99
|
+
transport: z.literal("stdio").optional(),
|
|
100
|
+
command: z.string().min(1, "command is required for stdio transport"),
|
|
101
|
+
args: z.array(z.string()).optional(),
|
|
102
|
+
});
|
|
103
|
+
const ServerEntrySchema = z.union([HttpSseSchema, StdioSchema]);
|
|
104
|
+
// Test case: Config with command field but empty (should fail validation)
|
|
105
|
+
const invalidConfig = { command: "" }; // Empty command
|
|
106
|
+
const result = ServerEntrySchema.safeParse(invalidConfig);
|
|
107
|
+
expect(result.success).toBe(false);
|
|
108
|
+
if (!result.success) {
|
|
109
|
+
const formatted = formatZodError(result.error);
|
|
110
|
+
// Should extract the specific error message about command
|
|
111
|
+
expect(formatted).toContain("command");
|
|
112
|
+
expect(formatted.toLowerCase()).toContain("required");
|
|
113
|
+
}
|
|
114
|
+
});
|
|
115
|
+
it("should provide helpful message for invalid URL format", () => {
|
|
116
|
+
const HttpSseSchema = z.object({
|
|
117
|
+
transport: z.enum(["http", "sse"]).optional(),
|
|
118
|
+
url: z
|
|
119
|
+
.string()
|
|
120
|
+
.min(1, "'url' is required for HTTP/SSE transport")
|
|
121
|
+
.url("url must be a valid URL"),
|
|
122
|
+
});
|
|
123
|
+
const StdioSchema = z.object({
|
|
124
|
+
transport: z.literal("stdio").optional(),
|
|
125
|
+
command: z.string().min(1, "command is required for stdio transport"),
|
|
126
|
+
});
|
|
127
|
+
const ServerEntrySchema = z.union([HttpSseSchema, StdioSchema]);
|
|
128
|
+
// Test case: Invalid URL format
|
|
129
|
+
const invalidConfig = { url: "not-a-valid-url" };
|
|
130
|
+
const result = ServerEntrySchema.safeParse(invalidConfig);
|
|
131
|
+
expect(result.success).toBe(false);
|
|
132
|
+
if (!result.success) {
|
|
133
|
+
const formatted = formatZodError(result.error);
|
|
134
|
+
// Should provide helpful message about URL format
|
|
135
|
+
expect(formatted.toLowerCase()).toContain("url");
|
|
136
|
+
expect(formatted.toLowerCase()).toContain("valid");
|
|
137
|
+
}
|
|
138
|
+
});
|
|
139
|
+
it("should handle completely empty config object", () => {
|
|
140
|
+
const HttpSseSchema = z.object({
|
|
141
|
+
transport: z.enum(["http", "sse"]).optional(),
|
|
142
|
+
url: z.string().min(1, "'url' is required for HTTP/SSE transport"),
|
|
143
|
+
});
|
|
144
|
+
const StdioSchema = z.object({
|
|
145
|
+
transport: z.literal("stdio").optional(),
|
|
146
|
+
command: z.string().min(1, "command is required for stdio transport"),
|
|
147
|
+
});
|
|
148
|
+
const ServerEntrySchema = z.union([HttpSseSchema, StdioSchema]);
|
|
149
|
+
// Test case: Completely empty config
|
|
150
|
+
const invalidConfig = {};
|
|
151
|
+
const result = ServerEntrySchema.safeParse(invalidConfig);
|
|
152
|
+
expect(result.success).toBe(false);
|
|
153
|
+
if (!result.success) {
|
|
154
|
+
const formatted = formatZodError(result.error);
|
|
155
|
+
// Should provide some helpful message (not just "Invalid input")
|
|
156
|
+
expect(formatted).toBeTruthy();
|
|
157
|
+
expect(formatted.length).toBeGreaterThan(10); // More than just "Invalid"
|
|
158
|
+
}
|
|
159
|
+
});
|
|
160
|
+
});
|
|
161
|
+
describe("formatZodError - non-union errors", () => {
|
|
162
|
+
it("should format simple validation error", () => {
|
|
163
|
+
const schema = z.object({
|
|
164
|
+
name: z.string(),
|
|
165
|
+
age: z.number().positive(),
|
|
166
|
+
});
|
|
167
|
+
const result = schema.safeParse({ name: "John", age: -5 });
|
|
168
|
+
expect(result.success).toBe(false);
|
|
169
|
+
if (!result.success) {
|
|
170
|
+
const formatted = formatZodError(result.error);
|
|
171
|
+
expect(formatted).toContain("age");
|
|
172
|
+
expect(formatted.toLowerCase()).toContain("greater than");
|
|
173
|
+
}
|
|
174
|
+
});
|
|
175
|
+
it("should format multiple validation errors", () => {
|
|
176
|
+
const schema = z.object({
|
|
177
|
+
name: z.string().min(3),
|
|
178
|
+
email: z.string().email(),
|
|
179
|
+
});
|
|
180
|
+
const result = schema.safeParse({ name: "Jo", email: "invalid" });
|
|
181
|
+
expect(result.success).toBe(false);
|
|
182
|
+
if (!result.success) {
|
|
183
|
+
const formatted = formatZodError(result.error);
|
|
184
|
+
expect(formatted).toContain("name");
|
|
185
|
+
expect(formatted).toContain("email");
|
|
186
|
+
}
|
|
187
|
+
});
|
|
188
|
+
});
|
|
189
|
+
describe("formatZodErrorIndented", () => {
|
|
190
|
+
it("should format errors with indentation", () => {
|
|
191
|
+
const schema = z.object({
|
|
192
|
+
name: z.string(),
|
|
193
|
+
});
|
|
194
|
+
const result = schema.safeParse({ name: 123 });
|
|
195
|
+
expect(result.success).toBe(false);
|
|
196
|
+
if (!result.success) {
|
|
197
|
+
const formatted = formatZodErrorIndented(result.error);
|
|
198
|
+
expect(formatted).toMatch(/^\s+/); // Starts with whitespace
|
|
199
|
+
expect(formatted).toContain("name");
|
|
200
|
+
}
|
|
201
|
+
});
|
|
202
|
+
it("should use custom indentation", () => {
|
|
203
|
+
const schema = z.object({
|
|
204
|
+
name: z.string(),
|
|
205
|
+
});
|
|
206
|
+
const result = schema.safeParse({ name: 123 });
|
|
207
|
+
expect(result.success).toBe(false);
|
|
208
|
+
if (!result.success) {
|
|
209
|
+
const formatted = formatZodErrorIndented(result.error, " "); // 4 spaces
|
|
210
|
+
expect(formatted).toMatch(/^ /); // Starts with 4 spaces
|
|
211
|
+
}
|
|
212
|
+
});
|
|
213
|
+
});
|
|
214
|
+
describe("zodErrorToArray", () => {
|
|
215
|
+
it("should convert ZodError to array of strings", () => {
|
|
216
|
+
const schema = z.object({
|
|
217
|
+
name: z.string(),
|
|
218
|
+
age: z.number(),
|
|
219
|
+
});
|
|
220
|
+
const result = schema.safeParse({ name: 123, age: "invalid" });
|
|
221
|
+
expect(result.success).toBe(false);
|
|
222
|
+
if (!result.success) {
|
|
223
|
+
const errors = zodErrorToArray(result.error);
|
|
224
|
+
expect(Array.isArray(errors)).toBe(true);
|
|
225
|
+
expect(errors.length).toBeGreaterThan(0);
|
|
226
|
+
expect(errors.some((e) => e.includes("name"))).toBe(true);
|
|
227
|
+
expect(errors.some((e) => e.includes("age"))).toBe(true);
|
|
228
|
+
}
|
|
229
|
+
});
|
|
230
|
+
});
|
|
231
|
+
describe("formatUserFriendlyError", () => {
|
|
232
|
+
it("should format single error", () => {
|
|
233
|
+
const schema = z.object({
|
|
234
|
+
name: z.string(),
|
|
235
|
+
});
|
|
236
|
+
const result = schema.safeParse({ name: 123 });
|
|
237
|
+
expect(result.success).toBe(false);
|
|
238
|
+
if (!result.success) {
|
|
239
|
+
const formatted = formatUserFriendlyError(result.error);
|
|
240
|
+
expect(formatted).toContain("name");
|
|
241
|
+
expect(formatted).not.toContain("Multiple validation errors");
|
|
242
|
+
}
|
|
243
|
+
});
|
|
244
|
+
it("should format multiple errors with list", () => {
|
|
245
|
+
const schema = z.object({
|
|
246
|
+
name: z.string(),
|
|
247
|
+
age: z.number(),
|
|
248
|
+
});
|
|
249
|
+
const result = schema.safeParse({ name: 123, age: "invalid" });
|
|
250
|
+
expect(result.success).toBe(false);
|
|
251
|
+
if (!result.success) {
|
|
252
|
+
const formatted = formatUserFriendlyError(result.error);
|
|
253
|
+
expect(formatted).toContain("Multiple validation errors");
|
|
254
|
+
expect(formatted).toContain("-"); // List bullet
|
|
255
|
+
}
|
|
256
|
+
});
|
|
257
|
+
it("should use field labels when provided", () => {
|
|
258
|
+
const schema = z.object({
|
|
259
|
+
serverUrl: z.string().url(),
|
|
260
|
+
});
|
|
261
|
+
const result = schema.safeParse({ serverUrl: "invalid" });
|
|
262
|
+
expect(result.success).toBe(false);
|
|
263
|
+
if (!result.success) {
|
|
264
|
+
const formatted = formatUserFriendlyError(result.error, {
|
|
265
|
+
serverUrl: "Server URL",
|
|
266
|
+
});
|
|
267
|
+
expect(formatted).toContain("Server URL");
|
|
268
|
+
}
|
|
269
|
+
});
|
|
270
|
+
});
|
|
271
|
+
describe("formatZodErrorForJson", () => {
|
|
272
|
+
it("should format error for JSON output", () => {
|
|
273
|
+
const schema = z.object({
|
|
274
|
+
name: z.string(),
|
|
275
|
+
});
|
|
276
|
+
const result = schema.safeParse({ name: 123 });
|
|
277
|
+
expect(result.success).toBe(false);
|
|
278
|
+
if (!result.success) {
|
|
279
|
+
const formatted = formatZodErrorForJson(result.error);
|
|
280
|
+
expect(formatted).toHaveProperty("message");
|
|
281
|
+
expect(formatted).toHaveProperty("errors");
|
|
282
|
+
expect(Array.isArray(formatted.errors)).toBe(true);
|
|
283
|
+
expect(formatted.errors[0]).toHaveProperty("path");
|
|
284
|
+
expect(formatted.errors[0]).toHaveProperty("message");
|
|
285
|
+
expect(formatted.errors[0]).toHaveProperty("code");
|
|
286
|
+
}
|
|
287
|
+
});
|
|
288
|
+
it("should include error details", () => {
|
|
289
|
+
const schema = z.object({
|
|
290
|
+
config: z.object({
|
|
291
|
+
port: z.number(),
|
|
292
|
+
}),
|
|
293
|
+
});
|
|
294
|
+
const result = schema.safeParse({ config: { port: "3000" } });
|
|
295
|
+
expect(result.success).toBe(false);
|
|
296
|
+
if (!result.success) {
|
|
297
|
+
const formatted = formatZodErrorForJson(result.error);
|
|
298
|
+
expect(formatted.errors[0].path).toEqual(["config", "port"]);
|
|
299
|
+
expect(formatted.errors[0].code).toBe("invalid_type");
|
|
300
|
+
}
|
|
301
|
+
});
|
|
302
|
+
});
|
|
303
|
+
describe("error message quality - regression tests", () => {
|
|
304
|
+
it("should never return just 'Invalid input' for server config errors", () => {
|
|
305
|
+
// This is the key test: ensure we never get generic "Invalid input"
|
|
306
|
+
// when validating server configurations
|
|
307
|
+
const HttpSseSchema = z.object({
|
|
308
|
+
transport: z.enum(["http", "sse"]).optional(),
|
|
309
|
+
url: z
|
|
310
|
+
.string()
|
|
311
|
+
.min(1, "'url' is required for HTTP/SSE transport")
|
|
312
|
+
.url("url must be a valid URL"),
|
|
313
|
+
});
|
|
314
|
+
const StdioSchema = z.object({
|
|
315
|
+
transport: z.literal("stdio").optional(),
|
|
316
|
+
command: z.string().min(1, "command is required for stdio transport"),
|
|
317
|
+
});
|
|
318
|
+
const ServerEntrySchema = z.union([HttpSseSchema, StdioSchema]);
|
|
319
|
+
// Test various invalid configs
|
|
320
|
+
const invalidConfigs = [
|
|
321
|
+
{ transport: "http" }, // Missing url
|
|
322
|
+
{ transport: "sse" }, // Missing url
|
|
323
|
+
{ url: "invalid-url" }, // Invalid URL format
|
|
324
|
+
{ command: "" }, // Empty command
|
|
325
|
+
{}, // Empty config
|
|
326
|
+
];
|
|
327
|
+
for (const config of invalidConfigs) {
|
|
328
|
+
const result = ServerEntrySchema.safeParse(config);
|
|
329
|
+
expect(result.success).toBe(false);
|
|
330
|
+
if (!result.success) {
|
|
331
|
+
const formatted = formatZodError(result.error);
|
|
332
|
+
// Key assertion: formatted error should NOT be just "Invalid input"
|
|
333
|
+
expect(formatted).not.toBe("Invalid input");
|
|
334
|
+
// Should contain at least one helpful keyword
|
|
335
|
+
const hasHelpfulKeyword = formatted.toLowerCase().includes("url") ||
|
|
336
|
+
formatted.toLowerCase().includes("command") ||
|
|
337
|
+
formatted.toLowerCase().includes("required") ||
|
|
338
|
+
formatted.toLowerCase().includes("expected") ||
|
|
339
|
+
formatted.toLowerCase().includes("valid");
|
|
340
|
+
expect(hasHelpfulKeyword).toBe(true);
|
|
341
|
+
// Should be reasonably descriptive (more than 10 chars)
|
|
342
|
+
expect(formatted.length).toBeGreaterThan(10);
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
});
|
|
346
|
+
});
|
|
347
|
+
describe("union error multi-error handling", () => {
|
|
348
|
+
it("should show ALL relevant errors from union validation (not just first)", () => {
|
|
349
|
+
// Stage 3 fix: formatZodError returns up to 3 unique errors for clarity
|
|
350
|
+
// Test that union errors show multiple validation failures
|
|
351
|
+
const HttpSseSchema = z.object({
|
|
352
|
+
transport: z.enum(["http", "sse"]).optional(),
|
|
353
|
+
url: z
|
|
354
|
+
.string()
|
|
355
|
+
.min(1, "'url' is required for HTTP/SSE transport")
|
|
356
|
+
.url("url must be a valid URL"),
|
|
357
|
+
});
|
|
358
|
+
const StdioSchema = z.object({
|
|
359
|
+
transport: z.literal("stdio").optional(),
|
|
360
|
+
command: z.string().min(1, "command is required for stdio transport"),
|
|
361
|
+
args: z.array(z.string()).optional(),
|
|
362
|
+
});
|
|
363
|
+
const ServerEntrySchema = z.union([HttpSseSchema, StdioSchema]);
|
|
364
|
+
// Test case: Empty config fails both union branches
|
|
365
|
+
const invalidConfig = {};
|
|
366
|
+
const result = ServerEntrySchema.safeParse(invalidConfig);
|
|
367
|
+
expect(result.success).toBe(false);
|
|
368
|
+
if (!result.success) {
|
|
369
|
+
const formatted = formatZodError(result.error);
|
|
370
|
+
// Should contain errors from both union branches
|
|
371
|
+
// HTTP/SSE branch: url is required
|
|
372
|
+
// stdio branch: command is required
|
|
373
|
+
expect(formatted).toContain("url");
|
|
374
|
+
expect(formatted).toContain("command");
|
|
375
|
+
// Should be formatted as multiple lines (multiple errors)
|
|
376
|
+
const lines = formatted.split("\n").filter((line) => line.trim());
|
|
377
|
+
expect(lines.length).toBeGreaterThan(1);
|
|
378
|
+
}
|
|
379
|
+
});
|
|
380
|
+
it("should deduplicate identical errors from union branches", () => {
|
|
381
|
+
// If multiple union branches have the same error, show it once
|
|
382
|
+
const Schema1 = z.object({
|
|
383
|
+
field: z.string().min(5, "field must be at least 5 characters"),
|
|
384
|
+
});
|
|
385
|
+
const Schema2 = z.object({
|
|
386
|
+
field: z.string().min(5, "field must be at least 5 characters"),
|
|
387
|
+
extra: z.string().optional(),
|
|
388
|
+
});
|
|
389
|
+
const UnionSchema = z.union([Schema1, Schema2]);
|
|
390
|
+
const invalidInput = { field: "abc" }; // Too short
|
|
391
|
+
const result = UnionSchema.safeParse(invalidInput);
|
|
392
|
+
expect(result.success).toBe(false);
|
|
393
|
+
if (!result.success) {
|
|
394
|
+
const formatted = formatZodError(result.error);
|
|
395
|
+
// Should not duplicate the same error message
|
|
396
|
+
const lines = formatted.split("\n").filter((line) => line.trim());
|
|
397
|
+
const uniqueLines = new Set(lines);
|
|
398
|
+
expect(lines.length).toBe(uniqueLines.size);
|
|
399
|
+
}
|
|
400
|
+
});
|
|
401
|
+
it("should return all unique errors from union branches", () => {
|
|
402
|
+
// Stage 3 fix: Return all unique errors (deduplication)
|
|
403
|
+
const Schema1 = z.object({
|
|
404
|
+
a: z.string().min(1, "a is required"),
|
|
405
|
+
b: z.string().min(1, "b is required"),
|
|
406
|
+
c: z.string().min(1, "c is required"),
|
|
407
|
+
});
|
|
408
|
+
const Schema2 = z.object({
|
|
409
|
+
x: z.string().min(1, "x is required"),
|
|
410
|
+
y: z.string().min(1, "y is required"),
|
|
411
|
+
z: z.string().min(1, "z is required"),
|
|
412
|
+
});
|
|
413
|
+
const UnionSchema = z.union([Schema1, Schema2]);
|
|
414
|
+
const invalidInput = {}; // Empty, fails all validations
|
|
415
|
+
const result = UnionSchema.safeParse(invalidInput);
|
|
416
|
+
expect(result.success).toBe(false);
|
|
417
|
+
if (!result.success) {
|
|
418
|
+
const formatted = formatZodError(result.error);
|
|
419
|
+
const lines = formatted.split("\n").filter((line) => line.trim());
|
|
420
|
+
// Should have errors from both branches (6 total)
|
|
421
|
+
expect(lines.length).toBeGreaterThan(0);
|
|
422
|
+
// All lines should be non-empty and descriptive
|
|
423
|
+
for (const line of lines) {
|
|
424
|
+
expect(line.length).toBeGreaterThan(5);
|
|
425
|
+
}
|
|
426
|
+
// Verify we have unique errors (no duplicates)
|
|
427
|
+
const uniqueLines = new Set(lines);
|
|
428
|
+
expect(lines.length).toBe(uniqueLines.size);
|
|
429
|
+
}
|
|
430
|
+
});
|
|
431
|
+
it("should prioritize specific errors over generic ones", () => {
|
|
432
|
+
// When union has both specific and generic errors, prefer specific
|
|
433
|
+
const HttpSseSchema = z.object({
|
|
434
|
+
transport: z.enum(["http", "sse"]).optional(),
|
|
435
|
+
url: z
|
|
436
|
+
.string()
|
|
437
|
+
.min(1, "'url' is required for HTTP/SSE transport")
|
|
438
|
+
.url("url must be a valid URL"),
|
|
439
|
+
});
|
|
440
|
+
const StdioSchema = z.object({
|
|
441
|
+
transport: z.literal("stdio").optional(),
|
|
442
|
+
command: z.string().min(1, "command is required for stdio transport"),
|
|
443
|
+
});
|
|
444
|
+
const ServerEntrySchema = z.union([HttpSseSchema, StdioSchema]);
|
|
445
|
+
const invalidConfig = { transport: "http" }; // Missing url
|
|
446
|
+
const result = ServerEntrySchema.safeParse(invalidConfig);
|
|
447
|
+
expect(result.success).toBe(false);
|
|
448
|
+
if (!result.success) {
|
|
449
|
+
const formatted = formatZodError(result.error);
|
|
450
|
+
// Should show specific error about missing url
|
|
451
|
+
expect(formatted.toLowerCase()).toContain("url");
|
|
452
|
+
expect(formatted.toLowerCase()).toContain("required");
|
|
453
|
+
// Should not show generic "Required" or "Invalid"
|
|
454
|
+
expect(formatted).not.toBe("Required");
|
|
455
|
+
expect(formatted).not.toBe("Invalid input");
|
|
456
|
+
}
|
|
457
|
+
});
|
|
458
|
+
it("should handle complex union with nested objects", () => {
|
|
459
|
+
const ConfigA = z.object({
|
|
460
|
+
type: z.literal("a"),
|
|
461
|
+
nested: z.object({
|
|
462
|
+
field1: z.string().min(1, "field1 is required"),
|
|
463
|
+
field2: z.number().positive("field2 must be positive"),
|
|
464
|
+
}),
|
|
465
|
+
});
|
|
466
|
+
const ConfigB = z.object({
|
|
467
|
+
type: z.literal("b"),
|
|
468
|
+
nested: z.object({
|
|
469
|
+
field3: z.string().email("field3 must be a valid email"),
|
|
470
|
+
field4: z.boolean(),
|
|
471
|
+
}),
|
|
472
|
+
});
|
|
473
|
+
const UnionSchema = z.union([ConfigA, ConfigB]);
|
|
474
|
+
const invalidInput = {
|
|
475
|
+
type: "a",
|
|
476
|
+
nested: {
|
|
477
|
+
field1: "",
|
|
478
|
+
field2: -5,
|
|
479
|
+
},
|
|
480
|
+
};
|
|
481
|
+
const result = UnionSchema.safeParse(invalidInput);
|
|
482
|
+
expect(result.success).toBe(false);
|
|
483
|
+
if (!result.success) {
|
|
484
|
+
const formatted = formatZodError(result.error);
|
|
485
|
+
// Should show nested path errors
|
|
486
|
+
expect(formatted).toContain("nested");
|
|
487
|
+
// Should be helpful and descriptive
|
|
488
|
+
const hasHelpfulInfo = formatted.includes("field1") ||
|
|
489
|
+
formatted.includes("field2") ||
|
|
490
|
+
formatted.includes("required") ||
|
|
491
|
+
formatted.includes("positive");
|
|
492
|
+
expect(hasHelpfulInfo).toBe(true);
|
|
493
|
+
}
|
|
494
|
+
});
|
|
495
|
+
it("should format multiple union errors with proper line breaks", () => {
|
|
496
|
+
const Schema1 = z.object({
|
|
497
|
+
url: z.string().url("url must be a valid URL"),
|
|
498
|
+
});
|
|
499
|
+
const Schema2 = z.object({
|
|
500
|
+
command: z.string().min(1, "command is required"),
|
|
501
|
+
});
|
|
502
|
+
const UnionSchema = z.union([Schema1, Schema2]);
|
|
503
|
+
const invalidInput = { url: "not-a-url" }; // Invalid URL
|
|
504
|
+
const result = UnionSchema.safeParse(invalidInput);
|
|
505
|
+
expect(result.success).toBe(false);
|
|
506
|
+
if (!result.success) {
|
|
507
|
+
const formatted = formatZodError(result.error);
|
|
508
|
+
// Should have proper formatting with line breaks if multiple errors
|
|
509
|
+
expect(formatted).toBeTruthy();
|
|
510
|
+
// Each error should be on its own line or separated
|
|
511
|
+
if (formatted.includes("\n")) {
|
|
512
|
+
const lines = formatted.split("\n").filter((line) => line.trim());
|
|
513
|
+
expect(lines.length).toBeGreaterThan(0);
|
|
514
|
+
// Each line should be descriptive
|
|
515
|
+
for (const line of lines) {
|
|
516
|
+
expect(line.length).toBeGreaterThan(5);
|
|
517
|
+
}
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
});
|
|
521
|
+
/**
|
|
522
|
+
* T-REQ-001: Union validation extracts specific errors from each branch
|
|
523
|
+
* Test that formatZodError shows specific validation messages from the
|
|
524
|
+
* union branch that Zod tries to match based on input shape.
|
|
525
|
+
*/
|
|
526
|
+
it("T-REQ-001: should show specific error from matched union branch", () => {
|
|
527
|
+
const HttpSseSchema = z.object({
|
|
528
|
+
transport: z.enum(["http", "sse"]).optional(),
|
|
529
|
+
url: z
|
|
530
|
+
.string()
|
|
531
|
+
.min(1, "'url' is required for HTTP/SSE transport")
|
|
532
|
+
.url("url must be a valid URL"),
|
|
533
|
+
});
|
|
534
|
+
const StdioSchema = z.object({
|
|
535
|
+
transport: z.literal("stdio").optional(),
|
|
536
|
+
command: z.string().min(1, "command is required for stdio transport"),
|
|
537
|
+
args: z.array(z.string()).optional(),
|
|
538
|
+
});
|
|
539
|
+
const ServerEntrySchema = z.union([HttpSseSchema, StdioSchema]);
|
|
540
|
+
// Test case 1: Invalid URL - matches HTTP schema branch
|
|
541
|
+
const invalidUrlConfig = { url: "not-a-valid-url" };
|
|
542
|
+
const urlResult = ServerEntrySchema.safeParse(invalidUrlConfig);
|
|
543
|
+
expect(urlResult.success).toBe(false);
|
|
544
|
+
if (!urlResult.success) {
|
|
545
|
+
const formatted = formatZodError(urlResult.error);
|
|
546
|
+
expect(formatted.toLowerCase()).toContain("url");
|
|
547
|
+
expect(formatted.toLowerCase()).toContain("valid");
|
|
548
|
+
expect(formatted).not.toBe("Invalid input");
|
|
549
|
+
}
|
|
550
|
+
// Test case 2: Empty command - matches stdio schema branch
|
|
551
|
+
const emptyCommandConfig = { command: "" };
|
|
552
|
+
const commandResult = ServerEntrySchema.safeParse(emptyCommandConfig);
|
|
553
|
+
expect(commandResult.success).toBe(false);
|
|
554
|
+
if (!commandResult.success) {
|
|
555
|
+
const formatted = formatZodError(commandResult.error);
|
|
556
|
+
expect(formatted.toLowerCase()).toContain("command");
|
|
557
|
+
expect(formatted).not.toBe("Invalid input");
|
|
558
|
+
}
|
|
559
|
+
});
|
|
560
|
+
/**
|
|
561
|
+
* T-REQ-002: Verify detailed errors are extracted (not generic "Invalid input")
|
|
562
|
+
* Ensure formatZodError extracts specific validation messages from union branches.
|
|
563
|
+
*/
|
|
564
|
+
it("T-REQ-002: should return detailed errors from union validation (not generic messages)", () => {
|
|
565
|
+
const HttpSseSchema = z.object({
|
|
566
|
+
transport: z.enum(["http", "sse"]).optional(),
|
|
567
|
+
url: z
|
|
568
|
+
.string()
|
|
569
|
+
.min(1, "'url' is required for HTTP/SSE transport")
|
|
570
|
+
.url("url must be a valid URL"),
|
|
571
|
+
});
|
|
572
|
+
const StdioSchema = z.object({
|
|
573
|
+
transport: z.literal("stdio").optional(),
|
|
574
|
+
command: z.string().min(1, "command is required for stdio transport"),
|
|
575
|
+
args: z.array(z.string()).optional(),
|
|
576
|
+
});
|
|
577
|
+
const ServerEntrySchema = z.union([HttpSseSchema, StdioSchema]);
|
|
578
|
+
// Test cases with specific validation failures
|
|
579
|
+
const invalidConfigs = [
|
|
580
|
+
{ url: "not-a-url" }, // Invalid URL - shows URL error
|
|
581
|
+
{ command: "" }, // Empty command - shows command error
|
|
582
|
+
];
|
|
583
|
+
for (const config of invalidConfigs) {
|
|
584
|
+
const result = ServerEntrySchema.safeParse(config);
|
|
585
|
+
expect(result.success).toBe(false);
|
|
586
|
+
if (!result.success) {
|
|
587
|
+
const formatted = formatZodError(result.error);
|
|
588
|
+
const errors = zodErrorToArray(result.error);
|
|
589
|
+
// Should return at least one error
|
|
590
|
+
expect(errors.length).toBeGreaterThan(0);
|
|
591
|
+
// Formatted string should contain errors
|
|
592
|
+
const lines = formatted.split("\n").filter((line) => line.trim());
|
|
593
|
+
expect(lines.length).toBeGreaterThan(0);
|
|
594
|
+
// Should not lose error details in formatting
|
|
595
|
+
const formattedLower = formatted.toLowerCase();
|
|
596
|
+
const hasUrlOrCommand = formattedLower.includes("url") ||
|
|
597
|
+
formattedLower.includes("command");
|
|
598
|
+
expect(hasUrlOrCommand).toBe(true);
|
|
599
|
+
}
|
|
600
|
+
}
|
|
601
|
+
});
|
|
602
|
+
});
|
|
603
|
+
/**
|
|
604
|
+
* T-REQ-003: End-to-end test: invalid config file -> user-friendly error message
|
|
605
|
+
* Full path: load invalid config -> Zod validation -> formatZodError -> helpful message
|
|
606
|
+
*/
|
|
607
|
+
describe("end-to-end config validation workflow", () => {
|
|
608
|
+
it("T-REQ-003: should provide user-friendly error messages for invalid config files", () => {
|
|
609
|
+
// Import the actual schema used for server config validation
|
|
610
|
+
const HttpSseSchema = z.object({
|
|
611
|
+
transport: z.enum(["http", "sse"]).optional(),
|
|
612
|
+
url: z
|
|
613
|
+
.string()
|
|
614
|
+
.min(1, "'url' is required for HTTP/SSE transport")
|
|
615
|
+
.url("url must be a valid URL"),
|
|
616
|
+
});
|
|
617
|
+
const StdioSchema = z.object({
|
|
618
|
+
transport: z.literal("stdio").optional(),
|
|
619
|
+
command: z.string().min(1, "command is required for stdio transport"),
|
|
620
|
+
args: z.array(z.string()).optional(),
|
|
621
|
+
env: z.record(z.string()).optional(),
|
|
622
|
+
});
|
|
623
|
+
const ServerEntrySchema = z.union([HttpSseSchema, StdioSchema]);
|
|
624
|
+
// Simulate various invalid config file scenarios
|
|
625
|
+
// Note: Zod union validation matches based on input shape, so expectedKeywords
|
|
626
|
+
// should reflect what the matched branch would report
|
|
627
|
+
const invalidConfigScenarios = [
|
|
628
|
+
{
|
|
629
|
+
name: "Invalid URL format",
|
|
630
|
+
config: { transport: "http", url: "not-a-valid-url" },
|
|
631
|
+
expectedKeywords: ["url", "valid"],
|
|
632
|
+
},
|
|
633
|
+
{
|
|
634
|
+
name: "Empty command for stdio",
|
|
635
|
+
config: { transport: "stdio", command: "" },
|
|
636
|
+
expectedKeywords: ["command", "required"],
|
|
637
|
+
},
|
|
638
|
+
{
|
|
639
|
+
name: "Invalid URL only (matches HTTP branch)",
|
|
640
|
+
config: { url: "not-valid" },
|
|
641
|
+
expectedKeywords: ["url"],
|
|
642
|
+
},
|
|
643
|
+
{
|
|
644
|
+
name: "Empty command only (matches stdio branch)",
|
|
645
|
+
config: { command: "" },
|
|
646
|
+
expectedKeywords: ["command"],
|
|
647
|
+
},
|
|
648
|
+
];
|
|
649
|
+
for (const scenario of invalidConfigScenarios) {
|
|
650
|
+
const result = ServerEntrySchema.safeParse(scenario.config);
|
|
651
|
+
expect(result.success).toBe(false);
|
|
652
|
+
if (!result.success) {
|
|
653
|
+
// Step 1: Validate with Zod
|
|
654
|
+
const zodError = result.error;
|
|
655
|
+
// Step 2: Format with formatZodError
|
|
656
|
+
const formatted = formatZodError(zodError);
|
|
657
|
+
// Step 3: Verify helpful message is produced
|
|
658
|
+
// Should NOT be generic "Invalid input"
|
|
659
|
+
expect(formatted).not.toBe("Invalid input");
|
|
660
|
+
expect(formatted).not.toBe("Required");
|
|
661
|
+
// Should be descriptive (more than just a few characters)
|
|
662
|
+
expect(formatted.length).toBeGreaterThan(15);
|
|
663
|
+
// Should contain expected keywords for the scenario
|
|
664
|
+
const formattedLower = formatted.toLowerCase();
|
|
665
|
+
for (const keyword of scenario.expectedKeywords) {
|
|
666
|
+
expect(formattedLower).toContain(keyword.toLowerCase());
|
|
667
|
+
}
|
|
668
|
+
// Should not contain internal implementation details
|
|
669
|
+
expect(formatted).not.toContain("unionErrors");
|
|
670
|
+
expect(formatted).not.toContain("ZodError");
|
|
671
|
+
// Should be suitable for showing to end users
|
|
672
|
+
const hasUserFriendlyTerms = formattedLower.includes("required") ||
|
|
673
|
+
formattedLower.includes("valid") ||
|
|
674
|
+
formattedLower.includes("must be") ||
|
|
675
|
+
formattedLower.includes("expected");
|
|
676
|
+
expect(hasUserFriendlyTerms).toBe(true);
|
|
677
|
+
}
|
|
678
|
+
}
|
|
679
|
+
});
|
|
680
|
+
it("T-REQ-003-extended: should handle real-world config file parsing workflow", () => {
|
|
681
|
+
// Simulate the full workflow from assess-security.ts
|
|
682
|
+
const HttpSseSchema = z.object({
|
|
683
|
+
transport: z.enum(["http", "sse"]).optional(),
|
|
684
|
+
url: z
|
|
685
|
+
.string()
|
|
686
|
+
.min(1, "'url' is required for HTTP/SSE transport")
|
|
687
|
+
.url("url must be a valid URL"),
|
|
688
|
+
});
|
|
689
|
+
const StdioSchema = z.object({
|
|
690
|
+
transport: z.literal("stdio").optional(),
|
|
691
|
+
command: z.string().min(1, "command is required for stdio transport"),
|
|
692
|
+
args: z.array(z.string()).optional(),
|
|
693
|
+
env: z.record(z.string()).optional(),
|
|
694
|
+
});
|
|
695
|
+
const ServerEntrySchema = z.union([HttpSseSchema, StdioSchema]);
|
|
696
|
+
// Simulate JSON.parse() from file + validation
|
|
697
|
+
const rawConfigFromFile = '{"url": "not-a-url", "command": ""}';
|
|
698
|
+
const parsedConfig = JSON.parse(rawConfigFromFile);
|
|
699
|
+
// Validate parsed config
|
|
700
|
+
const validationResult = ServerEntrySchema.safeParse(parsedConfig);
|
|
701
|
+
expect(validationResult.success).toBe(false);
|
|
702
|
+
if (!validationResult.success) {
|
|
703
|
+
// Format for CLI output
|
|
704
|
+
const cliMessage = formatZodError(validationResult.error);
|
|
705
|
+
// Should be ready to display to user
|
|
706
|
+
expect(cliMessage).toBeTruthy();
|
|
707
|
+
expect(cliMessage.length).toBeGreaterThan(20);
|
|
708
|
+
// Should guide user to fix the issue
|
|
709
|
+
const providesGuidance = cliMessage.toLowerCase().includes("url") ||
|
|
710
|
+
cliMessage.toLowerCase().includes("command") ||
|
|
711
|
+
cliMessage.toLowerCase().includes("valid") ||
|
|
712
|
+
cliMessage.toLowerCase().includes("required");
|
|
713
|
+
expect(providesGuidance).toBe(true);
|
|
714
|
+
// Verify formatUserFriendlyError also works
|
|
715
|
+
const userFriendly = formatUserFriendlyError(validationResult.error);
|
|
716
|
+
expect(userFriendly).toBeTruthy();
|
|
717
|
+
expect(userFriendly.length).toBeGreaterThan(20);
|
|
718
|
+
}
|
|
719
|
+
});
|
|
720
|
+
});
|
|
721
|
+
});
|