@bryan-thompson/inspector-assessment-cli 1.35.2 → 1.36.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/__tests__/assess-full-e2e.test.js +35 -9
- package/build/__tests__/assess-full.test.js +4 -1
- package/build/__tests__/assessment-runner/assessment-executor.test.js +15 -2
- package/build/__tests__/assessment-runner/config-builder.test.js +6 -0
- package/build/__tests__/assessment-runner/index.test.js +9 -4
- package/build/__tests__/assessment-runner/path-resolver.test.js +112 -0
- package/build/__tests__/assessment-runner/server-config.test.js +3 -0
- package/build/__tests__/assessment-runner/server-connection.test.js +9 -1
- package/build/__tests__/assessment-runner/source-loader.test.js +121 -16
- package/build/__tests__/assessment-runner/tool-wrapper.test.js +3 -0
- package/build/__tests__/assessment-runner-facade.test.js +7 -2
- package/build/__tests__/cli-build-fixes.test.js +4 -1
- package/build/__tests__/flag-parsing.test.js +3 -2
- package/build/__tests__/http-transport-integration.test.js +19 -5
- package/build/__tests__/jsonl-events.test.js +1 -1
- package/build/__tests__/lib/server-configSchemas.test.js +4 -1
- package/build/__tests__/lib/zodErrorFormatter.test.js +4 -1
- package/build/__tests__/profiles.test.js +19 -8
- package/build/__tests__/security/security-pattern-count.test.js +6 -3
- package/build/__tests__/stage3-fix-validation.test.js +4 -1
- package/build/__tests__/testbed-integration.test.js +19 -5
- package/build/__tests__/transport.test.js +4 -1
- package/build/lib/__tests__/cli-parserSchemas.test.js +4 -1
- package/build/lib/assessment-runner/__tests__/server-configSchemas.test.js +4 -1
- package/build/lib/assessment-runner/assessment-executor.js +23 -4
- package/build/lib/assessment-runner/index.js +2 -0
- package/build/lib/assessment-runner/path-resolver.js +48 -0
- package/build/lib/assessment-runner/source-loader.js +47 -5
- package/build/lib/cli-parser.js +10 -0
- package/package.json +1 -1
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
*
|
|
15
15
|
* @see https://github.com/triepod-ai/inspector-assessment/issues/97
|
|
16
16
|
*/
|
|
17
|
-
import { describe, it, expect, beforeAll, afterAll } from "@jest/globals";
|
|
17
|
+
import { jest, describe, it, expect, beforeAll, afterAll, afterEach, } from "@jest/globals";
|
|
18
18
|
import { spawn } from "child_process";
|
|
19
19
|
import * as fs from "fs";
|
|
20
20
|
import * as path from "path";
|
|
@@ -76,6 +76,10 @@ async function spawnCLI(args, timeout = 60000) {
|
|
|
76
76
|
// Set timeout
|
|
77
77
|
const timer = setTimeout(() => {
|
|
78
78
|
if (proc && !proc.killed) {
|
|
79
|
+
// Destroy streams before killing to prevent memory leaks
|
|
80
|
+
proc.stdout?.destroy();
|
|
81
|
+
proc.stderr?.destroy();
|
|
82
|
+
proc.stdin?.destroy();
|
|
79
83
|
proc.kill("SIGTERM");
|
|
80
84
|
exitCode = -1; // Indicate timeout
|
|
81
85
|
}
|
|
@@ -83,6 +87,10 @@ async function spawnCLI(args, timeout = 60000) {
|
|
|
83
87
|
// Handle process exit
|
|
84
88
|
proc.on("close", (code) => {
|
|
85
89
|
clearTimeout(timer);
|
|
90
|
+
// Destroy streams to prevent memory leaks
|
|
91
|
+
proc?.stdout?.destroy();
|
|
92
|
+
proc?.stderr?.destroy();
|
|
93
|
+
proc?.stdin?.destroy();
|
|
86
94
|
// Don't overwrite timeout exit code (-1)
|
|
87
95
|
if (exitCode !== -1) {
|
|
88
96
|
exitCode = code;
|
|
@@ -100,6 +108,10 @@ async function spawnCLI(args, timeout = 60000) {
|
|
|
100
108
|
// Handle errors
|
|
101
109
|
proc.on("error", (err) => {
|
|
102
110
|
clearTimeout(timer);
|
|
111
|
+
// Destroy streams to prevent memory leaks
|
|
112
|
+
proc?.stdout?.destroy();
|
|
113
|
+
proc?.stderr?.destroy();
|
|
114
|
+
proc?.stdin?.destroy();
|
|
103
115
|
stderr += `\nProcess error: ${err.message}`;
|
|
104
116
|
resolve({
|
|
105
117
|
stdout,
|
|
@@ -151,10 +163,10 @@ function parseJSONLEvents(stderr) {
|
|
|
151
163
|
* @returns True if server responds, false otherwise
|
|
152
164
|
*/
|
|
153
165
|
async function checkServerAvailable(url) {
|
|
166
|
+
const controller = new AbortController();
|
|
167
|
+
// Give enough time to receive initial response but not wait forever
|
|
168
|
+
const timeoutId = setTimeout(() => controller.abort(), 5000);
|
|
154
169
|
try {
|
|
155
|
-
const controller = new AbortController();
|
|
156
|
-
// Give enough time to receive initial response but not wait forever
|
|
157
|
-
const timeoutId = setTimeout(() => controller.abort(), 5000);
|
|
158
170
|
const response = await fetch(url, {
|
|
159
171
|
method: "POST",
|
|
160
172
|
headers: DEFAULT_HEADERS,
|
|
@@ -172,26 +184,22 @@ async function checkServerAvailable(url) {
|
|
|
172
184
|
});
|
|
173
185
|
// Server responded with a status code - check if it's OK
|
|
174
186
|
if (response.status >= 500) {
|
|
175
|
-
clearTimeout(timeoutId);
|
|
176
187
|
return false;
|
|
177
188
|
}
|
|
178
189
|
// For SSE responses, check if we can read any data
|
|
179
190
|
// This confirms the server is actually responding
|
|
180
191
|
const reader = response.body?.getReader();
|
|
181
192
|
if (!reader) {
|
|
182
|
-
clearTimeout(timeoutId);
|
|
183
193
|
return response.status < 500;
|
|
184
194
|
}
|
|
185
195
|
try {
|
|
186
196
|
// Try to read the first chunk
|
|
187
197
|
const { done, value } = await reader.read();
|
|
188
|
-
clearTimeout(timeoutId);
|
|
189
198
|
reader.cancel(); // Cancel the stream - we don't need more data
|
|
190
199
|
// If we got any data, the server is available
|
|
191
200
|
return !done && value && value.length > 0;
|
|
192
201
|
}
|
|
193
202
|
catch {
|
|
194
|
-
clearTimeout(timeoutId);
|
|
195
203
|
// If read fails after successful fetch, server still responded
|
|
196
204
|
return true;
|
|
197
205
|
}
|
|
@@ -199,6 +207,11 @@ async function checkServerAvailable(url) {
|
|
|
199
207
|
catch {
|
|
200
208
|
return false;
|
|
201
209
|
}
|
|
210
|
+
finally {
|
|
211
|
+
// Always clean up timeout and abort controller
|
|
212
|
+
clearTimeout(timeoutId);
|
|
213
|
+
controller.abort();
|
|
214
|
+
}
|
|
202
215
|
}
|
|
203
216
|
/**
|
|
204
217
|
* Create a temporary config file for testing
|
|
@@ -235,9 +248,22 @@ function createInvalidConfig(content, filename) {
|
|
|
235
248
|
return configPath;
|
|
236
249
|
}
|
|
237
250
|
// ============================================================================
|
|
251
|
+
// E2E Skip Check
|
|
252
|
+
// ============================================================================
|
|
253
|
+
/**
|
|
254
|
+
* Skip E2E tests unless RUN_E2E_TESTS is set.
|
|
255
|
+
* This prevents long timeouts when testbed servers aren't running.
|
|
256
|
+
*
|
|
257
|
+
* To run E2E tests: RUN_E2E_TESTS=1 npm test -- --testPathPattern="e2e"
|
|
258
|
+
*/
|
|
259
|
+
const describeE2E = process.env.RUN_E2E_TESTS ? describe : describe.skip;
|
|
260
|
+
// ============================================================================
|
|
238
261
|
// Test Setup
|
|
239
262
|
// ============================================================================
|
|
240
|
-
|
|
263
|
+
describeE2E("CLI E2E Integration Tests", () => {
|
|
264
|
+
afterEach(() => {
|
|
265
|
+
jest.clearAllMocks();
|
|
266
|
+
});
|
|
241
267
|
let vulnerableAvailable = false;
|
|
242
268
|
let hardenedAvailable = false;
|
|
243
269
|
beforeAll(async () => {
|
|
@@ -6,13 +6,16 @@
|
|
|
6
6
|
* external dependencies. For integration testing of the full CLI, use the
|
|
7
7
|
* actual CLI binary.
|
|
8
8
|
*/
|
|
9
|
-
import { describe, it, expect } from "@jest/globals";
|
|
9
|
+
import { jest, describe, it, expect, afterEach } from "@jest/globals";
|
|
10
10
|
import * as path from "path";
|
|
11
11
|
/**
|
|
12
12
|
* Pure function tests - these test logic concepts used in the CLI
|
|
13
13
|
* without needing to import the actual module (which has side effects)
|
|
14
14
|
*/
|
|
15
15
|
describe("CLI Argument Parsing Concepts", () => {
|
|
16
|
+
afterEach(() => {
|
|
17
|
+
jest.clearAllMocks();
|
|
18
|
+
});
|
|
16
19
|
describe("Profile Flag Parsing", () => {
|
|
17
20
|
const VALID_PROFILES = ["quick", "security", "compliance", "full"];
|
|
18
21
|
function parseProfile(args) {
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Tests for runFullAssessment() orchestration logic.
|
|
5
5
|
*/
|
|
6
|
-
import { jest, describe, it, expect, beforeEach, afterEach, } from "@jest/globals";
|
|
6
|
+
import { jest, describe, it, expect, beforeEach, afterEach, afterAll, } from "@jest/globals";
|
|
7
7
|
// Mock all dependencies with explicit any types for flexibility
|
|
8
8
|
const mockLoadServerConfig = jest.fn();
|
|
9
9
|
const mockConnectToServer = jest.fn();
|
|
@@ -137,6 +137,19 @@ describe("runFullAssessment", () => {
|
|
|
137
137
|
afterEach(() => {
|
|
138
138
|
jest.restoreAllMocks();
|
|
139
139
|
});
|
|
140
|
+
afterAll(() => {
|
|
141
|
+
// Clean up module mocks to prevent memory leaks
|
|
142
|
+
jest.unmock("../../lib/assessment-runner/server-config.js");
|
|
143
|
+
jest.unmock("../../lib/assessment-runner/server-connection.js");
|
|
144
|
+
jest.unmock("../../lib/assessment-runner/source-loader.js");
|
|
145
|
+
jest.unmock("../../lib/assessment-runner/tool-wrapper.js");
|
|
146
|
+
jest.unmock("../../lib/assessment-runner/config-builder.js");
|
|
147
|
+
jest.unmock("../../../../client/lib/services/assessment/AssessmentOrchestrator.js");
|
|
148
|
+
jest.unmock("../../assessmentState.js");
|
|
149
|
+
jest.unmock("../../lib/jsonl-events.js");
|
|
150
|
+
jest.unmock("fs");
|
|
151
|
+
jest.unmock("../../../../client/lib/lib/assessmentTypes.js");
|
|
152
|
+
});
|
|
140
153
|
describe("orchestration flow", () => {
|
|
141
154
|
it("should load server config", async () => {
|
|
142
155
|
await runFullAssessment(defaultOptions);
|
|
@@ -203,7 +216,7 @@ describe("runFullAssessment", () => {
|
|
|
203
216
|
...defaultOptions,
|
|
204
217
|
sourceCodePath: "/path/to/source",
|
|
205
218
|
});
|
|
206
|
-
expect(mockLoadSourceFiles).toHaveBeenCalledWith("/path/to/source");
|
|
219
|
+
expect(mockLoadSourceFiles).toHaveBeenCalledWith("/path/to/source", undefined);
|
|
207
220
|
});
|
|
208
221
|
it("should not load source files when path does not exist", async () => {
|
|
209
222
|
const fs = await import("fs");
|
|
@@ -62,6 +62,12 @@ describe("buildConfig", () => {
|
|
|
62
62
|
afterEach(() => {
|
|
63
63
|
process.env = originalEnv;
|
|
64
64
|
});
|
|
65
|
+
afterAll(() => {
|
|
66
|
+
jest.unmock("../../profiles.js");
|
|
67
|
+
jest.unmock("../../../../client/lib/lib/assessmentTypes.js");
|
|
68
|
+
jest.unmock("../../../../client/lib/services/assessment/lib/claudeCodeBridge.js");
|
|
69
|
+
jest.unmock("../../../../client/lib/services/assessment/config/performanceConfig.js");
|
|
70
|
+
});
|
|
65
71
|
describe("default configuration", () => {
|
|
66
72
|
it("should spread DEFAULT_ASSESSMENT_CONFIG", () => {
|
|
67
73
|
const result = buildConfig({ serverName: "test" });
|
|
@@ -3,29 +3,34 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Tests for the facade module exports.
|
|
5
5
|
*/
|
|
6
|
-
import { describe, it, expect } from "@jest/globals";
|
|
6
|
+
import { jest, describe, it, expect, afterEach } from "@jest/globals";
|
|
7
7
|
// Import the barrel/facade module
|
|
8
8
|
import * as assessmentRunner from "../../lib/assessment-runner/index.js";
|
|
9
9
|
describe("assessment-runner index exports", () => {
|
|
10
|
+
afterEach(() => {
|
|
11
|
+
jest.clearAllMocks();
|
|
12
|
+
});
|
|
10
13
|
describe("function exports", () => {
|
|
11
|
-
it("should export all
|
|
14
|
+
it("should export all 7 public functions", () => {
|
|
12
15
|
expect(typeof assessmentRunner.loadServerConfig).toBe("function");
|
|
13
16
|
expect(typeof assessmentRunner.loadSourceFiles).toBe("function");
|
|
17
|
+
expect(typeof assessmentRunner.resolveSourcePath).toBe("function");
|
|
14
18
|
expect(typeof assessmentRunner.connectToServer).toBe("function");
|
|
15
19
|
expect(typeof assessmentRunner.createCallToolWrapper).toBe("function");
|
|
16
20
|
expect(typeof assessmentRunner.buildConfig).toBe("function");
|
|
17
21
|
expect(typeof assessmentRunner.runFullAssessment).toBe("function");
|
|
18
22
|
});
|
|
19
|
-
it("should export exactly
|
|
23
|
+
it("should export exactly 7 functions", () => {
|
|
20
24
|
const functionNames = Object.keys(assessmentRunner).filter((key) => typeof assessmentRunner[key] ===
|
|
21
25
|
"function");
|
|
22
|
-
expect(functionNames).toHaveLength(
|
|
26
|
+
expect(functionNames).toHaveLength(7);
|
|
23
27
|
expect(functionNames.sort()).toEqual([
|
|
24
28
|
"buildConfig",
|
|
25
29
|
"connectToServer",
|
|
26
30
|
"createCallToolWrapper",
|
|
27
31
|
"loadServerConfig",
|
|
28
32
|
"loadSourceFiles",
|
|
33
|
+
"resolveSourcePath",
|
|
29
34
|
"runFullAssessment",
|
|
30
35
|
]);
|
|
31
36
|
});
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Path Resolver Unit Tests
|
|
3
|
+
*
|
|
4
|
+
* Tests for resolveSourcePath() that handles path normalization.
|
|
5
|
+
*/
|
|
6
|
+
import { jest, describe, it, expect, beforeEach, afterAll, } from "@jest/globals";
|
|
7
|
+
import * as path from "path";
|
|
8
|
+
import * as os from "os";
|
|
9
|
+
// Mock fs module
|
|
10
|
+
jest.unstable_mockModule("fs", () => ({
|
|
11
|
+
existsSync: jest.fn(),
|
|
12
|
+
realpathSync: jest.fn(),
|
|
13
|
+
}));
|
|
14
|
+
// Import after mocking
|
|
15
|
+
const fs = await import("fs");
|
|
16
|
+
const { resolveSourcePath } = await import("../../lib/assessment-runner/path-resolver.js");
|
|
17
|
+
describe("resolveSourcePath", () => {
|
|
18
|
+
const mockExistsSync = fs.existsSync;
|
|
19
|
+
const mockRealpathSync = fs.realpathSync;
|
|
20
|
+
beforeEach(() => {
|
|
21
|
+
mockExistsSync.mockReturnValue(false);
|
|
22
|
+
mockRealpathSync.mockReturnValue("");
|
|
23
|
+
});
|
|
24
|
+
afterEach(() => {
|
|
25
|
+
jest.clearAllMocks();
|
|
26
|
+
});
|
|
27
|
+
afterAll(() => {
|
|
28
|
+
jest.unmock("fs");
|
|
29
|
+
});
|
|
30
|
+
describe("tilde expansion", () => {
|
|
31
|
+
it("should expand ~ to home directory", () => {
|
|
32
|
+
mockExistsSync.mockReturnValue(false);
|
|
33
|
+
const result = resolveSourcePath("~/project");
|
|
34
|
+
expect(result).toBe(path.join(os.homedir(), "project"));
|
|
35
|
+
});
|
|
36
|
+
it("should expand ~/subdir/path correctly", () => {
|
|
37
|
+
mockExistsSync.mockReturnValue(false);
|
|
38
|
+
const result = resolveSourcePath("~/foo/bar/baz");
|
|
39
|
+
expect(result).toBe(path.join(os.homedir(), "foo/bar/baz"));
|
|
40
|
+
});
|
|
41
|
+
it("should not modify paths without tilde", () => {
|
|
42
|
+
mockExistsSync.mockReturnValue(false);
|
|
43
|
+
const result = resolveSourcePath("/absolute/path");
|
|
44
|
+
expect(result).toBe("/absolute/path");
|
|
45
|
+
});
|
|
46
|
+
});
|
|
47
|
+
describe("relative path resolution", () => {
|
|
48
|
+
it("should resolve relative paths to absolute", () => {
|
|
49
|
+
mockExistsSync.mockReturnValue(false);
|
|
50
|
+
const cwd = process.cwd();
|
|
51
|
+
const result = resolveSourcePath("./src");
|
|
52
|
+
expect(result).toBe(path.resolve(cwd, "./src"));
|
|
53
|
+
});
|
|
54
|
+
it("should resolve parent directory references", () => {
|
|
55
|
+
mockExistsSync.mockReturnValue(false);
|
|
56
|
+
const cwd = process.cwd();
|
|
57
|
+
const result = resolveSourcePath("../sibling");
|
|
58
|
+
expect(result).toBe(path.resolve(cwd, "../sibling"));
|
|
59
|
+
});
|
|
60
|
+
it("should handle bare directory names", () => {
|
|
61
|
+
mockExistsSync.mockReturnValue(false);
|
|
62
|
+
const cwd = process.cwd();
|
|
63
|
+
const result = resolveSourcePath("mydir");
|
|
64
|
+
expect(result).toBe(path.resolve(cwd, "mydir"));
|
|
65
|
+
});
|
|
66
|
+
});
|
|
67
|
+
describe("symlink resolution", () => {
|
|
68
|
+
it("should follow symlinks when path exists", () => {
|
|
69
|
+
const symlinkPath = "/tmp/symlink";
|
|
70
|
+
const realPath = "/actual/target/path";
|
|
71
|
+
mockExistsSync.mockImplementation((p) => p === symlinkPath);
|
|
72
|
+
mockRealpathSync.mockReturnValue(realPath);
|
|
73
|
+
const result = resolveSourcePath(symlinkPath);
|
|
74
|
+
expect(mockRealpathSync).toHaveBeenCalledWith(symlinkPath);
|
|
75
|
+
expect(result).toBe(realPath);
|
|
76
|
+
});
|
|
77
|
+
it("should not call realpathSync when path does not exist", () => {
|
|
78
|
+
mockExistsSync.mockReturnValue(false);
|
|
79
|
+
resolveSourcePath("/nonexistent/path");
|
|
80
|
+
expect(mockRealpathSync).not.toHaveBeenCalled();
|
|
81
|
+
});
|
|
82
|
+
it("should handle broken symlinks gracefully", () => {
|
|
83
|
+
const brokenSymlink = "/tmp/broken-symlink";
|
|
84
|
+
mockExistsSync.mockReturnValue(true);
|
|
85
|
+
mockRealpathSync.mockImplementation(() => {
|
|
86
|
+
throw new Error("ENOENT: no such file or directory");
|
|
87
|
+
});
|
|
88
|
+
// Should not throw, should return the resolved path without realpath
|
|
89
|
+
const result = resolveSourcePath(brokenSymlink);
|
|
90
|
+
expect(result).toBe(brokenSymlink);
|
|
91
|
+
});
|
|
92
|
+
});
|
|
93
|
+
describe("combined scenarios", () => {
|
|
94
|
+
it("should handle ~ with symlink resolution", () => {
|
|
95
|
+
const tildePathExpanded = path.join(os.homedir(), "project");
|
|
96
|
+
const realPath = "/real/project/path";
|
|
97
|
+
mockExistsSync.mockImplementation((p) => p === tildePathExpanded);
|
|
98
|
+
mockRealpathSync.mockReturnValue(realPath);
|
|
99
|
+
const result = resolveSourcePath("~/project");
|
|
100
|
+
expect(result).toBe(realPath);
|
|
101
|
+
});
|
|
102
|
+
it("should handle relative path with symlink resolution", () => {
|
|
103
|
+
const cwd = process.cwd();
|
|
104
|
+
const resolvedRelative = path.resolve(cwd, "./src");
|
|
105
|
+
const realPath = "/real/src/path";
|
|
106
|
+
mockExistsSync.mockImplementation((p) => p === resolvedRelative);
|
|
107
|
+
mockRealpathSync.mockReturnValue(realPath);
|
|
108
|
+
const result = resolveSourcePath("./src");
|
|
109
|
+
expect(result).toBe(realPath);
|
|
110
|
+
});
|
|
111
|
+
});
|
|
112
|
+
});
|
|
@@ -22,6 +22,9 @@ describe("loadServerConfig", () => {
|
|
|
22
22
|
jest.clearAllMocks();
|
|
23
23
|
mockExistsSync.mockReturnValue(false);
|
|
24
24
|
});
|
|
25
|
+
afterAll(() => {
|
|
26
|
+
jest.unmock("fs");
|
|
27
|
+
});
|
|
25
28
|
describe("config path resolution", () => {
|
|
26
29
|
it("should search explicit configPath first when provided", () => {
|
|
27
30
|
const configPath = "/custom/path/config.json";
|
|
@@ -39,10 +39,18 @@ const { StreamableHTTPClientTransport } = await import("@modelcontextprotocol/sd
|
|
|
39
39
|
const { connectToServer } = await import("../../lib/assessment-runner/server-connection.js");
|
|
40
40
|
describe("connectToServer", () => {
|
|
41
41
|
beforeEach(() => {
|
|
42
|
-
jest.clearAllMocks();
|
|
43
42
|
mockConnect.mockResolvedValue(undefined);
|
|
44
43
|
mockStdioTransport.stderr.on.mockClear();
|
|
45
44
|
});
|
|
45
|
+
afterEach(() => {
|
|
46
|
+
jest.clearAllMocks();
|
|
47
|
+
});
|
|
48
|
+
afterAll(() => {
|
|
49
|
+
jest.unmock("@modelcontextprotocol/sdk/client/index.js");
|
|
50
|
+
jest.unmock("@modelcontextprotocol/sdk/client/stdio.js");
|
|
51
|
+
jest.unmock("@modelcontextprotocol/sdk/client/sse.js");
|
|
52
|
+
jest.unmock("@modelcontextprotocol/sdk/client/streamableHttp.js");
|
|
53
|
+
});
|
|
46
54
|
describe("HTTP transport", () => {
|
|
47
55
|
it("should create StreamableHTTPClientTransport for transport:http", async () => {
|
|
48
56
|
const config = {
|
|
@@ -34,10 +34,15 @@ describe("loadSourceFiles", () => {
|
|
|
34
34
|
const mockReadFileSync = fs.readFileSync;
|
|
35
35
|
const mockReaddirSync = fs.readdirSync;
|
|
36
36
|
beforeEach(() => {
|
|
37
|
-
jest.clearAllMocks();
|
|
38
37
|
mockExistsSync.mockReturnValue(false);
|
|
39
38
|
mockReaddirSync.mockReturnValue([]);
|
|
40
39
|
});
|
|
40
|
+
afterEach(() => {
|
|
41
|
+
jest.clearAllMocks();
|
|
42
|
+
});
|
|
43
|
+
afterAll(() => {
|
|
44
|
+
jest.unmock("fs");
|
|
45
|
+
});
|
|
41
46
|
describe("README discovery", () => {
|
|
42
47
|
it("should find README.md in source directory", () => {
|
|
43
48
|
const sourcePath = "/project";
|
|
@@ -75,6 +80,98 @@ describe("loadSourceFiles", () => {
|
|
|
75
80
|
const result = loadSourceFiles(sourcePath);
|
|
76
81
|
expect(result.readmeContent).toBeUndefined();
|
|
77
82
|
});
|
|
83
|
+
// Issue #151: Extended README patterns
|
|
84
|
+
it("should find README.markdown", () => {
|
|
85
|
+
const sourcePath = "/project";
|
|
86
|
+
mockExistsSync.mockImplementation((p) => p === path.join(sourcePath, "README.markdown"));
|
|
87
|
+
mockReadFileSync.mockReturnValue("# Markdown README");
|
|
88
|
+
const result = loadSourceFiles(sourcePath);
|
|
89
|
+
expect(result.readmeContent).toBe("# Markdown README");
|
|
90
|
+
});
|
|
91
|
+
it("should find README.txt", () => {
|
|
92
|
+
const sourcePath = "/project";
|
|
93
|
+
mockExistsSync.mockImplementation((p) => p === path.join(sourcePath, "README.txt"));
|
|
94
|
+
mockReadFileSync.mockReturnValue("Text README content");
|
|
95
|
+
const result = loadSourceFiles(sourcePath);
|
|
96
|
+
expect(result.readmeContent).toBe("Text README content");
|
|
97
|
+
});
|
|
98
|
+
it("should find README without extension", () => {
|
|
99
|
+
const sourcePath = "/project";
|
|
100
|
+
mockExistsSync.mockImplementation((p) => p === path.join(sourcePath, "README"));
|
|
101
|
+
mockReadFileSync.mockReturnValue("No extension README");
|
|
102
|
+
const result = loadSourceFiles(sourcePath);
|
|
103
|
+
expect(result.readmeContent).toBe("No extension README");
|
|
104
|
+
});
|
|
105
|
+
it("should prioritize README.md over other variants", () => {
|
|
106
|
+
const sourcePath = "/project";
|
|
107
|
+
// Multiple README files exist - README.md should be found first
|
|
108
|
+
mockExistsSync.mockImplementation((p) => {
|
|
109
|
+
return (p === path.join(sourcePath, "README.md") ||
|
|
110
|
+
p === path.join(sourcePath, "README.txt") ||
|
|
111
|
+
p === path.join(sourcePath, "README"));
|
|
112
|
+
});
|
|
113
|
+
mockReadFileSync.mockImplementation((p) => {
|
|
114
|
+
if (p.endsWith("README.md"))
|
|
115
|
+
return "# MD README";
|
|
116
|
+
if (p.endsWith("README.txt"))
|
|
117
|
+
return "TXT README";
|
|
118
|
+
if (p.endsWith("README"))
|
|
119
|
+
return "Plain README";
|
|
120
|
+
return "";
|
|
121
|
+
});
|
|
122
|
+
const result = loadSourceFiles(sourcePath);
|
|
123
|
+
expect(result.readmeContent).toBe("# MD README");
|
|
124
|
+
});
|
|
125
|
+
});
|
|
126
|
+
describe("debug logging", () => {
|
|
127
|
+
it("should log debug output when debug flag is true", () => {
|
|
128
|
+
const sourcePath = "/project";
|
|
129
|
+
const consoleSpy = jest
|
|
130
|
+
.spyOn(console, "log")
|
|
131
|
+
.mockImplementation(() => { });
|
|
132
|
+
try {
|
|
133
|
+
mockExistsSync.mockReturnValue(false);
|
|
134
|
+
mockReaddirSync.mockReturnValue([]);
|
|
135
|
+
loadSourceFiles(sourcePath, true);
|
|
136
|
+
// Verify debug logging was called
|
|
137
|
+
expect(consoleSpy).toHaveBeenCalledWith(expect.stringContaining("[source-loader]"));
|
|
138
|
+
}
|
|
139
|
+
finally {
|
|
140
|
+
consoleSpy.mockRestore();
|
|
141
|
+
}
|
|
142
|
+
});
|
|
143
|
+
it("should not log debug output when debug flag is false", () => {
|
|
144
|
+
const sourcePath = "/project";
|
|
145
|
+
const consoleSpy = jest
|
|
146
|
+
.spyOn(console, "log")
|
|
147
|
+
.mockImplementation(() => { });
|
|
148
|
+
try {
|
|
149
|
+
mockExistsSync.mockReturnValue(false);
|
|
150
|
+
mockReaddirSync.mockReturnValue([]);
|
|
151
|
+
loadSourceFiles(sourcePath, false);
|
|
152
|
+
// Verify debug logging was NOT called
|
|
153
|
+
expect(consoleSpy).not.toHaveBeenCalledWith(expect.stringContaining("[source-loader]"));
|
|
154
|
+
}
|
|
155
|
+
finally {
|
|
156
|
+
consoleSpy.mockRestore();
|
|
157
|
+
}
|
|
158
|
+
});
|
|
159
|
+
it("should not log debug output by default", () => {
|
|
160
|
+
const sourcePath = "/project";
|
|
161
|
+
const consoleSpy = jest
|
|
162
|
+
.spyOn(console, "log")
|
|
163
|
+
.mockImplementation(() => { });
|
|
164
|
+
try {
|
|
165
|
+
mockExistsSync.mockReturnValue(false);
|
|
166
|
+
mockReaddirSync.mockReturnValue([]);
|
|
167
|
+
loadSourceFiles(sourcePath);
|
|
168
|
+
// Verify debug logging was NOT called by default
|
|
169
|
+
expect(consoleSpy).not.toHaveBeenCalledWith(expect.stringContaining("[source-loader]"));
|
|
170
|
+
}
|
|
171
|
+
finally {
|
|
172
|
+
consoleSpy.mockRestore();
|
|
173
|
+
}
|
|
174
|
+
});
|
|
78
175
|
});
|
|
79
176
|
describe("package.json parsing", () => {
|
|
80
177
|
it("should parse package.json when present", () => {
|
|
@@ -118,13 +215,17 @@ describe("loadSourceFiles", () => {
|
|
|
118
215
|
const consoleSpy = jest
|
|
119
216
|
.spyOn(console, "warn")
|
|
120
217
|
.mockImplementation(() => { });
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
218
|
+
try {
|
|
219
|
+
mockExistsSync.mockImplementation((p) => p === path.join(sourcePath, "manifest.json"));
|
|
220
|
+
mockReadFileSync.mockReturnValue("{ invalid json }");
|
|
221
|
+
const result = loadSourceFiles(sourcePath);
|
|
222
|
+
expect(result.manifestRaw).toBe("{ invalid json }");
|
|
223
|
+
expect(result.manifestJson).toBeUndefined();
|
|
224
|
+
expect(consoleSpy).toHaveBeenCalledWith(expect.stringContaining("Failed to parse manifest.json"));
|
|
225
|
+
}
|
|
226
|
+
finally {
|
|
227
|
+
consoleSpy.mockRestore();
|
|
228
|
+
}
|
|
128
229
|
});
|
|
129
230
|
});
|
|
130
231
|
describe("source file collection", () => {
|
|
@@ -276,14 +377,18 @@ describe("loadSourceFiles", () => {
|
|
|
276
377
|
const consoleSpy = jest
|
|
277
378
|
.spyOn(console, "warn")
|
|
278
379
|
.mockImplementation(() => { });
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
380
|
+
try {
|
|
381
|
+
mockExistsSync.mockReturnValue(false);
|
|
382
|
+
mockReaddirSync.mockImplementation(() => {
|
|
383
|
+
throw new Error("Permission denied");
|
|
384
|
+
});
|
|
385
|
+
const result = loadSourceFiles(sourcePath);
|
|
386
|
+
expect(result.sourceCodeFiles?.size).toBe(0);
|
|
387
|
+
expect(consoleSpy).toHaveBeenCalledWith(expect.stringContaining("Could not load source files"), expect.any(Error));
|
|
388
|
+
}
|
|
389
|
+
finally {
|
|
390
|
+
consoleSpy.mockRestore();
|
|
391
|
+
}
|
|
287
392
|
});
|
|
288
393
|
it("should skip unreadable files silently", () => {
|
|
289
394
|
const sourcePath = "/project";
|
|
@@ -15,6 +15,9 @@ describe("createCallToolWrapper", () => {
|
|
|
15
15
|
callTool: mockCallTool,
|
|
16
16
|
};
|
|
17
17
|
});
|
|
18
|
+
afterEach(() => {
|
|
19
|
+
jest.clearAllMocks();
|
|
20
|
+
});
|
|
18
21
|
describe("successful tool calls", () => {
|
|
19
22
|
it("should wrap successful tool response with content array", async () => {
|
|
20
23
|
mockCallTool.mockResolvedValue({
|
|
@@ -6,12 +6,15 @@
|
|
|
6
6
|
*
|
|
7
7
|
* @see https://github.com/triepod-ai/inspector-assessment/issues/96
|
|
8
8
|
*/
|
|
9
|
-
import { describe, it, expect } from "@jest/globals";
|
|
9
|
+
import { jest, describe, it, expect, afterEach } from "@jest/globals";
|
|
10
10
|
// Test named imports (the primary consumer pattern)
|
|
11
11
|
import { loadServerConfig, loadSourceFiles, connectToServer, createCallToolWrapper, buildConfig, runFullAssessment, } from "../lib/assessment-runner.js";
|
|
12
12
|
// Test namespace import
|
|
13
13
|
import * as AssessmentRunner from "../lib/assessment-runner.js";
|
|
14
14
|
describe("Assessment Runner Facade", () => {
|
|
15
|
+
afterEach(() => {
|
|
16
|
+
jest.clearAllMocks();
|
|
17
|
+
});
|
|
15
18
|
describe("Function Exports", () => {
|
|
16
19
|
it("should export loadServerConfig function", () => {
|
|
17
20
|
expect(typeof loadServerConfig).toBe("function");
|
|
@@ -81,13 +84,14 @@ describe("Assessment Runner Facade", () => {
|
|
|
81
84
|
describe("Export Completeness", () => {
|
|
82
85
|
it("should export exactly 6 functions", () => {
|
|
83
86
|
const exportedFunctions = Object.entries(AssessmentRunner).filter(([, value]) => typeof value === "function");
|
|
84
|
-
expect(exportedFunctions.length).toBe(
|
|
87
|
+
expect(exportedFunctions.length).toBe(7);
|
|
85
88
|
expect(exportedFunctions.map(([name]) => name).sort()).toEqual([
|
|
86
89
|
"buildConfig",
|
|
87
90
|
"connectToServer",
|
|
88
91
|
"createCallToolWrapper",
|
|
89
92
|
"loadServerConfig",
|
|
90
93
|
"loadSourceFiles",
|
|
94
|
+
"resolveSourcePath",
|
|
91
95
|
"runFullAssessment",
|
|
92
96
|
]);
|
|
93
97
|
});
|
|
@@ -96,6 +100,7 @@ describe("Assessment Runner Facade", () => {
|
|
|
96
100
|
const expectedExports = [
|
|
97
101
|
"loadServerConfig",
|
|
98
102
|
"loadSourceFiles",
|
|
103
|
+
"resolveSourcePath",
|
|
99
104
|
"connectToServer",
|
|
100
105
|
"createCallToolWrapper",
|
|
101
106
|
"buildConfig",
|
|
@@ -8,9 +8,12 @@
|
|
|
8
8
|
* @see https://github.com/triepod-ai/inspector-assessment/issues/33
|
|
9
9
|
* @see https://github.com/triepod-ai/inspector-assessment/issues/37
|
|
10
10
|
*/
|
|
11
|
-
import { describe, it, expect } from "@jest/globals";
|
|
11
|
+
import { jest, describe, it, expect, afterEach } from "@jest/globals";
|
|
12
12
|
import { ScopedListenerConfig } from "../lib/event-config.js";
|
|
13
13
|
describe("CLI Build Fixes Regression Tests", () => {
|
|
14
|
+
afterEach(() => {
|
|
15
|
+
jest.clearAllMocks();
|
|
16
|
+
});
|
|
14
17
|
describe("event-config.ts - CLI_DEFAULT_MAX_LISTENERS constant", () => {
|
|
15
18
|
it("should use local constant instead of cross-workspace import", () => {
|
|
16
19
|
// Fix: Replaced DEFAULT_PERFORMANCE_CONFIG.eventEmitterMaxListeners
|
|
@@ -669,8 +669,9 @@ describe("parseArgs Zod Schema Integration", () => {
|
|
|
669
669
|
// Run any pending timers and restore
|
|
670
670
|
jest.runAllTimers();
|
|
671
671
|
jest.useRealTimers();
|
|
672
|
-
|
|
673
|
-
|
|
672
|
+
// Use optional chaining in case spies weren't created (prevents memory leaks)
|
|
673
|
+
processExitSpy?.mockRestore();
|
|
674
|
+
consoleErrorSpy?.mockRestore();
|
|
674
675
|
});
|
|
675
676
|
describe("LogLevelSchema integration", () => {
|
|
676
677
|
it("parseArgs validates log level with LogLevelSchema", () => {
|
|
@@ -7,8 +7,15 @@
|
|
|
7
7
|
*
|
|
8
8
|
* Note: Tests skip gracefully when testbed servers are unavailable.
|
|
9
9
|
*/
|
|
10
|
-
import { describe, it, expect, beforeAll } from "@jest/globals";
|
|
10
|
+
import { jest, describe, it, expect, beforeAll, afterEach, } from "@jest/globals";
|
|
11
|
+
/**
|
|
12
|
+
* Skip integration tests unless RUN_E2E_TESTS is set.
|
|
13
|
+
* This prevents long timeouts when testbed servers aren't running.
|
|
14
|
+
*
|
|
15
|
+
* To run: RUN_E2E_TESTS=1 npm test -- --testPathPattern="http-transport-integration"
|
|
16
|
+
*/
|
|
11
17
|
import { createTransport } from "../transport.js";
|
|
18
|
+
const describeE2E = process.env.RUN_E2E_TESTS ? describe : describe.skip;
|
|
12
19
|
// Testbed server URLs
|
|
13
20
|
const VULNERABLE_MCP_URL = "http://localhost:10900/mcp";
|
|
14
21
|
const HARDENED_MCP_URL = "http://localhost:10901/mcp";
|
|
@@ -24,9 +31,9 @@ const DEFAULT_HEADERS = {
|
|
|
24
31
|
* Check if a server is available by sending a basic HTTP request
|
|
25
32
|
*/
|
|
26
33
|
async function checkServerAvailable(url) {
|
|
34
|
+
const controller = new AbortController();
|
|
35
|
+
const timeoutId = setTimeout(() => controller.abort(), 5000);
|
|
27
36
|
try {
|
|
28
|
-
const controller = new AbortController();
|
|
29
|
-
const timeoutId = setTimeout(() => controller.abort(), 5000);
|
|
30
37
|
const response = await fetch(url, {
|
|
31
38
|
method: "POST",
|
|
32
39
|
headers: DEFAULT_HEADERS,
|
|
@@ -42,13 +49,17 @@ async function checkServerAvailable(url) {
|
|
|
42
49
|
}),
|
|
43
50
|
signal: controller.signal,
|
|
44
51
|
});
|
|
45
|
-
clearTimeout(timeoutId);
|
|
46
52
|
// Accept any response (200 or error) as indication server is up
|
|
47
53
|
return response.status < 500;
|
|
48
54
|
}
|
|
49
55
|
catch {
|
|
50
56
|
return false;
|
|
51
57
|
}
|
|
58
|
+
finally {
|
|
59
|
+
// Always clean up timeout and abort controller to prevent memory leaks
|
|
60
|
+
clearTimeout(timeoutId);
|
|
61
|
+
controller.abort();
|
|
62
|
+
}
|
|
52
63
|
}
|
|
53
64
|
/**
|
|
54
65
|
* Parse SSE response to extract JSON data
|
|
@@ -100,7 +111,10 @@ async function sendMcpRequest(url, method, params = {}, headers = {}) {
|
|
|
100
111
|
}
|
|
101
112
|
return { response, data };
|
|
102
113
|
}
|
|
103
|
-
|
|
114
|
+
describeE2E("HTTP Transport Integration", () => {
|
|
115
|
+
afterEach(() => {
|
|
116
|
+
jest.clearAllMocks();
|
|
117
|
+
});
|
|
104
118
|
let vulnerableServerAvailable = false;
|
|
105
119
|
let hardenedServerAvailable = false;
|
|
106
120
|
beforeAll(async () => {
|
|
@@ -40,7 +40,7 @@ describe("JSONL Event Emission", () => {
|
|
|
40
40
|
});
|
|
41
41
|
it("should include schemaVersion field", () => {
|
|
42
42
|
emitJSONL({ event: "test" });
|
|
43
|
-
expect(emittedEvents[0]).toHaveProperty("schemaVersion",
|
|
43
|
+
expect(emittedEvents[0]).toHaveProperty("schemaVersion", 3);
|
|
44
44
|
});
|
|
45
45
|
it("should handle complex nested objects", () => {
|
|
46
46
|
emitJSONL({
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
* Addresses QA requirement: Test that type guards are mutually exclusive
|
|
8
8
|
* and correctly discriminate between transport types.
|
|
9
9
|
*/
|
|
10
|
-
import { describe, it, expect } from "@jest/globals";
|
|
10
|
+
import { jest, describe, it, expect } from "@jest/globals";
|
|
11
11
|
import { z } from "zod";
|
|
12
12
|
// Define schemas inline to avoid import issues with client/lib in CLI tests
|
|
13
13
|
const HttpSseServerConfigSchema = z.object({
|
|
@@ -36,6 +36,9 @@ function isStdioConfig(entry) {
|
|
|
36
36
|
return "command" in entry && !("url" in entry);
|
|
37
37
|
}
|
|
38
38
|
describe("server-configSchemas type guards", () => {
|
|
39
|
+
afterEach(() => {
|
|
40
|
+
jest.clearAllMocks();
|
|
41
|
+
});
|
|
39
42
|
describe("isHttpSseConfig", () => {
|
|
40
43
|
it("should return true for HTTP transport config", () => {
|
|
41
44
|
const config = {
|
|
@@ -4,10 +4,13 @@
|
|
|
4
4
|
* Tests for formatZodError utility to ensure helpful error messages.
|
|
5
5
|
* Addresses QA requirement: verify Zod error messages are helpful (not just generic "Invalid").
|
|
6
6
|
*/
|
|
7
|
-
import { describe, it, expect } from "@jest/globals";
|
|
7
|
+
import { jest, describe, it, expect } from "@jest/globals";
|
|
8
8
|
import { z } from "zod";
|
|
9
9
|
import { formatZodError, formatZodIssue, formatZodErrorIndented, zodErrorToArray, formatUserFriendlyError, formatZodErrorForJson, } from "../../lib/zodErrorFormatter.js";
|
|
10
10
|
describe("zodErrorFormatter", () => {
|
|
11
|
+
afterEach(() => {
|
|
12
|
+
jest.clearAllMocks();
|
|
13
|
+
});
|
|
11
14
|
describe("formatZodIssue", () => {
|
|
12
15
|
it("should format issue with path", () => {
|
|
13
16
|
const issue = {
|
|
@@ -7,6 +7,9 @@
|
|
|
7
7
|
import { jest, describe, it, expect } from "@jest/globals";
|
|
8
8
|
import { ASSESSMENT_PROFILES, PROFILE_METADATA, MODULE_ALIASES, DEPRECATED_MODULES, TIER_1_CORE_SECURITY, TIER_2_COMPLIANCE, TIER_3_CAPABILITY, TIER_4_EXTENDED, ALL_MODULES, resolveModuleNames, getProfileModules, isValidProfileName, getProfileHelpText, mapLegacyConfigToModules, modulesToLegacyConfig, } from "../profiles.js";
|
|
9
9
|
describe("Profile Definitions", () => {
|
|
10
|
+
afterEach(() => {
|
|
11
|
+
jest.restoreAllMocks();
|
|
12
|
+
});
|
|
10
13
|
describe("Profile Constants", () => {
|
|
11
14
|
it("should have four profiles defined", () => {
|
|
12
15
|
const profiles = Object.keys(ASSESSMENT_PROFILES);
|
|
@@ -124,17 +127,25 @@ describe("resolveModuleNames", () => {
|
|
|
124
127
|
});
|
|
125
128
|
it("should emit warnings for deprecated modules when warn=true", () => {
|
|
126
129
|
const consoleSpy = jest.spyOn(console, "warn").mockImplementation(() => { });
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
130
|
+
try {
|
|
131
|
+
const modules = ["documentation"];
|
|
132
|
+
resolveModuleNames(modules, true);
|
|
133
|
+
expect(consoleSpy).toHaveBeenCalledWith(expect.stringContaining("deprecated"));
|
|
134
|
+
}
|
|
135
|
+
finally {
|
|
136
|
+
consoleSpy.mockRestore();
|
|
137
|
+
}
|
|
131
138
|
});
|
|
132
139
|
it("should not emit warnings when warn=false", () => {
|
|
133
140
|
const consoleSpy = jest.spyOn(console, "warn").mockImplementation(() => { });
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
141
|
+
try {
|
|
142
|
+
const modules = ["documentation"];
|
|
143
|
+
resolveModuleNames(modules, false);
|
|
144
|
+
expect(consoleSpy).not.toHaveBeenCalled();
|
|
145
|
+
}
|
|
146
|
+
finally {
|
|
147
|
+
consoleSpy.mockRestore();
|
|
148
|
+
}
|
|
138
149
|
});
|
|
139
150
|
});
|
|
140
151
|
describe("getProfileModules", () => {
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
* CLI tools override this to 30 patterns for comprehensive security assessment.
|
|
14
14
|
* This test verifies consistency within each context.
|
|
15
15
|
*/
|
|
16
|
-
import { describe, it, expect } from "@jest/globals";
|
|
16
|
+
import { jest, describe, it, expect, afterEach } from "@jest/globals";
|
|
17
17
|
import * as fs from "fs";
|
|
18
18
|
import * as path from "path";
|
|
19
19
|
import { fileURLToPath } from "url";
|
|
@@ -22,6 +22,9 @@ const __filename = fileURLToPath(import.meta.url);
|
|
|
22
22
|
const __dirname = path.dirname(__filename);
|
|
23
23
|
const projectRoot = path.resolve(__dirname, "../../../.."); // From cli/src/__tests__/security to root
|
|
24
24
|
describe("Security Pattern Count Consistency", () => {
|
|
25
|
+
afterEach(() => {
|
|
26
|
+
jest.clearAllMocks();
|
|
27
|
+
});
|
|
25
28
|
describe("CLI assess-security references", () => {
|
|
26
29
|
it("should use consistent pattern count in assess-security.ts", () => {
|
|
27
30
|
const filePath = path.join(projectRoot, "cli/src/assess-security.ts");
|
|
@@ -131,9 +134,9 @@ describe("Security Pattern Count Consistency", () => {
|
|
|
131
134
|
// Default: "default all 8" or "default 8"
|
|
132
135
|
expect(content).toMatch(/securityPatternsToTest\?\s*:\s*number;.*default.*8/i);
|
|
133
136
|
// Reviewer mode: "Test only 3 critical"
|
|
134
|
-
expect(content).toMatch(/securityPatternsToTest:\s*3
|
|
137
|
+
expect(content).toMatch(/securityPatternsToTest:\s*3,.*3 critical/i);
|
|
135
138
|
// Developer/audit modes: "all security patterns" or "all 8"
|
|
136
|
-
expect(content).toMatch(/securityPatternsToTest:\s*8
|
|
139
|
+
expect(content).toMatch(/securityPatternsToTest:\s*8,.*all.*8|8.*patterns/i);
|
|
137
140
|
});
|
|
138
141
|
it("should document pattern count in help text comments", () => {
|
|
139
142
|
const filePath = path.join(projectRoot, "cli/src/assess-security.ts");
|
|
@@ -6,9 +6,12 @@
|
|
|
6
6
|
*
|
|
7
7
|
* @see https://github.com/triepod-ai/inspector-assessment/issues/137
|
|
8
8
|
*/
|
|
9
|
-
import { describe, it, expect } from "@jest/globals";
|
|
9
|
+
import { jest, describe, it, expect, afterEach } from "@jest/globals";
|
|
10
10
|
import { AssessmentOptionsSchema, safeParseAssessmentOptions, validateAssessmentOptions, } from "../lib/cli-parserSchemas.js";
|
|
11
11
|
describe("Stage 3 Fix Validation Tests", () => {
|
|
12
|
+
afterEach(() => {
|
|
13
|
+
jest.clearAllMocks();
|
|
14
|
+
});
|
|
12
15
|
describe("[TEST-001] cli-parserSchemas.ts - stageBVerbose field (FIX-001)", () => {
|
|
13
16
|
describe("stageBVerbose field validation", () => {
|
|
14
17
|
it("should accept stageBVerbose with true value (happy path)", () => {
|
|
@@ -11,7 +11,14 @@
|
|
|
11
11
|
*
|
|
12
12
|
* Note: Tests skip gracefully when testbed servers are unavailable.
|
|
13
13
|
*/
|
|
14
|
-
import { describe, it, expect, beforeAll } from "@jest/globals";
|
|
14
|
+
import { jest, describe, it, expect, beforeAll, afterEach, } from "@jest/globals";
|
|
15
|
+
/**
|
|
16
|
+
* Skip integration tests unless RUN_E2E_TESTS is set.
|
|
17
|
+
* This prevents long timeouts when testbed servers aren't running.
|
|
18
|
+
*
|
|
19
|
+
* To run: RUN_E2E_TESTS=1 npm test -- --testPathPattern="testbed-integration"
|
|
20
|
+
*/
|
|
21
|
+
const describeE2E = process.env.RUN_E2E_TESTS ? describe : describe.skip;
|
|
15
22
|
// Testbed server URLs
|
|
16
23
|
const VULNERABLE_URL = "http://localhost:10900/mcp";
|
|
17
24
|
const HARDENED_URL = "http://localhost:10901/mcp";
|
|
@@ -26,9 +33,9 @@ const DEFAULT_HEADERS = {
|
|
|
26
33
|
* Check if a server is available by sending an initialize request
|
|
27
34
|
*/
|
|
28
35
|
async function checkServerAvailable(url) {
|
|
36
|
+
const controller = new AbortController();
|
|
37
|
+
const timeoutId = setTimeout(() => controller.abort(), 5000);
|
|
29
38
|
try {
|
|
30
|
-
const controller = new AbortController();
|
|
31
|
-
const timeoutId = setTimeout(() => controller.abort(), 5000);
|
|
32
39
|
const response = await fetch(url, {
|
|
33
40
|
method: "POST",
|
|
34
41
|
headers: DEFAULT_HEADERS,
|
|
@@ -44,12 +51,16 @@ async function checkServerAvailable(url) {
|
|
|
44
51
|
}),
|
|
45
52
|
signal: controller.signal,
|
|
46
53
|
});
|
|
47
|
-
clearTimeout(timeoutId);
|
|
48
54
|
return response.status < 500;
|
|
49
55
|
}
|
|
50
56
|
catch {
|
|
51
57
|
return false;
|
|
52
58
|
}
|
|
59
|
+
finally {
|
|
60
|
+
// Always clean up timeout and abort controller to prevent memory leaks
|
|
61
|
+
clearTimeout(timeoutId);
|
|
62
|
+
controller.abort();
|
|
63
|
+
}
|
|
53
64
|
}
|
|
54
65
|
/**
|
|
55
66
|
* Parse SSE response to extract JSON data
|
|
@@ -119,7 +130,10 @@ async function callTool(url, toolName, args) {
|
|
|
119
130
|
});
|
|
120
131
|
return data;
|
|
121
132
|
}
|
|
122
|
-
|
|
133
|
+
describeE2E("Testbed A/B Comparison", () => {
|
|
134
|
+
afterEach(() => {
|
|
135
|
+
jest.clearAllMocks();
|
|
136
|
+
});
|
|
123
137
|
let bothServersAvailable = false;
|
|
124
138
|
let vulnerableAvailable = false;
|
|
125
139
|
let hardenedAvailable = false;
|
|
@@ -5,9 +5,12 @@
|
|
|
5
5
|
* Tests focus on input validation and error handling rather than
|
|
6
6
|
* mocked transport implementations due to ESM limitations.
|
|
7
7
|
*/
|
|
8
|
-
import { describe, it, expect } from "@jest/globals";
|
|
8
|
+
import { jest, describe, it, expect, afterEach } from "@jest/globals";
|
|
9
9
|
import { createTransport } from "../transport.js";
|
|
10
10
|
describe("Transport Creation", () => {
|
|
11
|
+
afterEach(() => {
|
|
12
|
+
jest.clearAllMocks();
|
|
13
|
+
});
|
|
11
14
|
describe("Input Validation", () => {
|
|
12
15
|
it("should throw error when URL is missing for HTTP transport", () => {
|
|
13
16
|
const options = {
|
|
@@ -5,10 +5,13 @@
|
|
|
5
5
|
*
|
|
6
6
|
* @module cli/lib/__tests__/cli-parserSchemas
|
|
7
7
|
*/
|
|
8
|
-
|
|
8
|
+
import { jest, describe, test, expect, afterEach } from "@jest/globals";
|
|
9
9
|
import { ZodError } from "zod";
|
|
10
10
|
import { AssessmentProfileNameSchema, AssessmentModuleNameSchema, ServerConfigSchema, AssessmentOptionsSchema, ValidationResultSchema, validateAssessmentOptions, validateServerConfig, parseAssessmentOptions, safeParseAssessmentOptions, parseModuleNames, safeParseModuleNames, LogLevelSchema, ReportFormatSchema, TransportTypeSchema, ZOD_SCHEMA_VERSION, } from "../cli-parserSchemas.js";
|
|
11
11
|
describe("cli-parserSchemas", () => {
|
|
12
|
+
afterEach(() => {
|
|
13
|
+
jest.clearAllMocks();
|
|
14
|
+
});
|
|
12
15
|
describe("Re-exported schemas", () => {
|
|
13
16
|
test("exports ZOD_SCHEMA_VERSION", () => {
|
|
14
17
|
expect(ZOD_SCHEMA_VERSION).toBe(1);
|
|
@@ -5,10 +5,13 @@
|
|
|
5
5
|
*
|
|
6
6
|
* @module cli/lib/assessment-runner/__tests__/server-configSchemas
|
|
7
7
|
*/
|
|
8
|
-
|
|
8
|
+
import { jest, describe, test, expect, afterEach } from "@jest/globals";
|
|
9
9
|
import { ZodError } from "zod";
|
|
10
10
|
import { HttpSseServerConfigSchema, StdioServerConfigSchema, ServerEntrySchema, ClaudeDesktopConfigSchema, StandaloneConfigSchema, ConfigFileSchema, parseConfigFile, safeParseConfigFile, validateServerEntry, isHttpSseConfig, isStdioConfig, TransportTypeSchema, } from "../server-configSchemas.js";
|
|
11
11
|
describe("server-configSchemas", () => {
|
|
12
|
+
afterEach(() => {
|
|
13
|
+
jest.clearAllMocks();
|
|
14
|
+
});
|
|
12
15
|
describe("Re-exported schemas", () => {
|
|
13
16
|
test("exports TransportTypeSchema", () => {
|
|
14
17
|
expect(TransportTypeSchema.safeParse("stdio").success).toBe(true);
|
|
@@ -12,9 +12,12 @@ import { AssessmentStateManager } from "../../assessmentState.js";
|
|
|
12
12
|
import { emitServerConnected, emitToolDiscovered, emitToolsDiscoveryComplete, emitAssessmentComplete, emitTestBatch, emitVulnerabilityFound, emitAnnotationMissing, emitAnnotationMisaligned, emitAnnotationReviewRecommended, emitAnnotationAligned, emitModulesConfigured, emitPhaseStarted, emitPhaseComplete, emitToolTestComplete, emitValidationSummary, } from "../jsonl-events.js";
|
|
13
13
|
import { loadServerConfig } from "./server-config.js";
|
|
14
14
|
import { loadSourceFiles } from "./source-loader.js";
|
|
15
|
+
import { resolveSourcePath } from "./path-resolver.js";
|
|
15
16
|
import { connectToServer } from "./server-connection.js";
|
|
16
17
|
import { createCallToolWrapper } from "./tool-wrapper.js";
|
|
17
18
|
import { buildConfig } from "./config-builder.js";
|
|
19
|
+
// Issue #155: Import annotation debug mode setter
|
|
20
|
+
import { setAnnotationDebugMode } from "../../../../client/lib/services/assessment/modules/annotations/AlignmentChecker.js";
|
|
18
21
|
/**
|
|
19
22
|
* Run full assessment against an MCP server
|
|
20
23
|
*
|
|
@@ -22,6 +25,13 @@ import { buildConfig } from "./config-builder.js";
|
|
|
22
25
|
* @returns Assessment results
|
|
23
26
|
*/
|
|
24
27
|
export async function runFullAssessment(options) {
|
|
28
|
+
// Issue #155: Enable annotation debug mode if flag is set
|
|
29
|
+
if (options.debugAnnotations) {
|
|
30
|
+
setAnnotationDebugMode(true);
|
|
31
|
+
if (!options.jsonOnly) {
|
|
32
|
+
console.log("🔍 Annotation debug mode enabled (--debug-annotations)");
|
|
33
|
+
}
|
|
34
|
+
}
|
|
25
35
|
if (!options.jsonOnly) {
|
|
26
36
|
console.log(`\n🔍 Starting full assessment for: ${options.serverName}`);
|
|
27
37
|
}
|
|
@@ -199,10 +209,19 @@ export async function runFullAssessment(options) {
|
|
|
199
209
|
}
|
|
200
210
|
}
|
|
201
211
|
let sourceFiles = {};
|
|
202
|
-
if (options.sourceCodePath
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
212
|
+
if (options.sourceCodePath) {
|
|
213
|
+
// Resolve path using utility (handles ~, relative paths, symlinks)
|
|
214
|
+
const resolvedSourcePath = resolveSourcePath(options.sourceCodePath);
|
|
215
|
+
if (fs.existsSync(resolvedSourcePath)) {
|
|
216
|
+
sourceFiles = loadSourceFiles(resolvedSourcePath, options.debugSource);
|
|
217
|
+
if (!options.jsonOnly) {
|
|
218
|
+
console.log(`📁 Loaded source files from: ${resolvedSourcePath}`);
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
else if (!options.jsonOnly) {
|
|
222
|
+
// Issue #154: Always show warning, not just with --debug-source
|
|
223
|
+
console.log(`⚠️ Source path not found: ${options.sourceCodePath} (resolved: ${resolvedSourcePath})`);
|
|
224
|
+
console.log(` Use --source <existing-path> to enable full source file analysis.`);
|
|
206
225
|
}
|
|
207
226
|
}
|
|
208
227
|
// Create readResource wrapper for ResourceAssessor
|
|
@@ -10,6 +10,8 @@
|
|
|
10
10
|
export { loadServerConfig } from "./server-config.js";
|
|
11
11
|
// Source File Loading
|
|
12
12
|
export { loadSourceFiles } from "./source-loader.js";
|
|
13
|
+
// Path Resolution
|
|
14
|
+
export { resolveSourcePath } from "./path-resolver.js";
|
|
13
15
|
// Server Connection
|
|
14
16
|
export { connectToServer } from "./server-connection.js";
|
|
15
17
|
// Tool Wrapper
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Path Resolution Utilities
|
|
3
|
+
*
|
|
4
|
+
* Handles path normalization for source code paths including:
|
|
5
|
+
* - Home directory (~) expansion
|
|
6
|
+
* - Relative path resolution
|
|
7
|
+
* - Symlink resolution (for MCPB temp extraction paths)
|
|
8
|
+
*
|
|
9
|
+
* @module cli/lib/assessment-runner/path-resolver
|
|
10
|
+
*/
|
|
11
|
+
import * as fs from "fs";
|
|
12
|
+
import * as os from "os";
|
|
13
|
+
import * as path from "path";
|
|
14
|
+
/**
|
|
15
|
+
* Resolve a source code path to an absolute, real path
|
|
16
|
+
*
|
|
17
|
+
* Handles:
|
|
18
|
+
* - Tilde (~) expansion to home directory
|
|
19
|
+
* - Relative paths resolved to absolute
|
|
20
|
+
* - Symlinks followed to real paths (important for MCPB temp extraction)
|
|
21
|
+
*
|
|
22
|
+
* @param sourcePath - The source path to resolve (may be relative, contain ~, or be a symlink)
|
|
23
|
+
* @returns The resolved absolute path, or the original path if resolution fails
|
|
24
|
+
*
|
|
25
|
+
* @example
|
|
26
|
+
* resolveSourcePath("~/project") // => "/home/user/project"
|
|
27
|
+
* resolveSourcePath("./src") // => "/current/working/dir/src"
|
|
28
|
+
* resolveSourcePath("/tmp/symlink") // => "/actual/target/path"
|
|
29
|
+
*/
|
|
30
|
+
export function resolveSourcePath(sourcePath) {
|
|
31
|
+
let resolved = sourcePath;
|
|
32
|
+
// Expand home directory (~)
|
|
33
|
+
if (resolved.startsWith("~")) {
|
|
34
|
+
resolved = path.join(os.homedir(), resolved.slice(1));
|
|
35
|
+
}
|
|
36
|
+
// Resolve to absolute path
|
|
37
|
+
resolved = path.resolve(resolved);
|
|
38
|
+
// Follow symlinks if path exists (handles MCPB temp extraction paths)
|
|
39
|
+
try {
|
|
40
|
+
if (fs.existsSync(resolved)) {
|
|
41
|
+
resolved = fs.realpathSync(resolved);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
catch {
|
|
45
|
+
// realpathSync can fail on broken symlinks, continue with resolved path
|
|
46
|
+
}
|
|
47
|
+
return resolved;
|
|
48
|
+
}
|
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
* @module cli/lib/assessment-runner/source-loader
|
|
7
7
|
*/
|
|
8
8
|
import * as fs from "fs";
|
|
9
|
+
import * as os from "os";
|
|
9
10
|
import * as path from "path";
|
|
10
11
|
/** Maximum file size (in characters) to include in source code analysis */
|
|
11
12
|
const MAX_SOURCE_FILE_SIZE = 100_000;
|
|
@@ -13,34 +14,65 @@ const MAX_SOURCE_FILE_SIZE = 100_000;
|
|
|
13
14
|
* Load optional files from source code path
|
|
14
15
|
*
|
|
15
16
|
* @param sourcePath - Path to source code directory
|
|
17
|
+
* @param debug - Enable debug logging for path resolution troubleshooting
|
|
16
18
|
* @returns Object containing loaded source files
|
|
17
19
|
*/
|
|
18
|
-
export function loadSourceFiles(sourcePath) {
|
|
20
|
+
export function loadSourceFiles(sourcePath, debug = false) {
|
|
19
21
|
const result = {};
|
|
22
|
+
// Debug logging helper - masks home directory for privacy in logs
|
|
23
|
+
const log = (msg) => {
|
|
24
|
+
if (!debug)
|
|
25
|
+
return;
|
|
26
|
+
const maskedMsg = msg.replace(os.homedir(), "~");
|
|
27
|
+
console.log(`[source-loader] ${maskedMsg}`);
|
|
28
|
+
};
|
|
29
|
+
log(`Starting source file loading from: ${sourcePath}`);
|
|
20
30
|
// Search for README in source directory and parent directories (up to 3 levels)
|
|
21
31
|
// This handles cases where --source points to a subdirectory but README is at repo root
|
|
22
|
-
|
|
32
|
+
// Extended patterns to handle various README naming conventions
|
|
33
|
+
const readmePaths = [
|
|
34
|
+
"README.md",
|
|
35
|
+
"readme.md",
|
|
36
|
+
"Readme.md",
|
|
37
|
+
"README.markdown",
|
|
38
|
+
"readme.markdown",
|
|
39
|
+
"README.txt",
|
|
40
|
+
"readme.txt",
|
|
41
|
+
"README",
|
|
42
|
+
"Readme",
|
|
43
|
+
];
|
|
23
44
|
let readmeFound = false;
|
|
45
|
+
log(`Searching for README variants: ${readmePaths.join(", ")}`);
|
|
24
46
|
// First try the source directory itself
|
|
25
47
|
for (const readmePath of readmePaths) {
|
|
26
48
|
const fullPath = path.join(sourcePath, readmePath);
|
|
27
|
-
|
|
49
|
+
const exists = fs.existsSync(fullPath);
|
|
50
|
+
log(` Checking: ${fullPath} - exists: ${exists}`);
|
|
51
|
+
if (exists) {
|
|
28
52
|
result.readmeContent = fs.readFileSync(fullPath, "utf-8");
|
|
53
|
+
log(` ✓ Found README: ${fullPath} (${result.readmeContent.length} bytes)`);
|
|
29
54
|
readmeFound = true;
|
|
30
55
|
break;
|
|
31
56
|
}
|
|
32
57
|
}
|
|
33
58
|
// If not found, search parent directories (up to 3 levels)
|
|
34
59
|
if (!readmeFound) {
|
|
60
|
+
log(`README not found in source directory, searching parent directories...`);
|
|
35
61
|
let currentDir = sourcePath;
|
|
36
62
|
for (let i = 0; i < 3; i++) {
|
|
37
63
|
const parentDir = path.dirname(currentDir);
|
|
38
|
-
if (parentDir === currentDir)
|
|
64
|
+
if (parentDir === currentDir) {
|
|
65
|
+
log(` Reached filesystem root, stopping parent search`);
|
|
39
66
|
break; // Reached filesystem root
|
|
67
|
+
}
|
|
68
|
+
log(` Searching parent level ${i + 1}: ${parentDir}`);
|
|
40
69
|
for (const readmePath of readmePaths) {
|
|
41
70
|
const fullPath = path.join(parentDir, readmePath);
|
|
42
|
-
|
|
71
|
+
const exists = fs.existsSync(fullPath);
|
|
72
|
+
log(` Checking: ${fullPath} - exists: ${exists}`);
|
|
73
|
+
if (exists) {
|
|
43
74
|
result.readmeContent = fs.readFileSync(fullPath, "utf-8");
|
|
75
|
+
log(` ✓ Found README: ${fullPath} (${result.readmeContent.length} bytes)`);
|
|
44
76
|
readmeFound = true;
|
|
45
77
|
break;
|
|
46
78
|
}
|
|
@@ -50,6 +82,9 @@ export function loadSourceFiles(sourcePath) {
|
|
|
50
82
|
currentDir = parentDir;
|
|
51
83
|
}
|
|
52
84
|
}
|
|
85
|
+
if (!readmeFound) {
|
|
86
|
+
log(`✗ No README found in source directory or parent directories`);
|
|
87
|
+
}
|
|
53
88
|
const packagePath = path.join(sourcePath, "package.json");
|
|
54
89
|
if (fs.existsSync(packagePath)) {
|
|
55
90
|
result.packageJson = JSON.parse(fs.readFileSync(packagePath, "utf-8"));
|
|
@@ -135,5 +170,12 @@ export function loadSourceFiles(sourcePath) {
|
|
|
135
170
|
catch (e) {
|
|
136
171
|
console.warn("[Assessment] Could not load source files:", e);
|
|
137
172
|
}
|
|
173
|
+
// Summary logging
|
|
174
|
+
const sourceCodeFiles = result.sourceCodeFiles;
|
|
175
|
+
log(`Source loading complete:`);
|
|
176
|
+
log(` - README: ${result.readmeContent ? "found" : "not found"}`);
|
|
177
|
+
log(` - package.json: ${result.packageJson ? "found" : "not found"}`);
|
|
178
|
+
log(` - manifest.json: ${result.manifestJson ? "found" : "not found"}`);
|
|
179
|
+
log(` - Source files loaded: ${sourceCodeFiles.size}`);
|
|
138
180
|
return result;
|
|
139
181
|
}
|
package/build/lib/cli-parser.js
CHANGED
|
@@ -94,6 +94,13 @@ export function parseArgs(argv) {
|
|
|
94
94
|
case "--source":
|
|
95
95
|
options.sourceCodePath = args[++i];
|
|
96
96
|
break;
|
|
97
|
+
case "--debug-source":
|
|
98
|
+
options.debugSource = true;
|
|
99
|
+
break;
|
|
100
|
+
case "--debug-annotations":
|
|
101
|
+
// Issue #155: Enable debug logging for annotation extraction
|
|
102
|
+
options.debugAnnotations = true;
|
|
103
|
+
break;
|
|
97
104
|
case "--pattern-config":
|
|
98
105
|
case "-p":
|
|
99
106
|
options.patternConfigPath = args[++i];
|
|
@@ -374,6 +381,9 @@ Options:
|
|
|
374
381
|
--config, -c <path> Path to server config JSON
|
|
375
382
|
--output, -o <path> Output path (default: /tmp/inspector-full-assessment-<server>.<ext>)
|
|
376
383
|
--source <path> Source code path for deep analysis (AUP, portability, etc.)
|
|
384
|
+
--debug-source Enable debug logging for source file loading (Issue #151)
|
|
385
|
+
--debug-annotations Enable debug logging for annotation extraction (Issue #155)
|
|
386
|
+
Shows raw tool keys, annotations object, and direct hints
|
|
377
387
|
--pattern-config, -p <path> Path to custom annotation pattern JSON
|
|
378
388
|
--performance-config <path> Path to performance tuning JSON (batch sizes, timeouts, etc.)
|
|
379
389
|
--format, -f <type> Output format: json (default) or markdown
|
package/package.json
CHANGED