@bryan-thompson/inspector-assessment 1.35.3 → 1.36.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli/build/__tests__/assess-full-e2e.test.js +12 -2
- package/cli/build/__tests__/assess-full.test.js +1 -1
- package/cli/build/__tests__/assessment-runner/assessment-executor.test.js +1 -1
- package/cli/build/__tests__/assessment-runner/index.test.js +6 -4
- package/cli/build/__tests__/assessment-runner/path-resolver.test.js +112 -0
- package/cli/build/__tests__/assessment-runner/source-loader.test.js +92 -0
- package/cli/build/__tests__/assessment-runner-facade.test.js +4 -2
- package/cli/build/__tests__/cli-build-fixes.test.js +1 -1
- package/cli/build/__tests__/http-transport-integration.test.js +9 -2
- package/cli/build/__tests__/jsonl-events.test.js +1 -1
- package/cli/build/__tests__/security/security-pattern-count.test.js +3 -3
- package/cli/build/__tests__/stage3-fix-validation.test.js +1 -1
- package/cli/build/__tests__/testbed-integration.test.js +9 -2
- package/cli/build/__tests__/transport.test.js +1 -1
- package/cli/build/lib/__tests__/cli-parserSchemas.test.js +1 -1
- package/cli/build/lib/assessment-runner/__tests__/server-configSchemas.test.js +1 -1
- package/cli/build/lib/assessment-runner/assessment-executor.js +23 -4
- package/cli/build/lib/assessment-runner/index.js +2 -0
- package/cli/build/lib/assessment-runner/path-resolver.js +48 -0
- package/cli/build/lib/assessment-runner/source-loader.js +47 -5
- package/cli/build/lib/cli-parser.js +10 -0
- package/cli/package.json +1 -1
- package/client/dist/assets/{OAuthCallback-BglPlsRX.js → OAuthCallback-DHYVVs-U.js} +1 -1
- package/client/dist/assets/{OAuthDebugCallback-CWrUH2KB.js → OAuthDebugCallback-BrarwoAx.js} +1 -1
- package/client/dist/assets/{index-iCeeS8vg.js → index-Dup6F6QC.js} +4 -4
- package/client/dist/index.html +1 -1
- package/client/lib/lib/assessment/extendedTypes.d.ts +4 -0
- package/client/lib/lib/assessment/extendedTypes.d.ts.map +1 -1
- package/client/lib/lib/assessment/jsonlEventSchemas.d.ts +4 -4
- package/client/lib/lib/assessment/resultTypes.d.ts +22 -0
- package/client/lib/lib/assessment/resultTypes.d.ts.map +1 -1
- package/client/lib/lib/moduleScoring.d.ts.map +1 -1
- package/client/lib/lib/moduleScoring.js +45 -2
- package/client/lib/services/assessment/modules/ErrorHandlingAssessor.d.ts +1 -0
- package/client/lib/services/assessment/modules/ErrorHandlingAssessor.d.ts.map +1 -1
- package/client/lib/services/assessment/modules/ErrorHandlingAssessor.js +103 -8
- package/client/lib/services/assessment/modules/ProhibitedLibrariesAssessor.d.ts +5 -0
- package/client/lib/services/assessment/modules/ProhibitedLibrariesAssessor.d.ts.map +1 -1
- package/client/lib/services/assessment/modules/ProhibitedLibrariesAssessor.js +29 -0
- package/client/lib/services/assessment/modules/SecurityAssessor.d.ts.map +1 -1
- package/client/lib/services/assessment/modules/SecurityAssessor.js +13 -0
- package/client/lib/services/assessment/modules/annotations/AlignmentChecker.d.ts +7 -2
- package/client/lib/services/assessment/modules/annotations/AlignmentChecker.d.ts.map +1 -1
- package/client/lib/services/assessment/modules/annotations/AlignmentChecker.js +116 -18
- package/client/lib/services/assessment/modules/annotations/index.d.ts +1 -1
- package/client/lib/services/assessment/modules/annotations/index.d.ts.map +1 -1
- package/client/lib/services/assessment/modules/annotations/index.js +2 -1
- package/client/package.json +1 -1
- package/package.json +1 -1
- package/server/package.json +1 -1
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
*
|
|
15
15
|
* @see https://github.com/triepod-ai/inspector-assessment/issues/97
|
|
16
16
|
*/
|
|
17
|
-
import { describe, it, expect, beforeAll, afterAll } from "@jest/globals";
|
|
17
|
+
import { jest, describe, it, expect, beforeAll, afterAll, afterEach, } from "@jest/globals";
|
|
18
18
|
import { spawn } from "child_process";
|
|
19
19
|
import * as fs from "fs";
|
|
20
20
|
import * as path from "path";
|
|
@@ -248,9 +248,19 @@ function createInvalidConfig(content, filename) {
|
|
|
248
248
|
return configPath;
|
|
249
249
|
}
|
|
250
250
|
// ============================================================================
|
|
251
|
+
// E2E Skip Check
|
|
252
|
+
// ============================================================================
|
|
253
|
+
/**
|
|
254
|
+
* Skip E2E tests unless RUN_E2E_TESTS is set.
|
|
255
|
+
* This prevents long timeouts when testbed servers aren't running.
|
|
256
|
+
*
|
|
257
|
+
* To run E2E tests: RUN_E2E_TESTS=1 npm test -- --testPathPattern="e2e"
|
|
258
|
+
*/
|
|
259
|
+
const describeE2E = process.env.RUN_E2E_TESTS ? describe : describe.skip;
|
|
260
|
+
// ============================================================================
|
|
251
261
|
// Test Setup
|
|
252
262
|
// ============================================================================
|
|
253
|
-
|
|
263
|
+
describeE2E("CLI E2E Integration Tests", () => {
|
|
254
264
|
afterEach(() => {
|
|
255
265
|
jest.clearAllMocks();
|
|
256
266
|
});
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
* external dependencies. For integration testing of the full CLI, use the
|
|
7
7
|
* actual CLI binary.
|
|
8
8
|
*/
|
|
9
|
-
import { describe, it, expect } from "@jest/globals";
|
|
9
|
+
import { jest, describe, it, expect, afterEach } from "@jest/globals";
|
|
10
10
|
import * as path from "path";
|
|
11
11
|
/**
|
|
12
12
|
* Pure function tests - these test logic concepts used in the CLI
|
|
@@ -216,7 +216,7 @@ describe("runFullAssessment", () => {
|
|
|
216
216
|
...defaultOptions,
|
|
217
217
|
sourceCodePath: "/path/to/source",
|
|
218
218
|
});
|
|
219
|
-
expect(mockLoadSourceFiles).toHaveBeenCalledWith("/path/to/source");
|
|
219
|
+
expect(mockLoadSourceFiles).toHaveBeenCalledWith("/path/to/source", undefined);
|
|
220
220
|
});
|
|
221
221
|
it("should not load source files when path does not exist", async () => {
|
|
222
222
|
const fs = await import("fs");
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Tests for the facade module exports.
|
|
5
5
|
*/
|
|
6
|
-
import { describe, it, expect } from "@jest/globals";
|
|
6
|
+
import { jest, describe, it, expect, afterEach } from "@jest/globals";
|
|
7
7
|
// Import the barrel/facade module
|
|
8
8
|
import * as assessmentRunner from "../../lib/assessment-runner/index.js";
|
|
9
9
|
describe("assessment-runner index exports", () => {
|
|
@@ -11,24 +11,26 @@ describe("assessment-runner index exports", () => {
|
|
|
11
11
|
jest.clearAllMocks();
|
|
12
12
|
});
|
|
13
13
|
describe("function exports", () => {
|
|
14
|
-
it("should export all
|
|
14
|
+
it("should export all 7 public functions", () => {
|
|
15
15
|
expect(typeof assessmentRunner.loadServerConfig).toBe("function");
|
|
16
16
|
expect(typeof assessmentRunner.loadSourceFiles).toBe("function");
|
|
17
|
+
expect(typeof assessmentRunner.resolveSourcePath).toBe("function");
|
|
17
18
|
expect(typeof assessmentRunner.connectToServer).toBe("function");
|
|
18
19
|
expect(typeof assessmentRunner.createCallToolWrapper).toBe("function");
|
|
19
20
|
expect(typeof assessmentRunner.buildConfig).toBe("function");
|
|
20
21
|
expect(typeof assessmentRunner.runFullAssessment).toBe("function");
|
|
21
22
|
});
|
|
22
|
-
it("should export exactly
|
|
23
|
+
it("should export exactly 7 functions", () => {
|
|
23
24
|
const functionNames = Object.keys(assessmentRunner).filter((key) => typeof assessmentRunner[key] ===
|
|
24
25
|
"function");
|
|
25
|
-
expect(functionNames).toHaveLength(
|
|
26
|
+
expect(functionNames).toHaveLength(7);
|
|
26
27
|
expect(functionNames.sort()).toEqual([
|
|
27
28
|
"buildConfig",
|
|
28
29
|
"connectToServer",
|
|
29
30
|
"createCallToolWrapper",
|
|
30
31
|
"loadServerConfig",
|
|
31
32
|
"loadSourceFiles",
|
|
33
|
+
"resolveSourcePath",
|
|
32
34
|
"runFullAssessment",
|
|
33
35
|
]);
|
|
34
36
|
});
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Path Resolver Unit Tests
|
|
3
|
+
*
|
|
4
|
+
* Tests for resolveSourcePath() that handles path normalization.
|
|
5
|
+
*/
|
|
6
|
+
import { jest, describe, it, expect, beforeEach, afterAll, } from "@jest/globals";
|
|
7
|
+
import * as path from "path";
|
|
8
|
+
import * as os from "os";
|
|
9
|
+
// Mock fs module
|
|
10
|
+
jest.unstable_mockModule("fs", () => ({
|
|
11
|
+
existsSync: jest.fn(),
|
|
12
|
+
realpathSync: jest.fn(),
|
|
13
|
+
}));
|
|
14
|
+
// Import after mocking
|
|
15
|
+
const fs = await import("fs");
|
|
16
|
+
const { resolveSourcePath } = await import("../../lib/assessment-runner/path-resolver.js");
|
|
17
|
+
describe("resolveSourcePath", () => {
|
|
18
|
+
const mockExistsSync = fs.existsSync;
|
|
19
|
+
const mockRealpathSync = fs.realpathSync;
|
|
20
|
+
beforeEach(() => {
|
|
21
|
+
mockExistsSync.mockReturnValue(false);
|
|
22
|
+
mockRealpathSync.mockReturnValue("");
|
|
23
|
+
});
|
|
24
|
+
afterEach(() => {
|
|
25
|
+
jest.clearAllMocks();
|
|
26
|
+
});
|
|
27
|
+
afterAll(() => {
|
|
28
|
+
jest.unmock("fs");
|
|
29
|
+
});
|
|
30
|
+
describe("tilde expansion", () => {
|
|
31
|
+
it("should expand ~ to home directory", () => {
|
|
32
|
+
mockExistsSync.mockReturnValue(false);
|
|
33
|
+
const result = resolveSourcePath("~/project");
|
|
34
|
+
expect(result).toBe(path.join(os.homedir(), "project"));
|
|
35
|
+
});
|
|
36
|
+
it("should expand ~/subdir/path correctly", () => {
|
|
37
|
+
mockExistsSync.mockReturnValue(false);
|
|
38
|
+
const result = resolveSourcePath("~/foo/bar/baz");
|
|
39
|
+
expect(result).toBe(path.join(os.homedir(), "foo/bar/baz"));
|
|
40
|
+
});
|
|
41
|
+
it("should not modify paths without tilde", () => {
|
|
42
|
+
mockExistsSync.mockReturnValue(false);
|
|
43
|
+
const result = resolveSourcePath("/absolute/path");
|
|
44
|
+
expect(result).toBe("/absolute/path");
|
|
45
|
+
});
|
|
46
|
+
});
|
|
47
|
+
describe("relative path resolution", () => {
|
|
48
|
+
it("should resolve relative paths to absolute", () => {
|
|
49
|
+
mockExistsSync.mockReturnValue(false);
|
|
50
|
+
const cwd = process.cwd();
|
|
51
|
+
const result = resolveSourcePath("./src");
|
|
52
|
+
expect(result).toBe(path.resolve(cwd, "./src"));
|
|
53
|
+
});
|
|
54
|
+
it("should resolve parent directory references", () => {
|
|
55
|
+
mockExistsSync.mockReturnValue(false);
|
|
56
|
+
const cwd = process.cwd();
|
|
57
|
+
const result = resolveSourcePath("../sibling");
|
|
58
|
+
expect(result).toBe(path.resolve(cwd, "../sibling"));
|
|
59
|
+
});
|
|
60
|
+
it("should handle bare directory names", () => {
|
|
61
|
+
mockExistsSync.mockReturnValue(false);
|
|
62
|
+
const cwd = process.cwd();
|
|
63
|
+
const result = resolveSourcePath("mydir");
|
|
64
|
+
expect(result).toBe(path.resolve(cwd, "mydir"));
|
|
65
|
+
});
|
|
66
|
+
});
|
|
67
|
+
describe("symlink resolution", () => {
|
|
68
|
+
it("should follow symlinks when path exists", () => {
|
|
69
|
+
const symlinkPath = "/tmp/symlink";
|
|
70
|
+
const realPath = "/actual/target/path";
|
|
71
|
+
mockExistsSync.mockImplementation((p) => p === symlinkPath);
|
|
72
|
+
mockRealpathSync.mockReturnValue(realPath);
|
|
73
|
+
const result = resolveSourcePath(symlinkPath);
|
|
74
|
+
expect(mockRealpathSync).toHaveBeenCalledWith(symlinkPath);
|
|
75
|
+
expect(result).toBe(realPath);
|
|
76
|
+
});
|
|
77
|
+
it("should not call realpathSync when path does not exist", () => {
|
|
78
|
+
mockExistsSync.mockReturnValue(false);
|
|
79
|
+
resolveSourcePath("/nonexistent/path");
|
|
80
|
+
expect(mockRealpathSync).not.toHaveBeenCalled();
|
|
81
|
+
});
|
|
82
|
+
it("should handle broken symlinks gracefully", () => {
|
|
83
|
+
const brokenSymlink = "/tmp/broken-symlink";
|
|
84
|
+
mockExistsSync.mockReturnValue(true);
|
|
85
|
+
mockRealpathSync.mockImplementation(() => {
|
|
86
|
+
throw new Error("ENOENT: no such file or directory");
|
|
87
|
+
});
|
|
88
|
+
// Should not throw, should return the resolved path without realpath
|
|
89
|
+
const result = resolveSourcePath(brokenSymlink);
|
|
90
|
+
expect(result).toBe(brokenSymlink);
|
|
91
|
+
});
|
|
92
|
+
});
|
|
93
|
+
describe("combined scenarios", () => {
|
|
94
|
+
it("should handle ~ with symlink resolution", () => {
|
|
95
|
+
const tildePathExpanded = path.join(os.homedir(), "project");
|
|
96
|
+
const realPath = "/real/project/path";
|
|
97
|
+
mockExistsSync.mockImplementation((p) => p === tildePathExpanded);
|
|
98
|
+
mockRealpathSync.mockReturnValue(realPath);
|
|
99
|
+
const result = resolveSourcePath("~/project");
|
|
100
|
+
expect(result).toBe(realPath);
|
|
101
|
+
});
|
|
102
|
+
it("should handle relative path with symlink resolution", () => {
|
|
103
|
+
const cwd = process.cwd();
|
|
104
|
+
const resolvedRelative = path.resolve(cwd, "./src");
|
|
105
|
+
const realPath = "/real/src/path";
|
|
106
|
+
mockExistsSync.mockImplementation((p) => p === resolvedRelative);
|
|
107
|
+
mockRealpathSync.mockReturnValue(realPath);
|
|
108
|
+
const result = resolveSourcePath("./src");
|
|
109
|
+
expect(result).toBe(realPath);
|
|
110
|
+
});
|
|
111
|
+
});
|
|
112
|
+
});
|
|
@@ -80,6 +80,98 @@ describe("loadSourceFiles", () => {
|
|
|
80
80
|
const result = loadSourceFiles(sourcePath);
|
|
81
81
|
expect(result.readmeContent).toBeUndefined();
|
|
82
82
|
});
|
|
83
|
+
// Issue #151: Extended README patterns
|
|
84
|
+
it("should find README.markdown", () => {
|
|
85
|
+
const sourcePath = "/project";
|
|
86
|
+
mockExistsSync.mockImplementation((p) => p === path.join(sourcePath, "README.markdown"));
|
|
87
|
+
mockReadFileSync.mockReturnValue("# Markdown README");
|
|
88
|
+
const result = loadSourceFiles(sourcePath);
|
|
89
|
+
expect(result.readmeContent).toBe("# Markdown README");
|
|
90
|
+
});
|
|
91
|
+
it("should find README.txt", () => {
|
|
92
|
+
const sourcePath = "/project";
|
|
93
|
+
mockExistsSync.mockImplementation((p) => p === path.join(sourcePath, "README.txt"));
|
|
94
|
+
mockReadFileSync.mockReturnValue("Text README content");
|
|
95
|
+
const result = loadSourceFiles(sourcePath);
|
|
96
|
+
expect(result.readmeContent).toBe("Text README content");
|
|
97
|
+
});
|
|
98
|
+
it("should find README without extension", () => {
|
|
99
|
+
const sourcePath = "/project";
|
|
100
|
+
mockExistsSync.mockImplementation((p) => p === path.join(sourcePath, "README"));
|
|
101
|
+
mockReadFileSync.mockReturnValue("No extension README");
|
|
102
|
+
const result = loadSourceFiles(sourcePath);
|
|
103
|
+
expect(result.readmeContent).toBe("No extension README");
|
|
104
|
+
});
|
|
105
|
+
it("should prioritize README.md over other variants", () => {
|
|
106
|
+
const sourcePath = "/project";
|
|
107
|
+
// Multiple README files exist - README.md should be found first
|
|
108
|
+
mockExistsSync.mockImplementation((p) => {
|
|
109
|
+
return (p === path.join(sourcePath, "README.md") ||
|
|
110
|
+
p === path.join(sourcePath, "README.txt") ||
|
|
111
|
+
p === path.join(sourcePath, "README"));
|
|
112
|
+
});
|
|
113
|
+
mockReadFileSync.mockImplementation((p) => {
|
|
114
|
+
if (p.endsWith("README.md"))
|
|
115
|
+
return "# MD README";
|
|
116
|
+
if (p.endsWith("README.txt"))
|
|
117
|
+
return "TXT README";
|
|
118
|
+
if (p.endsWith("README"))
|
|
119
|
+
return "Plain README";
|
|
120
|
+
return "";
|
|
121
|
+
});
|
|
122
|
+
const result = loadSourceFiles(sourcePath);
|
|
123
|
+
expect(result.readmeContent).toBe("# MD README");
|
|
124
|
+
});
|
|
125
|
+
});
|
|
126
|
+
describe("debug logging", () => {
|
|
127
|
+
it("should log debug output when debug flag is true", () => {
|
|
128
|
+
const sourcePath = "/project";
|
|
129
|
+
const consoleSpy = jest
|
|
130
|
+
.spyOn(console, "log")
|
|
131
|
+
.mockImplementation(() => { });
|
|
132
|
+
try {
|
|
133
|
+
mockExistsSync.mockReturnValue(false);
|
|
134
|
+
mockReaddirSync.mockReturnValue([]);
|
|
135
|
+
loadSourceFiles(sourcePath, true);
|
|
136
|
+
// Verify debug logging was called
|
|
137
|
+
expect(consoleSpy).toHaveBeenCalledWith(expect.stringContaining("[source-loader]"));
|
|
138
|
+
}
|
|
139
|
+
finally {
|
|
140
|
+
consoleSpy.mockRestore();
|
|
141
|
+
}
|
|
142
|
+
});
|
|
143
|
+
it("should not log debug output when debug flag is false", () => {
|
|
144
|
+
const sourcePath = "/project";
|
|
145
|
+
const consoleSpy = jest
|
|
146
|
+
.spyOn(console, "log")
|
|
147
|
+
.mockImplementation(() => { });
|
|
148
|
+
try {
|
|
149
|
+
mockExistsSync.mockReturnValue(false);
|
|
150
|
+
mockReaddirSync.mockReturnValue([]);
|
|
151
|
+
loadSourceFiles(sourcePath, false);
|
|
152
|
+
// Verify debug logging was NOT called
|
|
153
|
+
expect(consoleSpy).not.toHaveBeenCalledWith(expect.stringContaining("[source-loader]"));
|
|
154
|
+
}
|
|
155
|
+
finally {
|
|
156
|
+
consoleSpy.mockRestore();
|
|
157
|
+
}
|
|
158
|
+
});
|
|
159
|
+
it("should not log debug output by default", () => {
|
|
160
|
+
const sourcePath = "/project";
|
|
161
|
+
const consoleSpy = jest
|
|
162
|
+
.spyOn(console, "log")
|
|
163
|
+
.mockImplementation(() => { });
|
|
164
|
+
try {
|
|
165
|
+
mockExistsSync.mockReturnValue(false);
|
|
166
|
+
mockReaddirSync.mockReturnValue([]);
|
|
167
|
+
loadSourceFiles(sourcePath);
|
|
168
|
+
// Verify debug logging was NOT called by default
|
|
169
|
+
expect(consoleSpy).not.toHaveBeenCalledWith(expect.stringContaining("[source-loader]"));
|
|
170
|
+
}
|
|
171
|
+
finally {
|
|
172
|
+
consoleSpy.mockRestore();
|
|
173
|
+
}
|
|
174
|
+
});
|
|
83
175
|
});
|
|
84
176
|
describe("package.json parsing", () => {
|
|
85
177
|
it("should parse package.json when present", () => {
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
*
|
|
7
7
|
* @see https://github.com/triepod-ai/inspector-assessment/issues/96
|
|
8
8
|
*/
|
|
9
|
-
import { describe, it, expect } from "@jest/globals";
|
|
9
|
+
import { jest, describe, it, expect, afterEach } from "@jest/globals";
|
|
10
10
|
// Test named imports (the primary consumer pattern)
|
|
11
11
|
import { loadServerConfig, loadSourceFiles, connectToServer, createCallToolWrapper, buildConfig, runFullAssessment, } from "../lib/assessment-runner.js";
|
|
12
12
|
// Test namespace import
|
|
@@ -84,13 +84,14 @@ describe("Assessment Runner Facade", () => {
|
|
|
84
84
|
describe("Export Completeness", () => {
|
|
85
85
|
it("should export exactly 6 functions", () => {
|
|
86
86
|
const exportedFunctions = Object.entries(AssessmentRunner).filter(([, value]) => typeof value === "function");
|
|
87
|
-
expect(exportedFunctions.length).toBe(
|
|
87
|
+
expect(exportedFunctions.length).toBe(7);
|
|
88
88
|
expect(exportedFunctions.map(([name]) => name).sort()).toEqual([
|
|
89
89
|
"buildConfig",
|
|
90
90
|
"connectToServer",
|
|
91
91
|
"createCallToolWrapper",
|
|
92
92
|
"loadServerConfig",
|
|
93
93
|
"loadSourceFiles",
|
|
94
|
+
"resolveSourcePath",
|
|
94
95
|
"runFullAssessment",
|
|
95
96
|
]);
|
|
96
97
|
});
|
|
@@ -99,6 +100,7 @@ describe("Assessment Runner Facade", () => {
|
|
|
99
100
|
const expectedExports = [
|
|
100
101
|
"loadServerConfig",
|
|
101
102
|
"loadSourceFiles",
|
|
103
|
+
"resolveSourcePath",
|
|
102
104
|
"connectToServer",
|
|
103
105
|
"createCallToolWrapper",
|
|
104
106
|
"buildConfig",
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
* @see https://github.com/triepod-ai/inspector-assessment/issues/33
|
|
9
9
|
* @see https://github.com/triepod-ai/inspector-assessment/issues/37
|
|
10
10
|
*/
|
|
11
|
-
import { describe, it, expect } from "@jest/globals";
|
|
11
|
+
import { jest, describe, it, expect, afterEach } from "@jest/globals";
|
|
12
12
|
import { ScopedListenerConfig } from "../lib/event-config.js";
|
|
13
13
|
describe("CLI Build Fixes Regression Tests", () => {
|
|
14
14
|
afterEach(() => {
|
|
@@ -7,8 +7,15 @@
|
|
|
7
7
|
*
|
|
8
8
|
* Note: Tests skip gracefully when testbed servers are unavailable.
|
|
9
9
|
*/
|
|
10
|
-
import { describe, it, expect, beforeAll } from "@jest/globals";
|
|
10
|
+
import { jest, describe, it, expect, beforeAll, afterEach, } from "@jest/globals";
|
|
11
|
+
/**
|
|
12
|
+
* Skip integration tests unless RUN_E2E_TESTS is set.
|
|
13
|
+
* This prevents long timeouts when testbed servers aren't running.
|
|
14
|
+
*
|
|
15
|
+
* To run: RUN_E2E_TESTS=1 npm test -- --testPathPattern="http-transport-integration"
|
|
16
|
+
*/
|
|
11
17
|
import { createTransport } from "../transport.js";
|
|
18
|
+
const describeE2E = process.env.RUN_E2E_TESTS ? describe : describe.skip;
|
|
12
19
|
// Testbed server URLs
|
|
13
20
|
const VULNERABLE_MCP_URL = "http://localhost:10900/mcp";
|
|
14
21
|
const HARDENED_MCP_URL = "http://localhost:10901/mcp";
|
|
@@ -104,7 +111,7 @@ async function sendMcpRequest(url, method, params = {}, headers = {}) {
|
|
|
104
111
|
}
|
|
105
112
|
return { response, data };
|
|
106
113
|
}
|
|
107
|
-
|
|
114
|
+
describeE2E("HTTP Transport Integration", () => {
|
|
108
115
|
afterEach(() => {
|
|
109
116
|
jest.clearAllMocks();
|
|
110
117
|
});
|
|
@@ -40,7 +40,7 @@ describe("JSONL Event Emission", () => {
|
|
|
40
40
|
});
|
|
41
41
|
it("should include schemaVersion field", () => {
|
|
42
42
|
emitJSONL({ event: "test" });
|
|
43
|
-
expect(emittedEvents[0]).toHaveProperty("schemaVersion",
|
|
43
|
+
expect(emittedEvents[0]).toHaveProperty("schemaVersion", 3);
|
|
44
44
|
});
|
|
45
45
|
it("should handle complex nested objects", () => {
|
|
46
46
|
emitJSONL({
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
* CLI tools override this to 30 patterns for comprehensive security assessment.
|
|
14
14
|
* This test verifies consistency within each context.
|
|
15
15
|
*/
|
|
16
|
-
import { describe, it, expect } from "@jest/globals";
|
|
16
|
+
import { jest, describe, it, expect, afterEach } from "@jest/globals";
|
|
17
17
|
import * as fs from "fs";
|
|
18
18
|
import * as path from "path";
|
|
19
19
|
import { fileURLToPath } from "url";
|
|
@@ -134,9 +134,9 @@ describe("Security Pattern Count Consistency", () => {
|
|
|
134
134
|
// Default: "default all 8" or "default 8"
|
|
135
135
|
expect(content).toMatch(/securityPatternsToTest\?\s*:\s*number;.*default.*8/i);
|
|
136
136
|
// Reviewer mode: "Test only 3 critical"
|
|
137
|
-
expect(content).toMatch(/securityPatternsToTest:\s*3
|
|
137
|
+
expect(content).toMatch(/securityPatternsToTest:\s*3,.*3 critical/i);
|
|
138
138
|
// Developer/audit modes: "all security patterns" or "all 8"
|
|
139
|
-
expect(content).toMatch(/securityPatternsToTest:\s*8
|
|
139
|
+
expect(content).toMatch(/securityPatternsToTest:\s*8,.*all.*8|8.*patterns/i);
|
|
140
140
|
});
|
|
141
141
|
it("should document pattern count in help text comments", () => {
|
|
142
142
|
const filePath = path.join(projectRoot, "cli/src/assess-security.ts");
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
*
|
|
7
7
|
* @see https://github.com/triepod-ai/inspector-assessment/issues/137
|
|
8
8
|
*/
|
|
9
|
-
import { describe, it, expect } from "@jest/globals";
|
|
9
|
+
import { jest, describe, it, expect, afterEach } from "@jest/globals";
|
|
10
10
|
import { AssessmentOptionsSchema, safeParseAssessmentOptions, validateAssessmentOptions, } from "../lib/cli-parserSchemas.js";
|
|
11
11
|
describe("Stage 3 Fix Validation Tests", () => {
|
|
12
12
|
afterEach(() => {
|
|
@@ -11,7 +11,14 @@
|
|
|
11
11
|
*
|
|
12
12
|
* Note: Tests skip gracefully when testbed servers are unavailable.
|
|
13
13
|
*/
|
|
14
|
-
import { describe, it, expect, beforeAll } from "@jest/globals";
|
|
14
|
+
import { jest, describe, it, expect, beforeAll, afterEach, } from "@jest/globals";
|
|
15
|
+
/**
|
|
16
|
+
* Skip integration tests unless RUN_E2E_TESTS is set.
|
|
17
|
+
* This prevents long timeouts when testbed servers aren't running.
|
|
18
|
+
*
|
|
19
|
+
* To run: RUN_E2E_TESTS=1 npm test -- --testPathPattern="testbed-integration"
|
|
20
|
+
*/
|
|
21
|
+
const describeE2E = process.env.RUN_E2E_TESTS ? describe : describe.skip;
|
|
15
22
|
// Testbed server URLs
|
|
16
23
|
const VULNERABLE_URL = "http://localhost:10900/mcp";
|
|
17
24
|
const HARDENED_URL = "http://localhost:10901/mcp";
|
|
@@ -123,7 +130,7 @@ async function callTool(url, toolName, args) {
|
|
|
123
130
|
});
|
|
124
131
|
return data;
|
|
125
132
|
}
|
|
126
|
-
|
|
133
|
+
describeE2E("Testbed A/B Comparison", () => {
|
|
127
134
|
afterEach(() => {
|
|
128
135
|
jest.clearAllMocks();
|
|
129
136
|
});
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
* Tests focus on input validation and error handling rather than
|
|
6
6
|
* mocked transport implementations due to ESM limitations.
|
|
7
7
|
*/
|
|
8
|
-
import { describe, it, expect } from "@jest/globals";
|
|
8
|
+
import { jest, describe, it, expect, afterEach } from "@jest/globals";
|
|
9
9
|
import { createTransport } from "../transport.js";
|
|
10
10
|
describe("Transport Creation", () => {
|
|
11
11
|
afterEach(() => {
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
*
|
|
6
6
|
* @module cli/lib/__tests__/cli-parserSchemas
|
|
7
7
|
*/
|
|
8
|
-
|
|
8
|
+
import { jest, describe, test, expect, afterEach } from "@jest/globals";
|
|
9
9
|
import { ZodError } from "zod";
|
|
10
10
|
import { AssessmentProfileNameSchema, AssessmentModuleNameSchema, ServerConfigSchema, AssessmentOptionsSchema, ValidationResultSchema, validateAssessmentOptions, validateServerConfig, parseAssessmentOptions, safeParseAssessmentOptions, parseModuleNames, safeParseModuleNames, LogLevelSchema, ReportFormatSchema, TransportTypeSchema, ZOD_SCHEMA_VERSION, } from "../cli-parserSchemas.js";
|
|
11
11
|
describe("cli-parserSchemas", () => {
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
*
|
|
6
6
|
* @module cli/lib/assessment-runner/__tests__/server-configSchemas
|
|
7
7
|
*/
|
|
8
|
-
|
|
8
|
+
import { jest, describe, test, expect, afterEach } from "@jest/globals";
|
|
9
9
|
import { ZodError } from "zod";
|
|
10
10
|
import { HttpSseServerConfigSchema, StdioServerConfigSchema, ServerEntrySchema, ClaudeDesktopConfigSchema, StandaloneConfigSchema, ConfigFileSchema, parseConfigFile, safeParseConfigFile, validateServerEntry, isHttpSseConfig, isStdioConfig, TransportTypeSchema, } from "../server-configSchemas.js";
|
|
11
11
|
describe("server-configSchemas", () => {
|
|
@@ -12,9 +12,12 @@ import { AssessmentStateManager } from "../../assessmentState.js";
|
|
|
12
12
|
import { emitServerConnected, emitToolDiscovered, emitToolsDiscoveryComplete, emitAssessmentComplete, emitTestBatch, emitVulnerabilityFound, emitAnnotationMissing, emitAnnotationMisaligned, emitAnnotationReviewRecommended, emitAnnotationAligned, emitModulesConfigured, emitPhaseStarted, emitPhaseComplete, emitToolTestComplete, emitValidationSummary, } from "../jsonl-events.js";
|
|
13
13
|
import { loadServerConfig } from "./server-config.js";
|
|
14
14
|
import { loadSourceFiles } from "./source-loader.js";
|
|
15
|
+
import { resolveSourcePath } from "./path-resolver.js";
|
|
15
16
|
import { connectToServer } from "./server-connection.js";
|
|
16
17
|
import { createCallToolWrapper } from "./tool-wrapper.js";
|
|
17
18
|
import { buildConfig } from "./config-builder.js";
|
|
19
|
+
// Issue #155: Import annotation debug mode setter
|
|
20
|
+
import { setAnnotationDebugMode } from "../../../../client/lib/services/assessment/modules/annotations/AlignmentChecker.js";
|
|
18
21
|
/**
|
|
19
22
|
* Run full assessment against an MCP server
|
|
20
23
|
*
|
|
@@ -22,6 +25,13 @@ import { buildConfig } from "./config-builder.js";
|
|
|
22
25
|
* @returns Assessment results
|
|
23
26
|
*/
|
|
24
27
|
export async function runFullAssessment(options) {
|
|
28
|
+
// Issue #155: Enable annotation debug mode if flag is set
|
|
29
|
+
if (options.debugAnnotations) {
|
|
30
|
+
setAnnotationDebugMode(true);
|
|
31
|
+
if (!options.jsonOnly) {
|
|
32
|
+
console.log("🔍 Annotation debug mode enabled (--debug-annotations)");
|
|
33
|
+
}
|
|
34
|
+
}
|
|
25
35
|
if (!options.jsonOnly) {
|
|
26
36
|
console.log(`\n🔍 Starting full assessment for: ${options.serverName}`);
|
|
27
37
|
}
|
|
@@ -199,10 +209,19 @@ export async function runFullAssessment(options) {
|
|
|
199
209
|
}
|
|
200
210
|
}
|
|
201
211
|
let sourceFiles = {};
|
|
202
|
-
if (options.sourceCodePath
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
212
|
+
if (options.sourceCodePath) {
|
|
213
|
+
// Resolve path using utility (handles ~, relative paths, symlinks)
|
|
214
|
+
const resolvedSourcePath = resolveSourcePath(options.sourceCodePath);
|
|
215
|
+
if (fs.existsSync(resolvedSourcePath)) {
|
|
216
|
+
sourceFiles = loadSourceFiles(resolvedSourcePath, options.debugSource);
|
|
217
|
+
if (!options.jsonOnly) {
|
|
218
|
+
console.log(`📁 Loaded source files from: ${resolvedSourcePath}`);
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
else if (!options.jsonOnly) {
|
|
222
|
+
// Issue #154: Always show warning, not just with --debug-source
|
|
223
|
+
console.log(`⚠️ Source path not found: ${options.sourceCodePath} (resolved: ${resolvedSourcePath})`);
|
|
224
|
+
console.log(` Use --source <existing-path> to enable full source file analysis.`);
|
|
206
225
|
}
|
|
207
226
|
}
|
|
208
227
|
// Create readResource wrapper for ResourceAssessor
|
|
@@ -10,6 +10,8 @@
|
|
|
10
10
|
export { loadServerConfig } from "./server-config.js";
|
|
11
11
|
// Source File Loading
|
|
12
12
|
export { loadSourceFiles } from "./source-loader.js";
|
|
13
|
+
// Path Resolution
|
|
14
|
+
export { resolveSourcePath } from "./path-resolver.js";
|
|
13
15
|
// Server Connection
|
|
14
16
|
export { connectToServer } from "./server-connection.js";
|
|
15
17
|
// Tool Wrapper
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Path Resolution Utilities
|
|
3
|
+
*
|
|
4
|
+
* Handles path normalization for source code paths including:
|
|
5
|
+
* - Home directory (~) expansion
|
|
6
|
+
* - Relative path resolution
|
|
7
|
+
* - Symlink resolution (for MCPB temp extraction paths)
|
|
8
|
+
*
|
|
9
|
+
* @module cli/lib/assessment-runner/path-resolver
|
|
10
|
+
*/
|
|
11
|
+
import * as fs from "fs";
|
|
12
|
+
import * as os from "os";
|
|
13
|
+
import * as path from "path";
|
|
14
|
+
/**
|
|
15
|
+
* Resolve a source code path to an absolute, real path
|
|
16
|
+
*
|
|
17
|
+
* Handles:
|
|
18
|
+
* - Tilde (~) expansion to home directory
|
|
19
|
+
* - Relative paths resolved to absolute
|
|
20
|
+
* - Symlinks followed to real paths (important for MCPB temp extraction)
|
|
21
|
+
*
|
|
22
|
+
* @param sourcePath - The source path to resolve (may be relative, contain ~, or be a symlink)
|
|
23
|
+
* @returns The resolved absolute path, or the original path if resolution fails
|
|
24
|
+
*
|
|
25
|
+
* @example
|
|
26
|
+
* resolveSourcePath("~/project") // => "/home/user/project"
|
|
27
|
+
* resolveSourcePath("./src") // => "/current/working/dir/src"
|
|
28
|
+
* resolveSourcePath("/tmp/symlink") // => "/actual/target/path"
|
|
29
|
+
*/
|
|
30
|
+
export function resolveSourcePath(sourcePath) {
|
|
31
|
+
let resolved = sourcePath;
|
|
32
|
+
// Expand home directory (~)
|
|
33
|
+
if (resolved.startsWith("~")) {
|
|
34
|
+
resolved = path.join(os.homedir(), resolved.slice(1));
|
|
35
|
+
}
|
|
36
|
+
// Resolve to absolute path
|
|
37
|
+
resolved = path.resolve(resolved);
|
|
38
|
+
// Follow symlinks if path exists (handles MCPB temp extraction paths)
|
|
39
|
+
try {
|
|
40
|
+
if (fs.existsSync(resolved)) {
|
|
41
|
+
resolved = fs.realpathSync(resolved);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
catch {
|
|
45
|
+
// realpathSync can fail on broken symlinks, continue with resolved path
|
|
46
|
+
}
|
|
47
|
+
return resolved;
|
|
48
|
+
}
|