@bryan-thompson/inspector-assessment 1.35.0 → 1.35.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli/build/__tests__/stage3-fix-validation.test.js +163 -0
- package/cli/build/__tests__/stage3-fixes.test.js +516 -0
- package/cli/build/lib/cli-parser.js +7 -0
- package/cli/build/lib/cli-parserSchemas.js +3 -0
- package/cli/build/lib/jsonl-events.js +3 -0
- package/cli/build/lib/result-output.js +8 -2
- package/cli/package.json +1 -1
- package/client/dist/assets/{OAuthCallback-DC1cIXHT.js → OAuthCallback-jfmizOMH.js} +1 -1
- package/client/dist/assets/{OAuthDebugCallback-C3gqJjgQ.js → OAuthDebugCallback-bU5kKvnt.js} +1 -1
- package/client/dist/assets/{index-Dn2w887x.js → index-Ce63ds7G.js} +4 -4
- package/client/dist/index.html +1 -1
- package/client/lib/lib/assessment/extendedTypes.d.ts +19 -5
- package/client/lib/lib/assessment/extendedTypes.d.ts.map +1 -1
- package/client/lib/lib/assessment/summarizer/AssessmentSummarizer.d.ts.map +1 -1
- package/client/lib/lib/assessment/summarizer/AssessmentSummarizer.js +14 -1
- package/client/lib/lib/assessment/summarizer/index.d.ts +4 -0
- package/client/lib/lib/assessment/summarizer/index.d.ts.map +1 -1
- package/client/lib/lib/assessment/summarizer/index.js +4 -0
- package/client/lib/lib/assessment/summarizer/stageBEnrichmentBuilder.d.ts +36 -0
- package/client/lib/lib/assessment/summarizer/stageBEnrichmentBuilder.d.ts.map +1 -0
- package/client/lib/lib/assessment/summarizer/stageBEnrichmentBuilder.js +282 -0
- package/client/lib/lib/assessment/summarizer/stageBTypes.d.ts +154 -0
- package/client/lib/lib/assessment/summarizer/stageBTypes.d.ts.map +1 -0
- package/client/lib/lib/assessment/summarizer/stageBTypes.js +24 -0
- package/client/lib/lib/assessment/summarizer/types.d.ts +5 -0
- package/client/lib/lib/assessment/summarizer/types.d.ts.map +1 -1
- package/client/lib/lib/assessment/summarizer/types.js +1 -0
- package/client/lib/lib/moduleScoring.d.ts +2 -1
- package/client/lib/lib/moduleScoring.d.ts.map +1 -1
- package/client/lib/lib/moduleScoring.js +2 -1
- package/client/lib/services/assessment/modules/ManifestValidationAssessor.d.ts +8 -0
- package/client/lib/services/assessment/modules/ManifestValidationAssessor.d.ts.map +1 -1
- package/client/lib/services/assessment/modules/ManifestValidationAssessor.js +51 -8
- package/client/package.json +1 -1
- package/package.json +1 -1
- package/server/package.json +1 -1
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Stage 3 Fix Validation Tests
|
|
3
|
+
*
|
|
4
|
+
* Tests for Issue #137 Stage 3 fixes (code review and corrections).
|
|
5
|
+
* Validates that FIX-001 (stageBVerbose schema) is correctly implemented.
|
|
6
|
+
*
|
|
7
|
+
* @see https://github.com/triepod-ai/inspector-assessment/issues/137
|
|
8
|
+
*/
|
|
9
|
+
import { describe, it, expect } from "@jest/globals";
|
|
10
|
+
import { AssessmentOptionsSchema, safeParseAssessmentOptions, validateAssessmentOptions, } from "../lib/cli-parserSchemas.js";
|
|
11
|
+
describe("Stage 3 Fix Validation Tests", () => {
|
|
12
|
+
describe("[TEST-001] cli-parserSchemas.ts - stageBVerbose field (FIX-001)", () => {
|
|
13
|
+
describe("stageBVerbose field validation", () => {
|
|
14
|
+
it("should accept stageBVerbose with true value (happy path)", () => {
|
|
15
|
+
const options = {
|
|
16
|
+
serverName: "test-server",
|
|
17
|
+
stageBVerbose: true,
|
|
18
|
+
};
|
|
19
|
+
const result = safeParseAssessmentOptions(options);
|
|
20
|
+
expect(result.success).toBe(true);
|
|
21
|
+
if (result.success) {
|
|
22
|
+
expect(result.data.stageBVerbose).toBe(true);
|
|
23
|
+
}
|
|
24
|
+
});
|
|
25
|
+
it("should accept stageBVerbose with false value (edge case)", () => {
|
|
26
|
+
const options = {
|
|
27
|
+
serverName: "test-server",
|
|
28
|
+
stageBVerbose: false,
|
|
29
|
+
};
|
|
30
|
+
const result = safeParseAssessmentOptions(options);
|
|
31
|
+
expect(result.success).toBe(true);
|
|
32
|
+
if (result.success) {
|
|
33
|
+
expect(result.data.stageBVerbose).toBe(false);
|
|
34
|
+
}
|
|
35
|
+
});
|
|
36
|
+
it("should accept undefined stageBVerbose - optional field (edge case)", () => {
|
|
37
|
+
const options = {
|
|
38
|
+
serverName: "test-server",
|
|
39
|
+
// stageBVerbose omitted
|
|
40
|
+
};
|
|
41
|
+
const result = safeParseAssessmentOptions(options);
|
|
42
|
+
expect(result.success).toBe(true);
|
|
43
|
+
if (result.success) {
|
|
44
|
+
expect(result.data.stageBVerbose).toBeUndefined();
|
|
45
|
+
}
|
|
46
|
+
});
|
|
47
|
+
it("should validate with validateAssessmentOptions() accepting stageBVerbose (error case prevention)", () => {
|
|
48
|
+
const options = {
|
|
49
|
+
serverName: "test-server",
|
|
50
|
+
stageBVerbose: true,
|
|
51
|
+
};
|
|
52
|
+
const errors = validateAssessmentOptions(options);
|
|
53
|
+
expect(errors).toHaveLength(0);
|
|
54
|
+
});
|
|
55
|
+
it("should reject non-boolean stageBVerbose values (error case)", () => {
|
|
56
|
+
const testCases = [
|
|
57
|
+
{ stageBVerbose: "true" }, // string instead of boolean
|
|
58
|
+
{ stageBVerbose: 1 }, // number instead of boolean
|
|
59
|
+
{ stageBVerbose: null },
|
|
60
|
+
{ stageBVerbose: {} },
|
|
61
|
+
{ stageBVerbose: [] },
|
|
62
|
+
];
|
|
63
|
+
testCases.forEach((testCase) => {
|
|
64
|
+
const options = {
|
|
65
|
+
serverName: "test-server",
|
|
66
|
+
...testCase,
|
|
67
|
+
};
|
|
68
|
+
const result = safeParseAssessmentOptions(options);
|
|
69
|
+
expect(result.success).toBe(false);
|
|
70
|
+
});
|
|
71
|
+
});
|
|
72
|
+
});
|
|
73
|
+
describe("stageBVerbose integration with other fields", () => {
|
|
74
|
+
it("should accept stageBVerbose with outputFormat=tiered", () => {
|
|
75
|
+
const options = {
|
|
76
|
+
serverName: "test-server",
|
|
77
|
+
outputFormat: "tiered",
|
|
78
|
+
stageBVerbose: true,
|
|
79
|
+
};
|
|
80
|
+
const result = safeParseAssessmentOptions(options);
|
|
81
|
+
expect(result.success).toBe(true);
|
|
82
|
+
if (result.success) {
|
|
83
|
+
expect(result.data.outputFormat).toBe("tiered");
|
|
84
|
+
expect(result.data.stageBVerbose).toBe(true);
|
|
85
|
+
}
|
|
86
|
+
});
|
|
87
|
+
it("should accept stageBVerbose with other CLI options", () => {
|
|
88
|
+
const options = {
|
|
89
|
+
serverName: "test-server",
|
|
90
|
+
verbose: true,
|
|
91
|
+
jsonOnly: false,
|
|
92
|
+
fullAssessment: true,
|
|
93
|
+
stageBVerbose: true,
|
|
94
|
+
};
|
|
95
|
+
const result = safeParseAssessmentOptions(options);
|
|
96
|
+
expect(result.success).toBe(true);
|
|
97
|
+
if (result.success) {
|
|
98
|
+
expect(result.data.verbose).toBe(true);
|
|
99
|
+
expect(result.data.jsonOnly).toBe(false);
|
|
100
|
+
expect(result.data.fullAssessment).toBe(true);
|
|
101
|
+
expect(result.data.stageBVerbose).toBe(true);
|
|
102
|
+
}
|
|
103
|
+
});
|
|
104
|
+
it("should work with validateAssessmentOptions for complex options", () => {
|
|
105
|
+
const options = {
|
|
106
|
+
serverName: "test-server",
|
|
107
|
+
outputFormat: "tiered",
|
|
108
|
+
autoTier: true,
|
|
109
|
+
stageBVerbose: true,
|
|
110
|
+
verbose: true,
|
|
111
|
+
};
|
|
112
|
+
const errors = validateAssessmentOptions(options);
|
|
113
|
+
expect(errors).toHaveLength(0);
|
|
114
|
+
});
|
|
115
|
+
});
|
|
116
|
+
describe("regression prevention for ISSUE-001", () => {
|
|
117
|
+
it("should maintain stageBVerbose in schema after validation", () => {
|
|
118
|
+
// Verify the field exists in the schema shape
|
|
119
|
+
const options = {
|
|
120
|
+
serverName: "test-server",
|
|
121
|
+
stageBVerbose: true,
|
|
122
|
+
};
|
|
123
|
+
const result = AssessmentOptionsSchema.safeParse(options);
|
|
124
|
+
expect(result.success).toBe(true);
|
|
125
|
+
if (result.success) {
|
|
126
|
+
expect(result.data).toHaveProperty("stageBVerbose");
|
|
127
|
+
expect(result.data.stageBVerbose).toBe(true);
|
|
128
|
+
}
|
|
129
|
+
});
|
|
130
|
+
it("should not strip stageBVerbose field during validation", () => {
|
|
131
|
+
const options = {
|
|
132
|
+
serverName: "test-server",
|
|
133
|
+
stageBVerbose: true,
|
|
134
|
+
outputFormat: "tiered",
|
|
135
|
+
};
|
|
136
|
+
const parsed = AssessmentOptionsSchema.parse(options);
|
|
137
|
+
// Field should be present after parsing
|
|
138
|
+
expect(parsed.stageBVerbose).toBe(true);
|
|
139
|
+
expect(parsed.outputFormat).toBe("tiered");
|
|
140
|
+
});
|
|
141
|
+
it("should backward compatibility - existing fields unaffected", () => {
|
|
142
|
+
const options = {
|
|
143
|
+
serverName: "test-server",
|
|
144
|
+
verbose: true,
|
|
145
|
+
jsonOnly: false,
|
|
146
|
+
format: "json",
|
|
147
|
+
fullAssessment: true,
|
|
148
|
+
// New field
|
|
149
|
+
stageBVerbose: true,
|
|
150
|
+
};
|
|
151
|
+
const result = safeParseAssessmentOptions(options);
|
|
152
|
+
expect(result.success).toBe(true);
|
|
153
|
+
if (result.success) {
|
|
154
|
+
expect(result.data.verbose).toBe(true);
|
|
155
|
+
expect(result.data.jsonOnly).toBe(false);
|
|
156
|
+
expect(result.data.format).toBe("json");
|
|
157
|
+
expect(result.data.fullAssessment).toBe(true);
|
|
158
|
+
expect(result.data.stageBVerbose).toBe(true);
|
|
159
|
+
}
|
|
160
|
+
});
|
|
161
|
+
});
|
|
162
|
+
});
|
|
163
|
+
});
|
|
@@ -0,0 +1,516 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Stage 3 Fixes Regression Tests
|
|
3
|
+
*
|
|
4
|
+
* Tests for Issue #134 code review fixes to ensure they don't regress.
|
|
5
|
+
* These tests validate schema validation and JSONL event emission.
|
|
6
|
+
*
|
|
7
|
+
* @see https://github.com/triepod-ai/inspector-assessment/issues/134
|
|
8
|
+
*/
|
|
9
|
+
import { describe, it, expect, jest, beforeEach, afterEach, } from "@jest/globals";
|
|
10
|
+
import { OutputFormatSchema, safeParseAssessmentOptions, } from "../lib/cli-parserSchemas.js";
|
|
11
|
+
import { emitTieredOutput, SCHEMA_VERSION, } from "../lib/jsonl-events.js";
|
|
12
|
+
import { INSPECTOR_VERSION } from "../../../client/lib/lib/moduleScoring.js";
|
|
13
|
+
describe("Stage 3 Fixes Regression Tests", () => {
|
|
14
|
+
describe("[TEST-REQ-001] cli-parserSchemas.ts - AssessmentOptionsSchema", () => {
|
|
15
|
+
describe("outputFormat field validation", () => {
|
|
16
|
+
it("should accept outputFormat with 'full' value", () => {
|
|
17
|
+
const options = {
|
|
18
|
+
serverName: "test-server",
|
|
19
|
+
outputFormat: "full",
|
|
20
|
+
};
|
|
21
|
+
const result = safeParseAssessmentOptions(options);
|
|
22
|
+
expect(result.success).toBe(true);
|
|
23
|
+
if (result.success) {
|
|
24
|
+
expect(result.data.outputFormat).toBe("full");
|
|
25
|
+
}
|
|
26
|
+
});
|
|
27
|
+
it("should accept outputFormat with 'tiered' value", () => {
|
|
28
|
+
const options = {
|
|
29
|
+
serverName: "test-server",
|
|
30
|
+
outputFormat: "tiered",
|
|
31
|
+
};
|
|
32
|
+
const result = safeParseAssessmentOptions(options);
|
|
33
|
+
expect(result.success).toBe(true);
|
|
34
|
+
if (result.success) {
|
|
35
|
+
expect(result.data.outputFormat).toBe("tiered");
|
|
36
|
+
}
|
|
37
|
+
});
|
|
38
|
+
it("should accept outputFormat with 'summary-only' value", () => {
|
|
39
|
+
const options = {
|
|
40
|
+
serverName: "test-server",
|
|
41
|
+
outputFormat: "summary-only",
|
|
42
|
+
};
|
|
43
|
+
const result = safeParseAssessmentOptions(options);
|
|
44
|
+
expect(result.success).toBe(true);
|
|
45
|
+
if (result.success) {
|
|
46
|
+
expect(result.data.outputFormat).toBe("summary-only");
|
|
47
|
+
}
|
|
48
|
+
});
|
|
49
|
+
it("should accept undefined outputFormat (optional field)", () => {
|
|
50
|
+
const options = {
|
|
51
|
+
serverName: "test-server",
|
|
52
|
+
// outputFormat omitted
|
|
53
|
+
};
|
|
54
|
+
const result = safeParseAssessmentOptions(options);
|
|
55
|
+
expect(result.success).toBe(true);
|
|
56
|
+
if (result.success) {
|
|
57
|
+
expect(result.data.outputFormat).toBeUndefined();
|
|
58
|
+
}
|
|
59
|
+
});
|
|
60
|
+
it("should reject invalid outputFormat value", () => {
|
|
61
|
+
const options = {
|
|
62
|
+
serverName: "test-server",
|
|
63
|
+
outputFormat: "invalid",
|
|
64
|
+
};
|
|
65
|
+
const result = safeParseAssessmentOptions(options);
|
|
66
|
+
expect(result.success).toBe(false);
|
|
67
|
+
if (!result.success) {
|
|
68
|
+
// Zod error messages include the full path and expected values
|
|
69
|
+
const errors = result.error.errors;
|
|
70
|
+
const hasOutputFormatError = errors.some((e) => e.path.includes("outputFormat") ||
|
|
71
|
+
e.message.includes("full") ||
|
|
72
|
+
e.message.includes("tiered") ||
|
|
73
|
+
e.message.includes("summary-only"));
|
|
74
|
+
expect(hasOutputFormatError).toBe(true);
|
|
75
|
+
}
|
|
76
|
+
});
|
|
77
|
+
it("should reject non-string outputFormat values", () => {
|
|
78
|
+
const testCases = [
|
|
79
|
+
{ outputFormat: 123 },
|
|
80
|
+
{ outputFormat: true },
|
|
81
|
+
{ outputFormat: null },
|
|
82
|
+
{ outputFormat: {} },
|
|
83
|
+
{ outputFormat: [] },
|
|
84
|
+
];
|
|
85
|
+
testCases.forEach((testCase) => {
|
|
86
|
+
const options = {
|
|
87
|
+
serverName: "test-server",
|
|
88
|
+
...testCase,
|
|
89
|
+
};
|
|
90
|
+
const result = safeParseAssessmentOptions(options);
|
|
91
|
+
expect(result.success).toBe(false);
|
|
92
|
+
});
|
|
93
|
+
});
|
|
94
|
+
});
|
|
95
|
+
describe("autoTier field validation", () => {
|
|
96
|
+
it("should accept autoTier with true value", () => {
|
|
97
|
+
const options = {
|
|
98
|
+
serverName: "test-server",
|
|
99
|
+
autoTier: true,
|
|
100
|
+
};
|
|
101
|
+
const result = safeParseAssessmentOptions(options);
|
|
102
|
+
expect(result.success).toBe(true);
|
|
103
|
+
if (result.success) {
|
|
104
|
+
expect(result.data.autoTier).toBe(true);
|
|
105
|
+
}
|
|
106
|
+
});
|
|
107
|
+
it("should accept autoTier with false value", () => {
|
|
108
|
+
const options = {
|
|
109
|
+
serverName: "test-server",
|
|
110
|
+
autoTier: false,
|
|
111
|
+
};
|
|
112
|
+
const result = safeParseAssessmentOptions(options);
|
|
113
|
+
expect(result.success).toBe(true);
|
|
114
|
+
if (result.success) {
|
|
115
|
+
expect(result.data.autoTier).toBe(false);
|
|
116
|
+
}
|
|
117
|
+
});
|
|
118
|
+
it("should accept undefined autoTier (optional field)", () => {
|
|
119
|
+
const options = {
|
|
120
|
+
serverName: "test-server",
|
|
121
|
+
// autoTier omitted
|
|
122
|
+
};
|
|
123
|
+
const result = safeParseAssessmentOptions(options);
|
|
124
|
+
expect(result.success).toBe(true);
|
|
125
|
+
if (result.success) {
|
|
126
|
+
expect(result.data.autoTier).toBeUndefined();
|
|
127
|
+
}
|
|
128
|
+
});
|
|
129
|
+
it("should reject non-boolean autoTier values", () => {
|
|
130
|
+
const testCases = [
|
|
131
|
+
{ autoTier: "true" }, // string instead of boolean
|
|
132
|
+
{ autoTier: 1 }, // number instead of boolean
|
|
133
|
+
{ autoTier: null },
|
|
134
|
+
{ autoTier: {} },
|
|
135
|
+
{ autoTier: [] },
|
|
136
|
+
];
|
|
137
|
+
testCases.forEach((testCase) => {
|
|
138
|
+
const options = {
|
|
139
|
+
serverName: "test-server",
|
|
140
|
+
...testCase,
|
|
141
|
+
};
|
|
142
|
+
const result = safeParseAssessmentOptions(options);
|
|
143
|
+
expect(result.success).toBe(false);
|
|
144
|
+
});
|
|
145
|
+
});
|
|
146
|
+
});
|
|
147
|
+
describe("combined outputFormat and autoTier validation", () => {
|
|
148
|
+
it("should accept both outputFormat and autoTier together", () => {
|
|
149
|
+
const options = {
|
|
150
|
+
serverName: "test-server",
|
|
151
|
+
outputFormat: "tiered",
|
|
152
|
+
autoTier: true,
|
|
153
|
+
};
|
|
154
|
+
const result = safeParseAssessmentOptions(options);
|
|
155
|
+
expect(result.success).toBe(true);
|
|
156
|
+
if (result.success) {
|
|
157
|
+
expect(result.data.outputFormat).toBe("tiered");
|
|
158
|
+
expect(result.data.autoTier).toBe(true);
|
|
159
|
+
}
|
|
160
|
+
});
|
|
161
|
+
it("should accept outputFormat without autoTier", () => {
|
|
162
|
+
const options = {
|
|
163
|
+
serverName: "test-server",
|
|
164
|
+
outputFormat: "summary-only",
|
|
165
|
+
};
|
|
166
|
+
const result = safeParseAssessmentOptions(options);
|
|
167
|
+
expect(result.success).toBe(true);
|
|
168
|
+
if (result.success) {
|
|
169
|
+
expect(result.data.outputFormat).toBe("summary-only");
|
|
170
|
+
expect(result.data.autoTier).toBeUndefined();
|
|
171
|
+
}
|
|
172
|
+
});
|
|
173
|
+
it("should accept autoTier without outputFormat", () => {
|
|
174
|
+
const options = {
|
|
175
|
+
serverName: "test-server",
|
|
176
|
+
autoTier: true,
|
|
177
|
+
};
|
|
178
|
+
const result = safeParseAssessmentOptions(options);
|
|
179
|
+
expect(result.success).toBe(true);
|
|
180
|
+
if (result.success) {
|
|
181
|
+
expect(result.data.autoTier).toBe(true);
|
|
182
|
+
expect(result.data.outputFormat).toBeUndefined();
|
|
183
|
+
}
|
|
184
|
+
});
|
|
185
|
+
});
|
|
186
|
+
describe("regression prevention", () => {
|
|
187
|
+
it("should fail if outputFormat schema loses required enum values", () => {
|
|
188
|
+
// Verify all three enum values are present
|
|
189
|
+
const validValues = OutputFormatSchema.options;
|
|
190
|
+
expect(validValues).toContain("full");
|
|
191
|
+
expect(validValues).toContain("tiered");
|
|
192
|
+
expect(validValues).toContain("summary-only");
|
|
193
|
+
expect(validValues.length).toBe(3);
|
|
194
|
+
});
|
|
195
|
+
it("should maintain backward compatibility with existing fields", () => {
|
|
196
|
+
// Test that adding new fields didn't break existing validation
|
|
197
|
+
const options = {
|
|
198
|
+
serverName: "test-server",
|
|
199
|
+
verbose: true,
|
|
200
|
+
jsonOnly: true,
|
|
201
|
+
format: "json",
|
|
202
|
+
// Include new fields
|
|
203
|
+
outputFormat: "tiered",
|
|
204
|
+
autoTier: true,
|
|
205
|
+
};
|
|
206
|
+
const result = safeParseAssessmentOptions(options);
|
|
207
|
+
expect(result.success).toBe(true);
|
|
208
|
+
if (result.success) {
|
|
209
|
+
expect(result.data.verbose).toBe(true);
|
|
210
|
+
expect(result.data.jsonOnly).toBe(true);
|
|
211
|
+
expect(result.data.format).toBe("json");
|
|
212
|
+
expect(result.data.outputFormat).toBe("tiered");
|
|
213
|
+
expect(result.data.autoTier).toBe(true);
|
|
214
|
+
}
|
|
215
|
+
});
|
|
216
|
+
});
|
|
217
|
+
});
|
|
218
|
+
describe("[TEST-REQ-002] jsonl-events.ts - TieredOutputEvent and emitTieredOutput", () => {
|
|
219
|
+
// Capture console.error output for testing
|
|
220
|
+
let consoleSpy;
|
|
221
|
+
let errorOutput = [];
|
|
222
|
+
beforeEach(() => {
|
|
223
|
+
errorOutput = [];
|
|
224
|
+
// Spy on console.error to capture JSONL output
|
|
225
|
+
consoleSpy = jest
|
|
226
|
+
.spyOn(console, "error")
|
|
227
|
+
.mockImplementation((msg) => {
|
|
228
|
+
errorOutput.push(msg);
|
|
229
|
+
});
|
|
230
|
+
});
|
|
231
|
+
afterEach(() => {
|
|
232
|
+
// Restore original console.error
|
|
233
|
+
if (consoleSpy) {
|
|
234
|
+
consoleSpy.mockRestore();
|
|
235
|
+
}
|
|
236
|
+
});
|
|
237
|
+
describe("emitTieredOutput function", () => {
|
|
238
|
+
it("should emit valid JSONL with correct event type", () => {
|
|
239
|
+
const tiers = {
|
|
240
|
+
executiveSummary: {
|
|
241
|
+
path: "/tmp/output/executive-summary.json",
|
|
242
|
+
estimatedTokens: 500,
|
|
243
|
+
},
|
|
244
|
+
toolSummaries: {
|
|
245
|
+
path: "/tmp/output/tool-summaries.json",
|
|
246
|
+
estimatedTokens: 1500,
|
|
247
|
+
toolCount: 10,
|
|
248
|
+
},
|
|
249
|
+
};
|
|
250
|
+
emitTieredOutput("/tmp/output", "summary-only", tiers);
|
|
251
|
+
expect(errorOutput.length).toBe(1);
|
|
252
|
+
const parsed = JSON.parse(errorOutput[0]);
|
|
253
|
+
expect(parsed.event).toBe("tiered_output_generated");
|
|
254
|
+
expect(parsed.outputDir).toBe("/tmp/output");
|
|
255
|
+
expect(parsed.outputFormat).toBe("summary-only");
|
|
256
|
+
expect(parsed.tiers).toEqual(tiers);
|
|
257
|
+
});
|
|
258
|
+
it("should include version and schemaVersion fields", () => {
|
|
259
|
+
const tiers = {
|
|
260
|
+
executiveSummary: {
|
|
261
|
+
path: "/tmp/output/executive-summary.json",
|
|
262
|
+
estimatedTokens: 500,
|
|
263
|
+
},
|
|
264
|
+
toolSummaries: {
|
|
265
|
+
path: "/tmp/output/tool-summaries.json",
|
|
266
|
+
estimatedTokens: 1500,
|
|
267
|
+
toolCount: 10,
|
|
268
|
+
},
|
|
269
|
+
};
|
|
270
|
+
emitTieredOutput("/tmp/output", "tiered", tiers);
|
|
271
|
+
expect(errorOutput.length).toBe(1);
|
|
272
|
+
const parsed = JSON.parse(errorOutput[0]);
|
|
273
|
+
expect(parsed.version).toBe(INSPECTOR_VERSION);
|
|
274
|
+
expect(parsed.schemaVersion).toBe(SCHEMA_VERSION);
|
|
275
|
+
});
|
|
276
|
+
it("should include all required tier properties", () => {
|
|
277
|
+
const tiers = {
|
|
278
|
+
executiveSummary: {
|
|
279
|
+
path: "/tmp/output/executive-summary.json",
|
|
280
|
+
estimatedTokens: 500,
|
|
281
|
+
},
|
|
282
|
+
toolSummaries: {
|
|
283
|
+
path: "/tmp/output/tool-summaries.json",
|
|
284
|
+
estimatedTokens: 1500,
|
|
285
|
+
toolCount: 10,
|
|
286
|
+
},
|
|
287
|
+
toolDetails: {
|
|
288
|
+
directory: "/tmp/output/tools",
|
|
289
|
+
fileCount: 10,
|
|
290
|
+
totalEstimatedTokens: 5000,
|
|
291
|
+
},
|
|
292
|
+
};
|
|
293
|
+
emitTieredOutput("/tmp/output", "tiered", tiers);
|
|
294
|
+
expect(errorOutput.length).toBe(1);
|
|
295
|
+
const parsed = JSON.parse(errorOutput[0]);
|
|
296
|
+
expect(parsed.tiers.executiveSummary).toEqual({
|
|
297
|
+
path: "/tmp/output/executive-summary.json",
|
|
298
|
+
estimatedTokens: 500,
|
|
299
|
+
});
|
|
300
|
+
expect(parsed.tiers.toolSummaries).toEqual({
|
|
301
|
+
path: "/tmp/output/tool-summaries.json",
|
|
302
|
+
estimatedTokens: 1500,
|
|
303
|
+
toolCount: 10,
|
|
304
|
+
});
|
|
305
|
+
expect(parsed.tiers.toolDetails).toEqual({
|
|
306
|
+
directory: "/tmp/output/tools",
|
|
307
|
+
fileCount: 10,
|
|
308
|
+
totalEstimatedTokens: 5000,
|
|
309
|
+
});
|
|
310
|
+
});
|
|
311
|
+
it("should handle missing optional toolDetails tier", () => {
|
|
312
|
+
const tiers = {
|
|
313
|
+
executiveSummary: {
|
|
314
|
+
path: "/tmp/output/executive-summary.json",
|
|
315
|
+
estimatedTokens: 500,
|
|
316
|
+
},
|
|
317
|
+
toolSummaries: {
|
|
318
|
+
path: "/tmp/output/tool-summaries.json",
|
|
319
|
+
estimatedTokens: 1500,
|
|
320
|
+
toolCount: 10,
|
|
321
|
+
},
|
|
322
|
+
// toolDetails omitted (optional)
|
|
323
|
+
};
|
|
324
|
+
emitTieredOutput("/tmp/output", "summary-only", tiers);
|
|
325
|
+
expect(errorOutput.length).toBe(1);
|
|
326
|
+
const parsed = JSON.parse(errorOutput[0]);
|
|
327
|
+
expect(parsed.tiers.executiveSummary).toBeDefined();
|
|
328
|
+
expect(parsed.tiers.toolSummaries).toBeDefined();
|
|
329
|
+
expect(parsed.tiers.toolDetails).toBeUndefined();
|
|
330
|
+
});
|
|
331
|
+
it("should emit valid JSON that can be parsed", () => {
|
|
332
|
+
const tiers = {
|
|
333
|
+
executiveSummary: {
|
|
334
|
+
path: "/tmp/output/executive-summary.json",
|
|
335
|
+
estimatedTokens: 500,
|
|
336
|
+
},
|
|
337
|
+
toolSummaries: {
|
|
338
|
+
path: "/tmp/output/tool-summaries.json",
|
|
339
|
+
estimatedTokens: 1500,
|
|
340
|
+
toolCount: 10,
|
|
341
|
+
},
|
|
342
|
+
};
|
|
343
|
+
emitTieredOutput("/tmp/output", "tiered", tiers);
|
|
344
|
+
expect(errorOutput.length).toBe(1);
|
|
345
|
+
// Should not throw when parsing
|
|
346
|
+
expect(() => JSON.parse(errorOutput[0])).not.toThrow();
|
|
347
|
+
});
|
|
348
|
+
});
|
|
349
|
+
describe("TieredOutputEvent type structure", () => {
|
|
350
|
+
it("should enforce correct event type", () => {
|
|
351
|
+
const event = {
|
|
352
|
+
event: "tiered_output_generated",
|
|
353
|
+
outputDir: "/tmp/output",
|
|
354
|
+
outputFormat: "tiered",
|
|
355
|
+
tiers: {
|
|
356
|
+
executiveSummary: {
|
|
357
|
+
path: "/tmp/output/executive-summary.json",
|
|
358
|
+
estimatedTokens: 500,
|
|
359
|
+
},
|
|
360
|
+
toolSummaries: {
|
|
361
|
+
path: "/tmp/output/tool-summaries.json",
|
|
362
|
+
estimatedTokens: 1500,
|
|
363
|
+
toolCount: 10,
|
|
364
|
+
},
|
|
365
|
+
},
|
|
366
|
+
};
|
|
367
|
+
expect(event.event).toBe("tiered_output_generated");
|
|
368
|
+
});
|
|
369
|
+
it("should support both 'tiered' and 'summary-only' output formats", () => {
|
|
370
|
+
const tieredEvent = {
|
|
371
|
+
event: "tiered_output_generated",
|
|
372
|
+
outputDir: "/tmp/output",
|
|
373
|
+
outputFormat: "tiered",
|
|
374
|
+
tiers: {
|
|
375
|
+
executiveSummary: { path: "/tmp/exec.json", estimatedTokens: 500 },
|
|
376
|
+
toolSummaries: {
|
|
377
|
+
path: "/tmp/tools.json",
|
|
378
|
+
estimatedTokens: 1500,
|
|
379
|
+
toolCount: 10,
|
|
380
|
+
},
|
|
381
|
+
},
|
|
382
|
+
};
|
|
383
|
+
const summaryOnlyEvent = {
|
|
384
|
+
event: "tiered_output_generated",
|
|
385
|
+
outputDir: "/tmp/output",
|
|
386
|
+
outputFormat: "summary-only",
|
|
387
|
+
tiers: {
|
|
388
|
+
executiveSummary: { path: "/tmp/exec.json", estimatedTokens: 500 },
|
|
389
|
+
toolSummaries: {
|
|
390
|
+
path: "/tmp/tools.json",
|
|
391
|
+
estimatedTokens: 1500,
|
|
392
|
+
toolCount: 10,
|
|
393
|
+
},
|
|
394
|
+
},
|
|
395
|
+
};
|
|
396
|
+
expect(tieredEvent.outputFormat).toBe("tiered");
|
|
397
|
+
expect(summaryOnlyEvent.outputFormat).toBe("summary-only");
|
|
398
|
+
});
|
|
399
|
+
it("should have correct structure for all tier properties", () => {
|
|
400
|
+
const event = {
|
|
401
|
+
event: "tiered_output_generated",
|
|
402
|
+
outputDir: "/tmp/output",
|
|
403
|
+
outputFormat: "tiered",
|
|
404
|
+
tiers: {
|
|
405
|
+
executiveSummary: {
|
|
406
|
+
path: "/tmp/output/executive-summary.json",
|
|
407
|
+
estimatedTokens: 500,
|
|
408
|
+
},
|
|
409
|
+
toolSummaries: {
|
|
410
|
+
path: "/tmp/output/tool-summaries.json",
|
|
411
|
+
estimatedTokens: 1500,
|
|
412
|
+
toolCount: 10,
|
|
413
|
+
},
|
|
414
|
+
toolDetails: {
|
|
415
|
+
directory: "/tmp/output/tools",
|
|
416
|
+
fileCount: 10,
|
|
417
|
+
totalEstimatedTokens: 5000,
|
|
418
|
+
},
|
|
419
|
+
},
|
|
420
|
+
};
|
|
421
|
+
// Verify executive summary structure
|
|
422
|
+
expect(event.tiers.executiveSummary).toHaveProperty("path");
|
|
423
|
+
expect(event.tiers.executiveSummary).toHaveProperty("estimatedTokens");
|
|
424
|
+
expect(typeof event.tiers.executiveSummary.path).toBe("string");
|
|
425
|
+
expect(typeof event.tiers.executiveSummary.estimatedTokens).toBe("number");
|
|
426
|
+
// Verify tool summaries structure
|
|
427
|
+
expect(event.tiers.toolSummaries).toHaveProperty("path");
|
|
428
|
+
expect(event.tiers.toolSummaries).toHaveProperty("estimatedTokens");
|
|
429
|
+
expect(event.tiers.toolSummaries).toHaveProperty("toolCount");
|
|
430
|
+
expect(typeof event.tiers.toolSummaries.path).toBe("string");
|
|
431
|
+
expect(typeof event.tiers.toolSummaries.estimatedTokens).toBe("number");
|
|
432
|
+
expect(typeof event.tiers.toolSummaries.toolCount).toBe("number");
|
|
433
|
+
// Verify tool details structure (optional)
|
|
434
|
+
expect(event.tiers.toolDetails).toHaveProperty("directory");
|
|
435
|
+
expect(event.tiers.toolDetails).toHaveProperty("fileCount");
|
|
436
|
+
expect(event.tiers.toolDetails).toHaveProperty("totalEstimatedTokens");
|
|
437
|
+
expect(typeof event.tiers.toolDetails?.directory).toBe("string");
|
|
438
|
+
expect(typeof event.tiers.toolDetails?.fileCount).toBe("number");
|
|
439
|
+
expect(typeof event.tiers.toolDetails?.totalEstimatedTokens).toBe("number");
|
|
440
|
+
});
|
|
441
|
+
});
|
|
442
|
+
describe("JSONL format compliance", () => {
|
|
443
|
+
it("should emit single-line JSON (no newlines within object)", () => {
|
|
444
|
+
const tiers = {
|
|
445
|
+
executiveSummary: {
|
|
446
|
+
path: "/tmp/output/executive-summary.json",
|
|
447
|
+
estimatedTokens: 500,
|
|
448
|
+
},
|
|
449
|
+
toolSummaries: {
|
|
450
|
+
path: "/tmp/output/tool-summaries.json",
|
|
451
|
+
estimatedTokens: 1500,
|
|
452
|
+
toolCount: 10,
|
|
453
|
+
},
|
|
454
|
+
};
|
|
455
|
+
emitTieredOutput("/tmp/output", "tiered", tiers);
|
|
456
|
+
expect(errorOutput.length).toBe(1);
|
|
457
|
+
const output = errorOutput[0];
|
|
458
|
+
// Should be single line (may have trailing newline from console.error)
|
|
459
|
+
const lines = output.trim().split("\n");
|
|
460
|
+
expect(lines.length).toBe(1);
|
|
461
|
+
});
|
|
462
|
+
it("should emit parseable JSON Lines format", () => {
|
|
463
|
+
const tiers1 = {
|
|
464
|
+
executiveSummary: { path: "/tmp/1/exec.json", estimatedTokens: 500 },
|
|
465
|
+
toolSummaries: {
|
|
466
|
+
path: "/tmp/1/tools.json",
|
|
467
|
+
estimatedTokens: 1500,
|
|
468
|
+
toolCount: 10,
|
|
469
|
+
},
|
|
470
|
+
};
|
|
471
|
+
const tiers2 = {
|
|
472
|
+
executiveSummary: { path: "/tmp/2/exec.json", estimatedTokens: 600 },
|
|
473
|
+
toolSummaries: {
|
|
474
|
+
path: "/tmp/2/tools.json",
|
|
475
|
+
estimatedTokens: 1600,
|
|
476
|
+
toolCount: 12,
|
|
477
|
+
},
|
|
478
|
+
};
|
|
479
|
+
emitTieredOutput("/tmp/output1", "tiered", tiers1);
|
|
480
|
+
emitTieredOutput("/tmp/output2", "summary-only", tiers2);
|
|
481
|
+
expect(errorOutput.length).toBe(2);
|
|
482
|
+
// Each line should be valid JSON
|
|
483
|
+
const parsed1 = JSON.parse(errorOutput[0]);
|
|
484
|
+
const parsed2 = JSON.parse(errorOutput[1]);
|
|
485
|
+
expect(parsed1.outputDir).toBe("/tmp/output1");
|
|
486
|
+
expect(parsed2.outputDir).toBe("/tmp/output2");
|
|
487
|
+
});
|
|
488
|
+
});
|
|
489
|
+
});
|
|
490
|
+
describe("Code duplication documentation", () => {
|
|
491
|
+
it("should have matching TieredOutputEvent interface across files", () => {
|
|
492
|
+
// This test documents the intentional duplication between
|
|
493
|
+
// cli/src/lib/jsonl-events.ts and scripts/lib/jsonl-events.ts
|
|
494
|
+
// as documented in FIX-002 and FIX-003
|
|
495
|
+
// The interface structure is tested above. This test serves as
|
|
496
|
+
// documentation that the duplication is intentional and tracked.
|
|
497
|
+
const expectedEventType = "tiered_output_generated";
|
|
498
|
+
const expectedOutputFormats = ["tiered", "summary-only"];
|
|
499
|
+
expect(expectedEventType).toBe("tiered_output_generated");
|
|
500
|
+
expect(expectedOutputFormats).toEqual(["tiered", "summary-only"]);
|
|
501
|
+
});
|
|
502
|
+
it("should maintain consistency reminder for developers", () => {
|
|
503
|
+
// This test serves as a reminder that TieredOutputEvent and
|
|
504
|
+
// emitTieredOutput are duplicated across two files and must
|
|
505
|
+
// be kept in sync when changes are made.
|
|
506
|
+
const files = [
|
|
507
|
+
"cli/src/lib/jsonl-events.ts",
|
|
508
|
+
"scripts/lib/jsonl-events.ts",
|
|
509
|
+
];
|
|
510
|
+
// Documentation reminder
|
|
511
|
+
expect(files.length).toBe(2);
|
|
512
|
+
expect(files[0]).toContain("cli/src/lib/jsonl-events.ts");
|
|
513
|
+
expect(files[1]).toContain("scripts/lib/jsonl-events.ts");
|
|
514
|
+
});
|
|
515
|
+
});
|
|
516
|
+
});
|