fraim-framework 2.0.44 → 2.0.45
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/fraim.js +1 -1
- package/dist/registry/ai-manager-rules/design-phases/design-completeness-review.md +73 -0
- package/dist/registry/ai-manager-rules/design-phases/design-design.md +145 -0
- package/dist/registry/ai-manager-rules/design-phases/design.md +108 -0
- package/dist/registry/ai-manager-rules/design-phases/finalize.md +60 -0
- package/dist/registry/ai-manager-rules/design-phases/validate.md +125 -0
- package/dist/registry/ai-manager-rules/implement-phases/code.md +323 -0
- package/dist/registry/ai-manager-rules/implement-phases/completeness-review.md +94 -0
- package/dist/registry/ai-manager-rules/implement-phases/finalize.md +177 -0
- package/dist/registry/ai-manager-rules/implement-phases/implement-code.md +286 -0
- package/dist/registry/ai-manager-rules/implement-phases/implement-completeness-review.md +120 -0
- package/dist/registry/ai-manager-rules/implement-phases/implement-regression.md +173 -0
- package/dist/registry/ai-manager-rules/implement-phases/implement-repro.md +104 -0
- package/dist/registry/ai-manager-rules/implement-phases/implement-scoping.md +100 -0
- package/dist/registry/ai-manager-rules/implement-phases/implement-smoke.md +230 -0
- package/dist/registry/ai-manager-rules/implement-phases/implement-spike.md +121 -0
- package/dist/registry/ai-manager-rules/implement-phases/implement-validate.md +371 -0
- package/dist/registry/ai-manager-rules/implement-phases/quality-review.md +304 -0
- package/dist/registry/ai-manager-rules/implement-phases/regression.md +159 -0
- package/dist/registry/ai-manager-rules/implement-phases/repro.md +101 -0
- package/dist/registry/ai-manager-rules/implement-phases/scoping.md +93 -0
- package/dist/registry/ai-manager-rules/implement-phases/smoke.md +225 -0
- package/dist/registry/ai-manager-rules/implement-phases/spike.md +118 -0
- package/dist/registry/ai-manager-rules/implement-phases/validate.md +347 -0
- package/dist/registry/ai-manager-rules/shared-phases/finalize.md +169 -0
- package/dist/registry/ai-manager-rules/shared-phases/submit-pr.md +202 -0
- package/dist/registry/ai-manager-rules/shared-phases/wait-for-pr-review.md +170 -0
- package/dist/registry/ai-manager-rules/spec-phases/finalize.md +60 -0
- package/dist/registry/ai-manager-rules/spec-phases/spec-completeness-review.md +66 -0
- package/dist/registry/ai-manager-rules/spec-phases/spec-spec.md +139 -0
- package/dist/registry/ai-manager-rules/spec-phases/spec.md +102 -0
- package/dist/registry/ai-manager-rules/spec-phases/validate.md +118 -0
- package/dist/src/ai-manager/ai-manager.js +380 -119
- package/dist/src/ai-manager/evidence-validator.js +309 -0
- package/dist/src/ai-manager/phase-flow.js +244 -0
- package/dist/src/ai-manager/types.js +5 -0
- package/dist/src/fraim-mcp-server.js +45 -153
- package/dist/src/static-website-middleware.js +75 -0
- package/dist/tests/test-ai-coach-edge-cases.js +415 -0
- package/dist/tests/test-ai-coach-mcp-integration.js +432 -0
- package/dist/tests/test-ai-coach-performance.js +328 -0
- package/dist/tests/test-ai-coach-phase-content.js +264 -0
- package/dist/tests/test-ai-coach-workflows.js +487 -0
- package/dist/tests/test-ai-manager-phase-protocol.js +147 -0
- package/dist/tests/test-ai-manager.js +60 -71
- package/dist/tests/test-evidence-validation.js +221 -0
- package/dist/tests/test-mcp-lifecycle-methods.js +18 -23
- package/dist/tests/test-pr-review-integration.js +1 -0
- package/dist/tests/test-pr-review-workflow.js +299 -0
- package/dist/website/.nojekyll +0 -0
- package/dist/website/404.html +101 -0
- package/dist/website/CNAME +1 -0
- package/dist/website/README.md +22 -0
- package/dist/website/demo.html +604 -0
- package/dist/website/images/.gitkeep +1 -0
- package/dist/website/images/fraim-logo.png +0 -0
- package/dist/website/index.html +290 -0
- package/dist/website/pricing.html +414 -0
- package/dist/website/script.js +55 -0
- package/dist/website/styles.css +2647 -0
- package/package.json +2 -1
- package/registry/agent-guardrails.md +1 -1
- package/registry/stubs/workflows/brainstorming/blue-sky-brainstorming.md +11 -0
- package/registry/stubs/workflows/brainstorming/codebase-brainstorming.md +11 -0
- package/registry/stubs/workflows/compliance/detect-compliance-requirements.md +11 -0
- package/registry/stubs/workflows/compliance/generate-audit-evidence.md +11 -0
- package/registry/stubs/workflows/learning/synthesize-learnings.md +11 -0
- package/registry/stubs/workflows/legal/nda.md +11 -0
- package/registry/stubs/workflows/legal/patent-filing.md +11 -0
- package/registry/stubs/workflows/legal/trademark-filing.md +11 -0
- package/registry/stubs/workflows/product-building/design.md +1 -1
- package/registry/stubs/workflows/product-building/implement.md +1 -2
|
@@ -0,0 +1,328 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Performance and load tests for AI Coach
|
|
4
|
+
* Tests system behavior under various load conditions
|
|
5
|
+
*/
|
|
6
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
7
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
8
|
+
};
|
|
9
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
10
|
+
const node_test_1 = require("node:test");
|
|
11
|
+
const node_assert_1 = __importDefault(require("node:assert"));
|
|
12
|
+
const ai_manager_js_1 = require("../src/ai-manager/ai-manager.js");
|
|
13
|
+
const fs_1 = require("fs");
|
|
14
|
+
const path_1 = require("path");
|
|
15
|
+
(0, node_test_1.describe)('AI Coach Performance Tests', () => {
|
|
16
|
+
let coach;
|
|
17
|
+
let fileIndex;
|
|
18
|
+
(0, node_test_1.before)(() => {
|
|
19
|
+
// Create a comprehensive file index
|
|
20
|
+
fileIndex = new Map();
|
|
21
|
+
const phaseFiles = [
|
|
22
|
+
'ai-manager-rules/shared-phases/submit-pr.md',
|
|
23
|
+
'ai-manager-rules/shared-phases/wait-for-pr-review.md',
|
|
24
|
+
'ai-manager-rules/implement-phases/implement-scoping.md',
|
|
25
|
+
'ai-manager-rules/implement-phases/implement-repro.md',
|
|
26
|
+
'ai-manager-rules/implement-phases/implement-spike.md',
|
|
27
|
+
'ai-manager-rules/implement-phases/implement-code.md',
|
|
28
|
+
'ai-manager-rules/implement-phases/implement-validate.md',
|
|
29
|
+
'ai-manager-rules/implement-phases/implement-smoke.md',
|
|
30
|
+
'ai-manager-rules/implement-phases/implement-regression.md',
|
|
31
|
+
'ai-manager-rules/implement-phases/implement-completeness-review.md',
|
|
32
|
+
'ai-manager-rules/spec-phases/spec-spec.md',
|
|
33
|
+
'ai-manager-rules/spec-phases/spec-completeness-review.md',
|
|
34
|
+
'ai-manager-rules/design-phases/design-design.md',
|
|
35
|
+
'ai-manager-rules/design-phases/design-completeness-review.md'
|
|
36
|
+
];
|
|
37
|
+
phaseFiles.forEach(path => {
|
|
38
|
+
const fullPath = (0, path_1.join)(process.cwd(), 'registry', path);
|
|
39
|
+
if ((0, fs_1.existsSync)(fullPath)) {
|
|
40
|
+
fileIndex.set(path, { fullPath });
|
|
41
|
+
}
|
|
42
|
+
});
|
|
43
|
+
coach = new ai_manager_js_1.AICoach(fileIndex);
|
|
44
|
+
});
|
|
45
|
+
(0, node_test_1.describe)('Response Time Performance', () => {
|
|
46
|
+
(0, node_test_1.it)('should respond to phase requests within reasonable time', async () => {
|
|
47
|
+
const testCases = [
|
|
48
|
+
{ workflow: 'spec', phase: 'spec-spec' },
|
|
49
|
+
{ workflow: 'design', phase: 'design-design' },
|
|
50
|
+
{ workflow: 'implement', phase: 'implement-scoping' },
|
|
51
|
+
{ workflow: 'implement', phase: 'implement-code' }
|
|
52
|
+
];
|
|
53
|
+
for (const testCase of testCases) {
|
|
54
|
+
const startTime = Date.now();
|
|
55
|
+
const result = await coach.handleCoachingRequest({
|
|
56
|
+
workflowType: testCase.workflow,
|
|
57
|
+
currentPhase: testCase.phase,
|
|
58
|
+
status: 'starting',
|
|
59
|
+
issueNumber: '123',
|
|
60
|
+
evidence: { issueType: 'feature' }
|
|
61
|
+
});
|
|
62
|
+
const duration = Date.now() - startTime;
|
|
63
|
+
node_assert_1.default.ok(typeof result === 'string');
|
|
64
|
+
node_assert_1.default.ok(result.length > 0);
|
|
65
|
+
node_assert_1.default.ok(duration < 1000, `${testCase.workflow}/${testCase.phase} should respond within 1 second (took ${duration}ms)`);
|
|
66
|
+
}
|
|
67
|
+
});
|
|
68
|
+
(0, node_test_1.it)('should handle phase transitions efficiently', async () => {
|
|
69
|
+
const startTime = Date.now();
|
|
70
|
+
const result = await coach.handleCoachingRequest({
|
|
71
|
+
workflowType: 'spec',
|
|
72
|
+
currentPhase: 'spec-spec',
|
|
73
|
+
status: 'complete',
|
|
74
|
+
issueNumber: '123',
|
|
75
|
+
evidence: { issueType: 'feature', specComplete: true }
|
|
76
|
+
});
|
|
77
|
+
const duration = Date.now() - startTime;
|
|
78
|
+
node_assert_1.default.ok(result.includes('Next Phase: spec-completeness-review'));
|
|
79
|
+
node_assert_1.default.ok(duration < 1000, `Phase transition should complete within 1 second (took ${duration}ms)`);
|
|
80
|
+
});
|
|
81
|
+
(0, node_test_1.it)('should handle completion messages efficiently', async () => {
|
|
82
|
+
const startTime = Date.now();
|
|
83
|
+
const result = await coach.handleCoachingRequest({
|
|
84
|
+
workflowType: 'spec',
|
|
85
|
+
currentPhase: 'wait-for-pr-review',
|
|
86
|
+
status: 'complete',
|
|
87
|
+
issueNumber: '123',
|
|
88
|
+
evidence: { issueType: 'feature', prApproved: true }
|
|
89
|
+
});
|
|
90
|
+
const duration = Date.now() - startTime;
|
|
91
|
+
node_assert_1.default.ok(result.includes('Spec Complete!'));
|
|
92
|
+
node_assert_1.default.ok(duration < 1000, `Completion message should generate within 1 second (took ${duration}ms)`);
|
|
93
|
+
});
|
|
94
|
+
});
|
|
95
|
+
(0, node_test_1.describe)('Load Testing', () => {
|
|
96
|
+
(0, node_test_1.it)('should handle 100 concurrent requests', async () => {
|
|
97
|
+
const concurrentRequests = 100;
|
|
98
|
+
const startTime = Date.now();
|
|
99
|
+
const requests = Array.from({ length: concurrentRequests }, (_, i) => coach.handleCoachingRequest({
|
|
100
|
+
workflowType: 'spec',
|
|
101
|
+
currentPhase: 'spec-spec',
|
|
102
|
+
status: 'starting',
|
|
103
|
+
issueNumber: `${i}`,
|
|
104
|
+
evidence: { issueType: 'feature' }
|
|
105
|
+
}));
|
|
106
|
+
const results = await Promise.all(requests);
|
|
107
|
+
const duration = Date.now() - startTime;
|
|
108
|
+
// All requests should complete successfully
|
|
109
|
+
node_assert_1.default.strictEqual(results.length, concurrentRequests);
|
|
110
|
+
for (const result of results) {
|
|
111
|
+
node_assert_1.default.ok(typeof result === 'string');
|
|
112
|
+
node_assert_1.default.ok(result.length > 0);
|
|
113
|
+
}
|
|
114
|
+
console.log(`${concurrentRequests} concurrent requests completed in ${duration}ms`);
|
|
115
|
+
node_assert_1.default.ok(duration < 10000, `${concurrentRequests} concurrent requests should complete within 10 seconds`);
|
|
116
|
+
});
|
|
117
|
+
(0, node_test_1.it)('should handle mixed workflow types under load', async () => {
|
|
118
|
+
const requestsPerWorkflow = 25;
|
|
119
|
+
const workflows = ['spec', 'design', 'implement'];
|
|
120
|
+
const phases = ['spec-spec', 'design-design', 'implement-scoping'];
|
|
121
|
+
const startTime = Date.now();
|
|
122
|
+
const allRequests = workflows.flatMap((workflow, workflowIndex) => Array.from({ length: requestsPerWorkflow }, (_, i) => coach.handleCoachingRequest({
|
|
123
|
+
workflowType: workflow,
|
|
124
|
+
currentPhase: phases[workflowIndex],
|
|
125
|
+
status: 'starting',
|
|
126
|
+
issueNumber: `${workflowIndex}-${i}`,
|
|
127
|
+
evidence: { issueType: 'feature' }
|
|
128
|
+
})));
|
|
129
|
+
const results = await Promise.all(allRequests);
|
|
130
|
+
const duration = Date.now() - startTime;
|
|
131
|
+
node_assert_1.default.strictEqual(results.length, requestsPerWorkflow * workflows.length);
|
|
132
|
+
for (const result of results) {
|
|
133
|
+
node_assert_1.default.ok(typeof result === 'string');
|
|
134
|
+
node_assert_1.default.ok(result.length > 0);
|
|
135
|
+
}
|
|
136
|
+
console.log(`${results.length} mixed workflow requests completed in ${duration}ms`);
|
|
137
|
+
node_assert_1.default.ok(duration < 15000, 'Mixed workflow load test should complete within 15 seconds');
|
|
138
|
+
});
|
|
139
|
+
(0, node_test_1.it)('should handle sequential high-frequency requests', async () => {
|
|
140
|
+
const requestCount = 200;
|
|
141
|
+
const startTime = Date.now();
|
|
142
|
+
for (let i = 0; i < requestCount; i++) {
|
|
143
|
+
const result = await coach.handleCoachingRequest({
|
|
144
|
+
workflowType: 'spec',
|
|
145
|
+
currentPhase: 'spec-spec',
|
|
146
|
+
status: 'starting',
|
|
147
|
+
issueNumber: `${i}`,
|
|
148
|
+
evidence: { issueType: 'feature' }
|
|
149
|
+
});
|
|
150
|
+
node_assert_1.default.ok(typeof result === 'string');
|
|
151
|
+
node_assert_1.default.ok(result.length > 0);
|
|
152
|
+
}
|
|
153
|
+
const duration = Date.now() - startTime;
|
|
154
|
+
const avgResponseTime = duration / requestCount;
|
|
155
|
+
console.log(`${requestCount} sequential requests completed in ${duration}ms (avg: ${avgResponseTime.toFixed(2)}ms per request)`);
|
|
156
|
+
node_assert_1.default.ok(avgResponseTime < 50, 'Average response time should be under 50ms per request');
|
|
157
|
+
});
|
|
158
|
+
});
|
|
159
|
+
(0, node_test_1.describe)('Memory Usage', () => {
|
|
160
|
+
(0, node_test_1.it)('should not leak memory during repeated operations', async () => {
|
|
161
|
+
const initialMemory = process.memoryUsage();
|
|
162
|
+
// Perform many operations
|
|
163
|
+
for (let i = 0; i < 1000; i++) {
|
|
164
|
+
await coach.handleCoachingRequest({
|
|
165
|
+
workflowType: 'spec',
|
|
166
|
+
currentPhase: 'spec-spec',
|
|
167
|
+
status: 'starting',
|
|
168
|
+
issueNumber: `${i}`,
|
|
169
|
+
evidence: { issueType: 'feature' }
|
|
170
|
+
});
|
|
171
|
+
// Occasionally force garbage collection if available
|
|
172
|
+
if (i % 100 === 0 && global.gc) {
|
|
173
|
+
global.gc();
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
const finalMemory = process.memoryUsage();
|
|
177
|
+
const memoryIncrease = finalMemory.heapUsed - initialMemory.heapUsed;
|
|
178
|
+
const memoryIncreaseMB = memoryIncrease / (1024 * 1024);
|
|
179
|
+
console.log(`Memory increase after 1000 operations: ${memoryIncreaseMB.toFixed(2)}MB`);
|
|
180
|
+
// Memory increase should be reasonable (less than 50MB)
|
|
181
|
+
node_assert_1.default.ok(memoryIncreaseMB < 50, 'Memory increase should be reasonable');
|
|
182
|
+
});
|
|
183
|
+
(0, node_test_1.it)('should handle large evidence objects efficiently', async () => {
|
|
184
|
+
const largeEvidence = {
|
|
185
|
+
issueType: 'feature',
|
|
186
|
+
largeArray: Array.from({ length: 10000 }, (_, i) => ({
|
|
187
|
+
id: i,
|
|
188
|
+
data: `data-${i}`,
|
|
189
|
+
metadata: {
|
|
190
|
+
timestamp: Date.now(),
|
|
191
|
+
tags: [`tag-${i % 10}`, `category-${i % 5}`]
|
|
192
|
+
}
|
|
193
|
+
}))
|
|
194
|
+
};
|
|
195
|
+
const startTime = Date.now();
|
|
196
|
+
const result = await coach.handleCoachingRequest({
|
|
197
|
+
workflowType: 'spec',
|
|
198
|
+
currentPhase: 'spec-spec',
|
|
199
|
+
status: 'starting',
|
|
200
|
+
issueNumber: '123',
|
|
201
|
+
evidence: largeEvidence
|
|
202
|
+
});
|
|
203
|
+
const duration = Date.now() - startTime;
|
|
204
|
+
node_assert_1.default.ok(typeof result === 'string');
|
|
205
|
+
node_assert_1.default.ok(result.length > 0);
|
|
206
|
+
node_assert_1.default.ok(duration < 2000, `Large evidence handling should complete within 2 seconds (took ${duration}ms)`);
|
|
207
|
+
});
|
|
208
|
+
});
|
|
209
|
+
(0, node_test_1.describe)('File System Performance', () => {
|
|
210
|
+
(0, node_test_1.it)('should cache file reads efficiently', async () => {
|
|
211
|
+
// First request (cold cache)
|
|
212
|
+
const startTime1 = Date.now();
|
|
213
|
+
const result1 = await coach.handleCoachingRequest({
|
|
214
|
+
workflowType: 'spec',
|
|
215
|
+
currentPhase: 'spec-spec',
|
|
216
|
+
status: 'starting',
|
|
217
|
+
issueNumber: '123',
|
|
218
|
+
evidence: { issueType: 'feature' }
|
|
219
|
+
});
|
|
220
|
+
const duration1 = Date.now() - startTime1;
|
|
221
|
+
// Second request (warm cache)
|
|
222
|
+
const startTime2 = Date.now();
|
|
223
|
+
const result2 = await coach.handleCoachingRequest({
|
|
224
|
+
workflowType: 'spec',
|
|
225
|
+
currentPhase: 'spec-spec',
|
|
226
|
+
status: 'starting',
|
|
227
|
+
issueNumber: '456',
|
|
228
|
+
evidence: { issueType: 'feature' }
|
|
229
|
+
});
|
|
230
|
+
const duration2 = Date.now() - startTime2;
|
|
231
|
+
node_assert_1.default.ok(typeof result1 === 'string');
|
|
232
|
+
node_assert_1.default.ok(typeof result2 === 'string');
|
|
233
|
+
console.log(`First request: ${duration1}ms, Second request: ${duration2}ms`);
|
|
234
|
+
// Second request should be faster or similar (file system caching)
|
|
235
|
+
// Allow some variance due to system factors
|
|
236
|
+
node_assert_1.default.ok(duration2 <= duration1 * 2, 'Subsequent requests should benefit from caching');
|
|
237
|
+
});
|
|
238
|
+
(0, node_test_1.it)('should handle multiple different phase files efficiently', async () => {
|
|
239
|
+
const phases = [
|
|
240
|
+
{ workflow: 'spec', phase: 'spec-spec' },
|
|
241
|
+
{ workflow: 'spec', phase: 'spec-completeness-review' },
|
|
242
|
+
{ workflow: 'design', phase: 'design-design' },
|
|
243
|
+
{ workflow: 'design', phase: 'design-completeness-review' },
|
|
244
|
+
{ workflow: 'implement', phase: 'implement-scoping' },
|
|
245
|
+
{ workflow: 'implement', phase: 'implement-code' }
|
|
246
|
+
];
|
|
247
|
+
const startTime = Date.now();
|
|
248
|
+
for (const { workflow, phase } of phases) {
|
|
249
|
+
const result = await coach.handleCoachingRequest({
|
|
250
|
+
workflowType: workflow,
|
|
251
|
+
currentPhase: phase,
|
|
252
|
+
status: 'starting',
|
|
253
|
+
issueNumber: '123',
|
|
254
|
+
evidence: { issueType: 'feature' }
|
|
255
|
+
});
|
|
256
|
+
node_assert_1.default.ok(typeof result === 'string');
|
|
257
|
+
node_assert_1.default.ok(result.length > 0);
|
|
258
|
+
}
|
|
259
|
+
const duration = Date.now() - startTime;
|
|
260
|
+
const avgTimePerFile = duration / phases.length;
|
|
261
|
+
console.log(`${phases.length} different phase files loaded in ${duration}ms (avg: ${avgTimePerFile.toFixed(2)}ms per file)`);
|
|
262
|
+
node_assert_1.default.ok(avgTimePerFile < 200, 'Average file load time should be under 200ms');
|
|
263
|
+
});
|
|
264
|
+
});
|
|
265
|
+
(0, node_test_1.describe)('Scalability Tests', () => {
|
|
266
|
+
(0, node_test_1.it)('should maintain performance with increasing request complexity', async () => {
|
|
267
|
+
const complexityLevels = [
|
|
268
|
+
{ name: 'simple', evidence: { issueType: 'feature' } },
|
|
269
|
+
{ name: 'medium', evidence: {
|
|
270
|
+
issueType: 'feature',
|
|
271
|
+
metadata: { tags: ['tag1', 'tag2'], priority: 'high' }
|
|
272
|
+
} },
|
|
273
|
+
{ name: 'complex', evidence: {
|
|
274
|
+
issueType: 'feature',
|
|
275
|
+
metadata: {
|
|
276
|
+
tags: Array.from({ length: 100 }, (_, i) => `tag-${i}`),
|
|
277
|
+
priority: 'high',
|
|
278
|
+
history: Array.from({ length: 50 }, (_, i) => ({
|
|
279
|
+
timestamp: Date.now() - i * 1000,
|
|
280
|
+
action: `action-${i}`
|
|
281
|
+
}))
|
|
282
|
+
}
|
|
283
|
+
} }
|
|
284
|
+
];
|
|
285
|
+
for (const level of complexityLevels) {
|
|
286
|
+
const startTime = Date.now();
|
|
287
|
+
const result = await coach.handleCoachingRequest({
|
|
288
|
+
workflowType: 'spec',
|
|
289
|
+
currentPhase: 'spec-spec',
|
|
290
|
+
status: 'starting',
|
|
291
|
+
issueNumber: '123',
|
|
292
|
+
evidence: level.evidence
|
|
293
|
+
});
|
|
294
|
+
const duration = Date.now() - startTime;
|
|
295
|
+
node_assert_1.default.ok(typeof result === 'string');
|
|
296
|
+
node_assert_1.default.ok(result.length > 0);
|
|
297
|
+
console.log(`${level.name} complexity request: ${duration}ms`);
|
|
298
|
+
node_assert_1.default.ok(duration < 1000, `${level.name} complexity should complete within 1 second`);
|
|
299
|
+
}
|
|
300
|
+
});
|
|
301
|
+
(0, node_test_1.it)('should handle burst traffic patterns', async () => {
|
|
302
|
+
// Simulate burst traffic: many requests in short time, then quiet period
|
|
303
|
+
const burstSize = 50;
|
|
304
|
+
const burstCount = 3;
|
|
305
|
+
for (let burst = 0; burst < burstCount; burst++) {
|
|
306
|
+
const startTime = Date.now();
|
|
307
|
+
const requests = Array.from({ length: burstSize }, (_, i) => coach.handleCoachingRequest({
|
|
308
|
+
workflowType: 'spec',
|
|
309
|
+
currentPhase: 'spec-spec',
|
|
310
|
+
status: 'starting',
|
|
311
|
+
issueNumber: `burst-${burst}-${i}`,
|
|
312
|
+
evidence: { issueType: 'feature' }
|
|
313
|
+
}));
|
|
314
|
+
const results = await Promise.all(requests);
|
|
315
|
+
const duration = Date.now() - startTime;
|
|
316
|
+
node_assert_1.default.strictEqual(results.length, burstSize);
|
|
317
|
+
for (const result of results) {
|
|
318
|
+
node_assert_1.default.ok(typeof result === 'string');
|
|
319
|
+
node_assert_1.default.ok(result.length > 0);
|
|
320
|
+
}
|
|
321
|
+
console.log(`Burst ${burst + 1}: ${burstSize} requests in ${duration}ms`);
|
|
322
|
+
node_assert_1.default.ok(duration < 5000, `Burst ${burst + 1} should complete within 5 seconds`);
|
|
323
|
+
// Brief pause between bursts
|
|
324
|
+
await new Promise(resolve => setTimeout(resolve, 100));
|
|
325
|
+
}
|
|
326
|
+
});
|
|
327
|
+
});
|
|
328
|
+
});
|
|
@@ -0,0 +1,264 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Tests for AI Coach phase content validation
|
|
4
|
+
* Ensures all phase files have proper structure and required content
|
|
5
|
+
*/
|
|
6
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
7
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
8
|
+
};
|
|
9
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
10
|
+
const node_test_1 = require("node:test");
|
|
11
|
+
const node_assert_1 = __importDefault(require("node:assert"));
|
|
12
|
+
const fs_1 = require("fs");
|
|
13
|
+
const path_1 = require("path");
|
|
14
|
+
(0, node_test_1.describe)('AI Coach Phase Content Validation', () => {
|
|
15
|
+
const registryPath = (0, path_1.join)(process.cwd(), 'registry', 'ai-manager-rules');
|
|
16
|
+
(0, node_test_1.describe)('Phase File Structure', () => {
|
|
17
|
+
(0, node_test_1.it)('should have all required phase directories', () => {
|
|
18
|
+
const requiredDirs = [
|
|
19
|
+
'implement-phases',
|
|
20
|
+
'spec-phases',
|
|
21
|
+
'design-phases',
|
|
22
|
+
'shared-phases'
|
|
23
|
+
];
|
|
24
|
+
for (const dir of requiredDirs) {
|
|
25
|
+
const dirPath = (0, path_1.join)(registryPath, dir);
|
|
26
|
+
node_assert_1.default.ok((0, fs_1.existsSync)(dirPath), `Directory should exist: ${dir}`);
|
|
27
|
+
}
|
|
28
|
+
});
|
|
29
|
+
(0, node_test_1.it)('should have all required implement phase files', () => {
|
|
30
|
+
const implementPhasesDir = (0, path_1.join)(registryPath, 'implement-phases');
|
|
31
|
+
const requiredFiles = [
|
|
32
|
+
'implement-scoping.md',
|
|
33
|
+
'implement-repro.md',
|
|
34
|
+
'implement-spike.md',
|
|
35
|
+
'implement-code.md',
|
|
36
|
+
'implement-validate.md',
|
|
37
|
+
'implement-smoke.md',
|
|
38
|
+
'implement-regression.md',
|
|
39
|
+
'implement-completeness-review.md'
|
|
40
|
+
];
|
|
41
|
+
for (const file of requiredFiles) {
|
|
42
|
+
const filePath = (0, path_1.join)(implementPhasesDir, file);
|
|
43
|
+
node_assert_1.default.ok((0, fs_1.existsSync)(filePath), `Implement phase file should exist: ${file}`);
|
|
44
|
+
}
|
|
45
|
+
});
|
|
46
|
+
(0, node_test_1.it)('should have all required spec phase files', () => {
|
|
47
|
+
const specPhasesDir = (0, path_1.join)(registryPath, 'spec-phases');
|
|
48
|
+
const requiredFiles = [
|
|
49
|
+
'spec-spec.md',
|
|
50
|
+
'spec-completeness-review.md'
|
|
51
|
+
];
|
|
52
|
+
for (const file of requiredFiles) {
|
|
53
|
+
const filePath = (0, path_1.join)(specPhasesDir, file);
|
|
54
|
+
node_assert_1.default.ok((0, fs_1.existsSync)(filePath), `Spec phase file should exist: ${file}`);
|
|
55
|
+
}
|
|
56
|
+
});
|
|
57
|
+
(0, node_test_1.it)('should have all required design phase files', () => {
|
|
58
|
+
const designPhasesDir = (0, path_1.join)(registryPath, 'design-phases');
|
|
59
|
+
const requiredFiles = [
|
|
60
|
+
'design-design.md',
|
|
61
|
+
'design-completeness-review.md'
|
|
62
|
+
];
|
|
63
|
+
for (const file of requiredFiles) {
|
|
64
|
+
const filePath = (0, path_1.join)(designPhasesDir, file);
|
|
65
|
+
node_assert_1.default.ok((0, fs_1.existsSync)(filePath), `Design phase file should exist: ${file}`);
|
|
66
|
+
}
|
|
67
|
+
});
|
|
68
|
+
(0, node_test_1.it)('should have all required shared phase files', () => {
|
|
69
|
+
const sharedPhasesDir = (0, path_1.join)(registryPath, 'shared-phases');
|
|
70
|
+
const requiredFiles = [
|
|
71
|
+
'submit-pr.md',
|
|
72
|
+
'wait-for-pr-review.md'
|
|
73
|
+
];
|
|
74
|
+
for (const file of requiredFiles) {
|
|
75
|
+
const filePath = (0, path_1.join)(sharedPhasesDir, file);
|
|
76
|
+
node_assert_1.default.ok((0, fs_1.existsSync)(filePath), `Shared phase file should exist: ${file}`);
|
|
77
|
+
}
|
|
78
|
+
});
|
|
79
|
+
});
|
|
80
|
+
(0, node_test_1.describe)('Phase File Content Structure', () => {
|
|
81
|
+
function validatePhaseFileContent(filePath, expectedPhase) {
|
|
82
|
+
const content = (0, fs_1.readFileSync)(filePath, 'utf8');
|
|
83
|
+
// Check required sections
|
|
84
|
+
node_assert_1.default.ok(content.includes('# Phase:'), `File should have phase header: ${filePath}`);
|
|
85
|
+
node_assert_1.default.ok(content.includes('## INTENT'), `File should have INTENT section: ${filePath}`);
|
|
86
|
+
node_assert_1.default.ok(content.includes('## OUTCOME'), `File should have OUTCOME section: ${filePath}`);
|
|
87
|
+
node_assert_1.default.ok(content.includes('## WORKFLOW') || content.includes('## PRINCIPLES'), `File should have WORKFLOW or PRINCIPLES section: ${filePath}`);
|
|
88
|
+
node_assert_1.default.ok(content.includes('## VALIDATION'), `File should have VALIDATION section: ${filePath}`);
|
|
89
|
+
// Check validation subsections
|
|
90
|
+
node_assert_1.default.ok(content.includes('### Phase Complete When:'), `File should have completion criteria: ${filePath}`);
|
|
91
|
+
node_assert_1.default.ok(content.includes('### Report Back:') || content.includes('### Phase Incomplete If:'), `File should have reporting instructions: ${filePath}`);
|
|
92
|
+
// Check for seekCoachingOnNextStep call
|
|
93
|
+
node_assert_1.default.ok(content.includes('seekCoachingOnNextStep'), `File should include seekCoachingOnNextStep call: ${filePath}`);
|
|
94
|
+
// Verify phase name in header matches expected
|
|
95
|
+
const headerMatch = content.match(/# Phase: (.+)/);
|
|
96
|
+
node_assert_1.default.ok(headerMatch, `File should have proper phase header: ${filePath}`);
|
|
97
|
+
const actualPhase = headerMatch[1].trim();
|
|
98
|
+
node_assert_1.default.ok(actualPhase.toLowerCase().includes(expectedPhase.toLowerCase()) ||
|
|
99
|
+
expectedPhase.toLowerCase().includes(actualPhase.toLowerCase()), `Phase header "${actualPhase}" should match expected "${expectedPhase}" in ${filePath}`);
|
|
100
|
+
}
|
|
101
|
+
(0, node_test_1.it)('should validate all implement phase files have proper structure', () => {
|
|
102
|
+
const implementPhasesDir = (0, path_1.join)(registryPath, 'implement-phases');
|
|
103
|
+
const files = (0, fs_1.readdirSync)(implementPhasesDir).filter(f => f.endsWith('.md'));
|
|
104
|
+
for (const file of files) {
|
|
105
|
+
const filePath = (0, path_1.join)(implementPhasesDir, file);
|
|
106
|
+
const expectedPhase = file.replace('.md', '');
|
|
107
|
+
validatePhaseFileContent(filePath, expectedPhase);
|
|
108
|
+
}
|
|
109
|
+
});
|
|
110
|
+
(0, node_test_1.it)('should validate all spec phase files have proper structure', () => {
|
|
111
|
+
const specPhasesDir = (0, path_1.join)(registryPath, 'spec-phases');
|
|
112
|
+
const files = (0, fs_1.readdirSync)(specPhasesDir).filter(f => f.endsWith('.md'));
|
|
113
|
+
for (const file of files) {
|
|
114
|
+
const filePath = (0, path_1.join)(specPhasesDir, file);
|
|
115
|
+
const expectedPhase = file.replace('.md', '');
|
|
116
|
+
validatePhaseFileContent(filePath, expectedPhase);
|
|
117
|
+
}
|
|
118
|
+
});
|
|
119
|
+
(0, node_test_1.it)('should validate all design phase files have proper structure', () => {
|
|
120
|
+
const designPhasesDir = (0, path_1.join)(registryPath, 'design-phases');
|
|
121
|
+
const files = (0, fs_1.readdirSync)(designPhasesDir).filter(f => f.endsWith('.md'));
|
|
122
|
+
for (const file of files) {
|
|
123
|
+
const filePath = (0, path_1.join)(designPhasesDir, file);
|
|
124
|
+
const expectedPhase = file.replace('.md', '');
|
|
125
|
+
validatePhaseFileContent(filePath, expectedPhase);
|
|
126
|
+
}
|
|
127
|
+
});
|
|
128
|
+
(0, node_test_1.it)('should validate shared phase files have proper structure', () => {
|
|
129
|
+
const sharedFiles = ['submit-pr.md', 'wait-for-pr-review.md'];
|
|
130
|
+
for (const file of sharedFiles) {
|
|
131
|
+
const filePath = (0, path_1.join)(registryPath, 'shared-phases', file);
|
|
132
|
+
const expectedPhase = file.replace('.md', '');
|
|
133
|
+
validatePhaseFileContent(filePath, expectedPhase);
|
|
134
|
+
}
|
|
135
|
+
});
|
|
136
|
+
});
|
|
137
|
+
(0, node_test_1.describe)('Phase File Naming Consistency', () => {
|
|
138
|
+
(0, node_test_1.it)('should have consistent implement phase naming', () => {
|
|
139
|
+
const implementPhasesDir = (0, path_1.join)(registryPath, 'implement-phases');
|
|
140
|
+
const files = (0, fs_1.readdirSync)(implementPhasesDir).filter(f => f.endsWith('.md'));
|
|
141
|
+
for (const file of files) {
|
|
142
|
+
node_assert_1.default.ok(file.startsWith('implement-'), `Implement phase file should start with 'implement-': ${file}`);
|
|
143
|
+
node_assert_1.default.ok(file.endsWith('.md'), `Implement phase file should end with '.md': ${file}`);
|
|
144
|
+
// Check that phase name in file matches filename
|
|
145
|
+
const content = (0, fs_1.readFileSync)((0, path_1.join)(implementPhasesDir, file), 'utf8');
|
|
146
|
+
const expectedPhase = file.replace('.md', '');
|
|
147
|
+
// Allow for some flexibility in header format
|
|
148
|
+
const hasCorrectHeader = content.includes(`# Phase: ${expectedPhase}`) ||
|
|
149
|
+
content.includes(`# Phase: Implement-`) ||
|
|
150
|
+
content.includes(`# Phase: ${expectedPhase.replace('implement-', '').charAt(0).toUpperCase() + expectedPhase.replace('implement-', '').slice(1)}`);
|
|
151
|
+
node_assert_1.default.ok(hasCorrectHeader, `Phase file header should match filename for: ${file}`);
|
|
152
|
+
}
|
|
153
|
+
});
|
|
154
|
+
(0, node_test_1.it)('should have consistent spec phase naming', () => {
|
|
155
|
+
const specPhasesDir = (0, path_1.join)(registryPath, 'spec-phases');
|
|
156
|
+
const files = (0, fs_1.readdirSync)(specPhasesDir).filter(f => f.endsWith('.md'));
|
|
157
|
+
for (const file of files) {
|
|
158
|
+
node_assert_1.default.ok(file.startsWith('spec-'), `Spec phase file should start with 'spec-': ${file}`);
|
|
159
|
+
const content = (0, fs_1.readFileSync)((0, path_1.join)(specPhasesDir, file), 'utf8');
|
|
160
|
+
const expectedPhase = file.replace('.md', '');
|
|
161
|
+
const hasCorrectHeader = content.includes(`# Phase: ${expectedPhase}`) ||
|
|
162
|
+
content.includes(`# Phase: Spec-`) ||
|
|
163
|
+
content.includes(`# Phase: Spec `);
|
|
164
|
+
node_assert_1.default.ok(hasCorrectHeader, `Phase file header should match filename for: ${file}`);
|
|
165
|
+
}
|
|
166
|
+
});
|
|
167
|
+
(0, node_test_1.it)('should have consistent design phase naming', () => {
|
|
168
|
+
const designPhasesDir = (0, path_1.join)(registryPath, 'design-phases');
|
|
169
|
+
const files = (0, fs_1.readdirSync)(designPhasesDir).filter(f => f.endsWith('.md'));
|
|
170
|
+
for (const file of files) {
|
|
171
|
+
node_assert_1.default.ok(file.startsWith('design-'), `Design phase file should start with 'design-': ${file}`);
|
|
172
|
+
const content = (0, fs_1.readFileSync)((0, path_1.join)(designPhasesDir, file), 'utf8');
|
|
173
|
+
const expectedPhase = file.replace('.md', '');
|
|
174
|
+
const hasCorrectHeader = content.includes(`# Phase: ${expectedPhase}`) ||
|
|
175
|
+
content.includes(`# Phase: Design-`) ||
|
|
176
|
+
content.includes(`# Phase: Design `);
|
|
177
|
+
node_assert_1.default.ok(hasCorrectHeader, `Phase file header should match filename for: ${file}`);
|
|
178
|
+
}
|
|
179
|
+
});
|
|
180
|
+
});
|
|
181
|
+
(0, node_test_1.describe)('Phase Content Quality', () => {
|
|
182
|
+
(0, node_test_1.it)('should have meaningful content in all phase files', () => {
|
|
183
|
+
const allDirs = ['implement-phases', 'spec-phases', 'design-phases', 'shared-phases'];
|
|
184
|
+
for (const dir of allDirs) {
|
|
185
|
+
const dirPath = (0, path_1.join)(registryPath, dir);
|
|
186
|
+
const files = (0, fs_1.readdirSync)(dirPath).filter(f => f.endsWith('.md'));
|
|
187
|
+
for (const file of files) {
|
|
188
|
+
const filePath = (0, path_1.join)(dirPath, file);
|
|
189
|
+
const content = (0, fs_1.readFileSync)(filePath, 'utf8');
|
|
190
|
+
// Check minimum content length
|
|
191
|
+
node_assert_1.default.ok(content.length > 500, `Phase file should have substantial content: ${file}`);
|
|
192
|
+
// Check for placeholder text
|
|
193
|
+
node_assert_1.default.ok(!content.includes('TODO'), `Phase file should not contain TODO placeholders: ${file}`);
|
|
194
|
+
node_assert_1.default.ok(!content.includes('FIXME'), `Phase file should not contain FIXME placeholders: ${file}`);
|
|
195
|
+
node_assert_1.default.ok(!content.includes('[PLACEHOLDER]'), `Phase file should not contain placeholder text: ${file}`);
|
|
196
|
+
// Check for proper seekCoachingOnNextStep usage
|
|
197
|
+
const seekCalls = content.match(/seekCoachingOnNextStep/g);
|
|
198
|
+
node_assert_1.default.ok(seekCalls && seekCalls.length >= 1, `Phase file should have at least one seekCoachingOnNextStep call: ${file}`);
|
|
199
|
+
// Check that currentPhase parameter matches file
|
|
200
|
+
const expectedPhase = file.replace('.md', '');
|
|
201
|
+
const phaseParamMatch = content.match(/currentPhase:\s*["']([^"']+)["']/);
|
|
202
|
+
if (phaseParamMatch) {
|
|
203
|
+
const actualPhase = phaseParamMatch[1];
|
|
204
|
+
node_assert_1.default.strictEqual(actualPhase, expectedPhase, `currentPhase parameter should match filename in ${file}`);
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
});
|
|
209
|
+
(0, node_test_1.it)('should have consistent workflow type references', () => {
|
|
210
|
+
const workflowDirs = [
|
|
211
|
+
{ dir: 'implement-phases', expectedWorkflow: 'implement' },
|
|
212
|
+
{ dir: 'spec-phases', expectedWorkflow: 'spec' },
|
|
213
|
+
{ dir: 'design-phases', expectedWorkflow: 'design' }
|
|
214
|
+
];
|
|
215
|
+
for (const { dir, expectedWorkflow } of workflowDirs) {
|
|
216
|
+
const dirPath = (0, path_1.join)(registryPath, dir);
|
|
217
|
+
const files = (0, fs_1.readdirSync)(dirPath).filter(f => f.endsWith('.md'));
|
|
218
|
+
for (const file of files) {
|
|
219
|
+
const filePath = (0, path_1.join)(dirPath, file);
|
|
220
|
+
const content = (0, fs_1.readFileSync)(filePath, 'utf8');
|
|
221
|
+
// Check workflowType parameter in seekCoachingOnNextStep calls
|
|
222
|
+
const workflowMatches = content.match(/workflowType:\s*["']([^"']+)["']/g);
|
|
223
|
+
if (workflowMatches) {
|
|
224
|
+
for (const match of workflowMatches) {
|
|
225
|
+
const workflowType = match.match(/["']([^"']+)["']/)?.[1];
|
|
226
|
+
node_assert_1.default.strictEqual(workflowType, expectedWorkflow, `workflowType should be "${expectedWorkflow}" in ${file}`);
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
});
|
|
232
|
+
});
|
|
233
|
+
(0, node_test_1.describe)('Shared Phase Consistency', () => {
|
|
234
|
+
(0, node_test_1.it)('should have only shared phases in shared-phases directory', () => {
|
|
235
|
+
// Check that submit-pr and wait-for-pr-review exist in shared-phases
|
|
236
|
+
const sharedSubmitPr = (0, path_1.join)(registryPath, 'shared-phases', 'submit-pr.md');
|
|
237
|
+
const sharedWaitForReview = (0, path_1.join)(registryPath, 'shared-phases', 'wait-for-pr-review.md');
|
|
238
|
+
node_assert_1.default.ok((0, fs_1.existsSync)(sharedSubmitPr), 'Shared submit-pr should exist');
|
|
239
|
+
node_assert_1.default.ok((0, fs_1.existsSync)(sharedWaitForReview), 'Shared wait-for-pr-review should exist');
|
|
240
|
+
// Check that no other directories have these files
|
|
241
|
+
const otherDirs = ['implement-phases', 'spec-phases', 'design-phases'];
|
|
242
|
+
for (const dir of otherDirs) {
|
|
243
|
+
const dirPath = (0, path_1.join)(registryPath, dir);
|
|
244
|
+
const submitPrFile = (0, path_1.join)(dirPath, 'submit-pr.md');
|
|
245
|
+
const waitForReviewFile = (0, path_1.join)(dirPath, 'wait-for-pr-review.md');
|
|
246
|
+
node_assert_1.default.ok(!(0, fs_1.existsSync)(submitPrFile), `submit-pr should not exist in ${dir} - should only be in shared-phases`);
|
|
247
|
+
node_assert_1.default.ok(!(0, fs_1.existsSync)(waitForReviewFile), `wait-for-pr-review should not exist in ${dir} - should only be in shared-phases`);
|
|
248
|
+
}
|
|
249
|
+
});
|
|
250
|
+
(0, node_test_1.it)('should have workflow-agnostic shared phase content', () => {
|
|
251
|
+
const sharedFiles = ['submit-pr.md', 'wait-for-pr-review.md'];
|
|
252
|
+
for (const file of sharedFiles) {
|
|
253
|
+
const filePath = (0, path_1.join)(registryPath, 'shared-phases', file);
|
|
254
|
+
const content = (0, fs_1.readFileSync)(filePath, 'utf8');
|
|
255
|
+
// Should contain generic workflow references
|
|
256
|
+
node_assert_1.default.ok(content.includes('{workflow_type}') || content.includes('workflow'), `${file} should reference workflows generically`);
|
|
257
|
+
// Should not contain workflow-specific content
|
|
258
|
+
node_assert_1.default.ok(!content.includes('implement-specific'), `${file} should not contain implement-specific content`);
|
|
259
|
+
node_assert_1.default.ok(!content.includes('spec-specific'), `${file} should not contain spec-specific content`);
|
|
260
|
+
node_assert_1.default.ok(!content.includes('design-specific'), `${file} should not contain design-specific content`);
|
|
261
|
+
}
|
|
262
|
+
});
|
|
263
|
+
});
|
|
264
|
+
});
|