fraim-framework 2.0.52 ā 2.0.55
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/registry/scripts/profile-server.js +2 -1
- package/dist/src/ai-manager/ai-manager.js +49 -1
- package/dist/src/ai-manager/phase-flow.js +68 -0
- package/dist/src/utils/digest-utils.js +18 -7
- package/dist/tests/test-debug-session.js +6 -2
- package/dist/tests/test-enhanced-session-init.js +6 -2
- package/dist/tests/test-mcp-lifecycle-methods.js +1 -2
- package/dist/tests/test-mcp-template-processing.js +6 -2
- package/dist/tests/test-modular-issue-tracking.js +6 -2
- package/dist/tests/test-node-compatibility.js +4 -2
- package/dist/tests/test-npm-install.js +4 -2
- package/dist/tests/test-productivity-integration.js +157 -0
- package/dist/tests/test-session-rehydration.js +1 -2
- package/dist/tests/test-telemetry.js +1 -2
- package/dist/tests/test-users-to-target-workflow.js +253 -0
- package/index.js +44 -55
- package/package.json +5 -5
- package/registry/agent-guardrails.md +62 -62
- package/registry/scripts/detect-tautological-tests.sh +38 -38
- package/registry/scripts/productivity/build-productivity-csv.mjs +242 -0
- package/registry/scripts/productivity/fetch-pr-details.mjs +144 -0
- package/registry/scripts/productivity/productivity-report.sh +147 -0
- package/registry/scripts/profile-server.ts +1 -1
- package/registry/scripts/validate-openapi-limits.ts +366 -366
- package/registry/scripts/validate-test-coverage.ts +280 -280
- package/registry/stubs/workflows/customer-development/ai-coach-phases/phase1-customer-profiling.md +11 -0
- package/registry/stubs/workflows/customer-development/ai-coach-phases/phase1-survey-scoping.md +11 -0
- package/registry/stubs/workflows/customer-development/ai-coach-phases/phase2-platform-discovery.md +11 -0
- package/registry/stubs/workflows/customer-development/ai-coach-phases/phase2-survey-build-linkedin.md +11 -0
- package/registry/stubs/workflows/customer-development/ai-coach-phases/phase3-prospect-qualification.md +11 -0
- package/registry/stubs/workflows/customer-development/ai-coach-phases/phase3-survey-build-reddit.md +11 -0
- package/registry/stubs/workflows/customer-development/ai-coach-phases/phase4-inventory-compilation.md +11 -0
- package/registry/stubs/workflows/customer-development/ai-coach-phases/phase4-survey-build-x.md +11 -0
- package/registry/stubs/workflows/customer-development/ai-coach-phases/phase5-survey-build-facebook.md +11 -0
- package/registry/stubs/workflows/customer-development/ai-coach-phases/phase6-survey-build-custom.md +11 -0
- package/registry/stubs/workflows/customer-development/ai-coach-phases/phase7-survey-dispatch.md +11 -0
- package/registry/stubs/workflows/customer-development/templates/customer-persona-template.md +11 -0
- package/registry/stubs/workflows/customer-development/templates/search-strategy-template.md +11 -0
- package/registry/stubs/workflows/customer-development/user-survey-dispatch.md +11 -0
- package/registry/stubs/workflows/customer-development/users-to-target.md +11 -0
- package/registry/stubs/workflows/productivity-report/productivity-report.md +11 -0
- package/bin/fraim.js +0 -8
- package/dist/registry/ai-manager-rules/design-phases/design.md +0 -108
- package/dist/registry/ai-manager-rules/design-phases/finalize.md +0 -60
- package/dist/registry/ai-manager-rules/design-phases/validate.md +0 -125
- package/dist/registry/ai-manager-rules/design.json +0 -97
- package/dist/registry/ai-manager-rules/implement-phases/code.md +0 -323
- package/dist/registry/ai-manager-rules/implement-phases/completeness-review.md +0 -94
- package/dist/registry/ai-manager-rules/implement-phases/finalize.md +0 -177
- package/dist/registry/ai-manager-rules/implement-phases/quality-review.md +0 -304
- package/dist/registry/ai-manager-rules/implement-phases/regression.md +0 -159
- package/dist/registry/ai-manager-rules/implement-phases/repro.md +0 -101
- package/dist/registry/ai-manager-rules/implement-phases/scoping.md +0 -93
- package/dist/registry/ai-manager-rules/implement-phases/smoke.md +0 -225
- package/dist/registry/ai-manager-rules/implement-phases/spike.md +0 -118
- package/dist/registry/ai-manager-rules/implement-phases/validate.md +0 -347
- package/dist/registry/ai-manager-rules/implement.json +0 -153
- package/dist/registry/ai-manager-rules/shared-phases/finalize.md +0 -169
- package/dist/registry/ai-manager-rules/spec-phases/finalize.md +0 -60
- package/dist/registry/ai-manager-rules/spec-phases/spec.md +0 -102
- package/dist/registry/ai-manager-rules/spec-phases/validate.md +0 -118
- package/dist/registry/ai-manager-rules/spec.json +0 -112
- package/dist/registry/ai-manager-rules/test.json +0 -98
- package/dist/registry/scripts/build-scripts-generator.js +0 -205
- package/dist/registry/scripts/fraim-config.js +0 -61
- package/dist/registry/scripts/generic-issues-api.js +0 -100
- package/dist/registry/scripts/openapi-generator.js +0 -664
- package/dist/registry/scripts/performance/profile-server.js +0 -390
- package/dist/src/ai-manager/evidence-validator.js +0 -309
- package/dist/src/fraim/issue-tracking/ado-provider.js +0 -304
- package/dist/src/fraim/issue-tracking/factory.js +0 -63
- package/dist/src/fraim/issue-tracking/github-provider.js +0 -200
- package/dist/src/fraim/issue-tracking/types.js +0 -7
- package/dist/src/fraim/issue-tracking-config.js +0 -83
- package/dist/src/static-website-middleware.js +0 -75
- package/dist/test-utils.js +0 -96
- package/dist/tests/esm-compat.js +0 -11
- package/dist/tests/test-ai-manager-phase-protocol.js +0 -147
- package/dist/tests/test-ai-manager.js +0 -118
- package/dist/tests/test-chalk-esm-issue.js +0 -159
- package/dist/tests/test-chalk-real-world.js +0 -265
- package/dist/tests/test-chalk-regression.js +0 -377
- package/dist/tests/test-chalk-resolution-issue.js +0 -304
- package/dist/tests/test-evidence-validation.js +0 -221
- package/dist/tests/test-first-run-interactive.js +0 -1
- package/dist/tests/test-fraim-install-chalk-issue.js +0 -254
- package/dist/tests/test-markdown-to-pdf.js +0 -454
- package/dist/tests/test-npm-resolution-diagnostic.js +0 -140
- package/dist/tests/test-pr-review-integration.js +0 -1
- package/dist/website/.nojekyll +0 -0
- package/dist/website/404.html +0 -101
- package/dist/website/CNAME +0 -1
- package/dist/website/README.md +0 -22
- package/dist/website/demo.html +0 -604
- package/dist/website/images/.gitkeep +0 -1
- package/dist/website/images/fraim-logo.png +0 -0
- package/dist/website/index.html +0 -290
- package/dist/website/pricing.html +0 -414
- package/dist/website/script.js +0 -55
- package/dist/website/styles.css +0 -2647
|
@@ -1,280 +1,280 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
/**
|
|
3
|
-
* Validation Plan Coverage Checker
|
|
4
|
-
*
|
|
5
|
-
* Extracts validation plan from spec/design documents and verifies evidence coverage.
|
|
6
|
-
* Used by Gate 9 in code-quality-check.sh
|
|
7
|
-
*
|
|
8
|
-
* Usage: npx tsx <this-script> <issue-number>
|
|
9
|
-
*/
|
|
10
|
-
|
|
11
|
-
import * as fs from 'fs';
|
|
12
|
-
import * as path from 'path';
|
|
13
|
-
|
|
14
|
-
interface ValidationScenario {
|
|
15
|
-
userScenario: string;
|
|
16
|
-
expectedOutcome: string;
|
|
17
|
-
validationMethod: string;
|
|
18
|
-
}
|
|
19
|
-
|
|
20
|
-
interface CoverageResult {
|
|
21
|
-
complete: boolean;
|
|
22
|
-
totalScenarios: number;
|
|
23
|
-
coveredScenarios: number;
|
|
24
|
-
missingScenarios: ValidationScenario[];
|
|
25
|
-
coverage: string; // "X/Y scenarios (Z%)"
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
/**
|
|
29
|
-
* Extract issue number from branch name
|
|
30
|
-
*/
|
|
31
|
-
function extractIssueNumberFromBranch(): number | null {
|
|
32
|
-
const branchName = process.env.GIT_BRANCH || '';
|
|
33
|
-
const match = branchName.match(/feature\/(\d+)/);
|
|
34
|
-
return match ? parseInt(match[1]) : null;
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
/**
|
|
38
|
-
* Find spec or design file for an issue
|
|
39
|
-
*/
|
|
40
|
-
function findSpecOrDesignFile(issueNumber: number): string | null {
|
|
41
|
-
const specDir = path.join(process.cwd(), 'docs', 'feature specs');
|
|
42
|
-
const rfcDir = path.join(process.cwd(), 'docs', 'rfcs');
|
|
43
|
-
|
|
44
|
-
// Try feature specs first
|
|
45
|
-
if (fs.existsSync(specDir)) {
|
|
46
|
-
const files = fs.readdirSync(specDir);
|
|
47
|
-
const specFile = files.find(f => f.startsWith(`${issueNumber}-`) && f.endsWith('.md'));
|
|
48
|
-
if (specFile) {
|
|
49
|
-
return path.join(specDir, specFile);
|
|
50
|
-
}
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
// Then try RFCs
|
|
54
|
-
if (fs.existsSync(rfcDir)) {
|
|
55
|
-
const files = fs.readdirSync(rfcDir);
|
|
56
|
-
const rfcFile = files.find(f => f.startsWith(`${issueNumber}-`) && f.endsWith('.md'));
|
|
57
|
-
if (rfcFile) {
|
|
58
|
-
return path.join(rfcDir, rfcFile);
|
|
59
|
-
}
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
return null;
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
/**
|
|
66
|
-
* Extract validation plan table from markdown file
|
|
67
|
-
*/
|
|
68
|
-
function extractValidationPlan(filePath: string): ValidationScenario[] {
|
|
69
|
-
const content = fs.readFileSync(filePath, 'utf-8');
|
|
70
|
-
const scenarios: ValidationScenario[] = [];
|
|
71
|
-
|
|
72
|
-
// Find "## Validation Plan" section (not "Validation Plan Coverage")
|
|
73
|
-
const validationPlanMatch = content.match(/##\s+Validation Plan\s*\n[\s\S]*?(?=\n##|$)/i);
|
|
74
|
-
if (!validationPlanMatch) {
|
|
75
|
-
return scenarios;
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
const validationSection = validationPlanMatch[0];
|
|
79
|
-
|
|
80
|
-
// Look for markdown table - find table header row with "User Scenario"
|
|
81
|
-
// Pattern: | User Scenario | Expected outcome | Validation method |
|
|
82
|
-
const tablePattern = /\|[\s\*]*User Scenario[\s\*]*\|[\s\*]*Expected outcome[\s\*]*\|[\s\*]*Validation method[\s\*]*\|[\s]*\n\|[\s:\-|]+\|[\s]*\n((?:\|.*?\|[\s]*\n)+)/i;
|
|
83
|
-
const tableMatch = validationSection.match(tablePattern);
|
|
84
|
-
|
|
85
|
-
if (!tableMatch) {
|
|
86
|
-
return scenarios;
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
const tableRows = tableMatch[1].trim().split('\n').filter(row => row.trim().startsWith('|'));
|
|
90
|
-
|
|
91
|
-
for (const row of tableRows) {
|
|
92
|
-
// Skip separator rows (all dashes/pipes)
|
|
93
|
-
if (row.match(/^\|[\s:\-|]+\|$/)) {
|
|
94
|
-
continue;
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
// Parse table row: | Scenario | Outcome | Method |
|
|
98
|
-
// Handle both formats: | **bold** | text | and | text | text |
|
|
99
|
-
const cells = row.split('|').map(c => c.trim()).filter(c => c && c !== '');
|
|
100
|
-
|
|
101
|
-
if (cells.length >= 2) {
|
|
102
|
-
// Remove markdown bold (**text**) from cells
|
|
103
|
-
const cleanCell = (cell: string): string => {
|
|
104
|
-
return cell
|
|
105
|
-
.replace(/^\*\*/, '')
|
|
106
|
-
.replace(/\*\*$/, '')
|
|
107
|
-
.replace(/^\*\*/g, '')
|
|
108
|
-
.replace(/\*\*/g, '')
|
|
109
|
-
.trim();
|
|
110
|
-
};
|
|
111
|
-
|
|
112
|
-
const userScenario = cleanCell(cells[0] || '');
|
|
113
|
-
const expectedOutcome = cleanCell(cells[1] || '');
|
|
114
|
-
const validationMethod = cleanCell(cells[2] || 'Manual validation');
|
|
115
|
-
|
|
116
|
-
// Skip header rows
|
|
117
|
-
const scenarioLower = userScenario.toLowerCase();
|
|
118
|
-
const outcomeLower = expectedOutcome.toLowerCase();
|
|
119
|
-
if (scenarioLower.includes('user scenario') ||
|
|
120
|
-
(scenarioLower.includes('scenario') && outcomeLower.includes('expected')) ||
|
|
121
|
-
scenarioLower === 'user scenario' ||
|
|
122
|
-
outcomeLower === 'expected outcome') {
|
|
123
|
-
continue;
|
|
124
|
-
}
|
|
125
|
-
|
|
126
|
-
// Skip empty rows or separator rows
|
|
127
|
-
if (!userScenario || userScenario === '-' || userScenario.length === 0) {
|
|
128
|
-
continue;
|
|
129
|
-
}
|
|
130
|
-
|
|
131
|
-
scenarios.push({
|
|
132
|
-
userScenario,
|
|
133
|
-
expectedOutcome,
|
|
134
|
-
validationMethod
|
|
135
|
-
});
|
|
136
|
-
}
|
|
137
|
-
}
|
|
138
|
-
|
|
139
|
-
return scenarios;
|
|
140
|
-
}
|
|
141
|
-
|
|
142
|
-
/**
|
|
143
|
-
* Check if evidence exists for a scenario
|
|
144
|
-
* Evidence can be in PR comments, test.log, or other sources
|
|
145
|
-
*/
|
|
146
|
-
function checkEvidenceForScenario(scenario: ValidationScenario, issueNumber: number): boolean {
|
|
147
|
-
// Extract key terms from scenario for searching
|
|
148
|
-
const scenarioKeywords = scenario.userScenario
|
|
149
|
-
.toLowerCase()
|
|
150
|
-
.replace(/\*\*/g, '')
|
|
151
|
-
.split(/\s+/)
|
|
152
|
-
.filter(w => w.length > 3 && !['agent', 'the', 'with', 'without'].includes(w));
|
|
153
|
-
|
|
154
|
-
const outcomeKeywords = scenario.expectedOutcome
|
|
155
|
-
.toLowerCase()
|
|
156
|
-
.replace(/\*\*/g, '')
|
|
157
|
-
.split(/\s+/)
|
|
158
|
-
.filter(w => w.length > 3);
|
|
159
|
-
|
|
160
|
-
// Method 1: Check test.log for scenario keywords
|
|
161
|
-
const testLogPath = path.join(process.cwd(), 'test.log');
|
|
162
|
-
if (fs.existsSync(testLogPath)) {
|
|
163
|
-
const testLogContent = fs.readFileSync(testLogPath, 'utf-8').toLowerCase();
|
|
164
|
-
// Check if multiple keywords from scenario appear
|
|
165
|
-
const matchingKeywords = scenarioKeywords.filter(kw => testLogContent.includes(kw));
|
|
166
|
-
if (matchingKeywords.length >= 2) {
|
|
167
|
-
return true;
|
|
168
|
-
}
|
|
169
|
-
}
|
|
170
|
-
|
|
171
|
-
// Method 2: Check for evidence file (created by agent with scenario validation)
|
|
172
|
-
const evidenceFiles = ['test-evidence.md', 'validation-evidence.md', 'implementation-evidence.md'];
|
|
173
|
-
for (const evidenceFile of evidenceFiles) {
|
|
174
|
-
const evidencePath = path.join(process.cwd(), evidenceFile);
|
|
175
|
-
if (fs.existsSync(evidencePath)) {
|
|
176
|
-
const evidenceContent = fs.readFileSync(evidencePath, 'utf-8').toLowerCase();
|
|
177
|
-
const matchingKeywords = scenarioKeywords.filter(kw => evidenceContent.includes(kw));
|
|
178
|
-
if (matchingKeywords.length >= 2) {
|
|
179
|
-
return true;
|
|
180
|
-
}
|
|
181
|
-
}
|
|
182
|
-
}
|
|
183
|
-
|
|
184
|
-
// Method 3: Check server.log for scenario-related activity
|
|
185
|
-
const serverLogPath = path.join(process.cwd(), 'server.log');
|
|
186
|
-
if (fs.existsSync(serverLogPath)) {
|
|
187
|
-
const serverLogContent = fs.readFileSync(serverLogPath, 'utf-8').toLowerCase();
|
|
188
|
-
// Look for validation-related keywords
|
|
189
|
-
if (scenarioKeywords.some(kw => serverLogContent.includes(kw)) ||
|
|
190
|
-
outcomeKeywords.some(kw => serverLogContent.includes('gate') && serverLogContent.includes(kw))) {
|
|
191
|
-
return true;
|
|
192
|
-
}
|
|
193
|
-
}
|
|
194
|
-
|
|
195
|
-
// Note: PR comment checking is done in the shell script (Gate 9)
|
|
196
|
-
// This TypeScript tool focuses on file-based evidence
|
|
197
|
-
|
|
198
|
-
return false; // Conservative: require evidence
|
|
199
|
-
}
|
|
200
|
-
|
|
201
|
-
/**
|
|
202
|
-
* Main function
|
|
203
|
-
*/
|
|
204
|
-
function main(): void {
|
|
205
|
-
const issueNumberArg = process.argv[2];
|
|
206
|
-
const issueNumber = issueNumberArg ? parseInt(issueNumberArg) : extractIssueNumberFromBranch();
|
|
207
|
-
|
|
208
|
-
if (!issueNumber) {
|
|
209
|
-
console.error('ā ERROR: Could not determine issue number');
|
|
210
|
-
console.error(' Usage: npx tsx validate-coverage.ts <issue-number>');
|
|
211
|
-
console.error(' Or set GIT_BRANCH environment variable with feature/N-* format');
|
|
212
|
-
process.exit(1);
|
|
213
|
-
}
|
|
214
|
-
|
|
215
|
-
const specFile = findSpecOrDesignFile(issueNumber);
|
|
216
|
-
|
|
217
|
-
if (!specFile) {
|
|
218
|
-
// No spec/design found - Gate 9 warns but doesn't block
|
|
219
|
-
console.log('ā¹ļø No spec/design found for issue #' + issueNumber);
|
|
220
|
-
console.log(' Skipping validation plan coverage check');
|
|
221
|
-
console.log(' Gate 9: Validation Plan Coverage - SKIPPED (no spec/design)');
|
|
222
|
-
process.exit(0);
|
|
223
|
-
}
|
|
224
|
-
|
|
225
|
-
console.log(`Found spec/design: ${specFile}`);
|
|
226
|
-
|
|
227
|
-
const scenarios = extractValidationPlan(specFile);
|
|
228
|
-
|
|
229
|
-
if (scenarios.length === 0) {
|
|
230
|
-
console.log('ā¹ļø No validation plan table found in spec/design');
|
|
231
|
-
console.log(' Gate 9: Validation Plan Coverage - SKIPPED (no validation plan)');
|
|
232
|
-
process.exit(0);
|
|
233
|
-
}
|
|
234
|
-
|
|
235
|
-
console.log(`Found ${scenarios.length} validation scenario(s)`);
|
|
236
|
-
|
|
237
|
-
// Check evidence for each scenario
|
|
238
|
-
const covered: ValidationScenario[] = [];
|
|
239
|
-
const missing: ValidationScenario[] = [];
|
|
240
|
-
|
|
241
|
-
for (const scenario of scenarios) {
|
|
242
|
-
if (checkEvidenceForScenario(scenario, issueNumber)) {
|
|
243
|
-
covered.push(scenario);
|
|
244
|
-
} else {
|
|
245
|
-
missing.push(scenario);
|
|
246
|
-
}
|
|
247
|
-
}
|
|
248
|
-
|
|
249
|
-
const coverage: CoverageResult = {
|
|
250
|
-
complete: missing.length === 0,
|
|
251
|
-
totalScenarios: scenarios.length,
|
|
252
|
-
coveredScenarios: covered.length,
|
|
253
|
-
missingScenarios: missing,
|
|
254
|
-
coverage: `${covered.length}/${scenarios.length} scenarios (${Math.round(covered.length / scenarios.length * 100)}%)`
|
|
255
|
-
};
|
|
256
|
-
|
|
257
|
-
// Output results
|
|
258
|
-
console.log('\nš Validation Plan Coverage:');
|
|
259
|
-
console.log(` ${coverage.coverage}`);
|
|
260
|
-
|
|
261
|
-
if (coverage.complete) {
|
|
262
|
-
console.log('ā
PASSED: All validation scenarios have evidence');
|
|
263
|
-
console.log(' Gate 9: Validation Plan Coverage - PASSED');
|
|
264
|
-
process.exit(0);
|
|
265
|
-
} else {
|
|
266
|
-
console.log('ā FAILED: Missing evidence for validation scenarios:');
|
|
267
|
-
for (const scenario of missing) {
|
|
268
|
-
console.log(` - "${scenario.userScenario}"`);
|
|
269
|
-
console.log(` Expected: ${scenario.expectedOutcome}`);
|
|
270
|
-
console.log(` Method: ${scenario.validationMethod}`);
|
|
271
|
-
}
|
|
272
|
-
console.log('\n Required: Provide evidence for all validation scenarios before marking PR ready');
|
|
273
|
-
console.log(' Evidence can be: PR comment, test output, manual verification log');
|
|
274
|
-
console.log(' Gate 9: Validation Plan Coverage - FAILED');
|
|
275
|
-
process.exit(1);
|
|
276
|
-
}
|
|
277
|
-
}
|
|
278
|
-
|
|
279
|
-
// Run main function
|
|
280
|
-
main();
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* Validation Plan Coverage Checker
|
|
4
|
+
*
|
|
5
|
+
* Extracts validation plan from spec/design documents and verifies evidence coverage.
|
|
6
|
+
* Used by Gate 9 in code-quality-check.sh
|
|
7
|
+
*
|
|
8
|
+
* Usage: npx tsx <this-script> <issue-number>
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import * as fs from 'fs';
|
|
12
|
+
import * as path from 'path';
|
|
13
|
+
|
|
14
|
+
interface ValidationScenario {
|
|
15
|
+
userScenario: string;
|
|
16
|
+
expectedOutcome: string;
|
|
17
|
+
validationMethod: string;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
interface CoverageResult {
|
|
21
|
+
complete: boolean;
|
|
22
|
+
totalScenarios: number;
|
|
23
|
+
coveredScenarios: number;
|
|
24
|
+
missingScenarios: ValidationScenario[];
|
|
25
|
+
coverage: string; // "X/Y scenarios (Z%)"
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Extract issue number from branch name
|
|
30
|
+
*/
|
|
31
|
+
function extractIssueNumberFromBranch(): number | null {
|
|
32
|
+
const branchName = process.env.GIT_BRANCH || '';
|
|
33
|
+
const match = branchName.match(/feature\/(\d+)/);
|
|
34
|
+
return match ? parseInt(match[1]) : null;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Find spec or design file for an issue
|
|
39
|
+
*/
|
|
40
|
+
function findSpecOrDesignFile(issueNumber: number): string | null {
|
|
41
|
+
const specDir = path.join(process.cwd(), 'docs', 'feature specs');
|
|
42
|
+
const rfcDir = path.join(process.cwd(), 'docs', 'rfcs');
|
|
43
|
+
|
|
44
|
+
// Try feature specs first
|
|
45
|
+
if (fs.existsSync(specDir)) {
|
|
46
|
+
const files = fs.readdirSync(specDir);
|
|
47
|
+
const specFile = files.find(f => f.startsWith(`${issueNumber}-`) && f.endsWith('.md'));
|
|
48
|
+
if (specFile) {
|
|
49
|
+
return path.join(specDir, specFile);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
// Then try RFCs
|
|
54
|
+
if (fs.existsSync(rfcDir)) {
|
|
55
|
+
const files = fs.readdirSync(rfcDir);
|
|
56
|
+
const rfcFile = files.find(f => f.startsWith(`${issueNumber}-`) && f.endsWith('.md'));
|
|
57
|
+
if (rfcFile) {
|
|
58
|
+
return path.join(rfcDir, rfcFile);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
return null;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Extract validation plan table from markdown file
|
|
67
|
+
*/
|
|
68
|
+
function extractValidationPlan(filePath: string): ValidationScenario[] {
|
|
69
|
+
const content = fs.readFileSync(filePath, 'utf-8');
|
|
70
|
+
const scenarios: ValidationScenario[] = [];
|
|
71
|
+
|
|
72
|
+
// Find "## Validation Plan" section (not "Validation Plan Coverage")
|
|
73
|
+
const validationPlanMatch = content.match(/##\s+Validation Plan\s*\n[\s\S]*?(?=\n##|$)/i);
|
|
74
|
+
if (!validationPlanMatch) {
|
|
75
|
+
return scenarios;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
const validationSection = validationPlanMatch[0];
|
|
79
|
+
|
|
80
|
+
// Look for markdown table - find table header row with "User Scenario"
|
|
81
|
+
// Pattern: | User Scenario | Expected outcome | Validation method |
|
|
82
|
+
const tablePattern = /\|[\s\*]*User Scenario[\s\*]*\|[\s\*]*Expected outcome[\s\*]*\|[\s\*]*Validation method[\s\*]*\|[\s]*\n\|[\s:\-|]+\|[\s]*\n((?:\|.*?\|[\s]*\n)+)/i;
|
|
83
|
+
const tableMatch = validationSection.match(tablePattern);
|
|
84
|
+
|
|
85
|
+
if (!tableMatch) {
|
|
86
|
+
return scenarios;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
const tableRows = tableMatch[1].trim().split('\n').filter(row => row.trim().startsWith('|'));
|
|
90
|
+
|
|
91
|
+
for (const row of tableRows) {
|
|
92
|
+
// Skip separator rows (all dashes/pipes)
|
|
93
|
+
if (row.match(/^\|[\s:\-|]+\|$/)) {
|
|
94
|
+
continue;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
// Parse table row: | Scenario | Outcome | Method |
|
|
98
|
+
// Handle both formats: | **bold** | text | and | text | text |
|
|
99
|
+
const cells = row.split('|').map(c => c.trim()).filter(c => c && c !== '');
|
|
100
|
+
|
|
101
|
+
if (cells.length >= 2) {
|
|
102
|
+
// Remove markdown bold (**text**) from cells
|
|
103
|
+
const cleanCell = (cell: string): string => {
|
|
104
|
+
return cell
|
|
105
|
+
.replace(/^\*\*/, '')
|
|
106
|
+
.replace(/\*\*$/, '')
|
|
107
|
+
.replace(/^\*\*/g, '')
|
|
108
|
+
.replace(/\*\*/g, '')
|
|
109
|
+
.trim();
|
|
110
|
+
};
|
|
111
|
+
|
|
112
|
+
const userScenario = cleanCell(cells[0] || '');
|
|
113
|
+
const expectedOutcome = cleanCell(cells[1] || '');
|
|
114
|
+
const validationMethod = cleanCell(cells[2] || 'Manual validation');
|
|
115
|
+
|
|
116
|
+
// Skip header rows
|
|
117
|
+
const scenarioLower = userScenario.toLowerCase();
|
|
118
|
+
const outcomeLower = expectedOutcome.toLowerCase();
|
|
119
|
+
if (scenarioLower.includes('user scenario') ||
|
|
120
|
+
(scenarioLower.includes('scenario') && outcomeLower.includes('expected')) ||
|
|
121
|
+
scenarioLower === 'user scenario' ||
|
|
122
|
+
outcomeLower === 'expected outcome') {
|
|
123
|
+
continue;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
// Skip empty rows or separator rows
|
|
127
|
+
if (!userScenario || userScenario === '-' || userScenario.length === 0) {
|
|
128
|
+
continue;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
scenarios.push({
|
|
132
|
+
userScenario,
|
|
133
|
+
expectedOutcome,
|
|
134
|
+
validationMethod
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
return scenarios;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* Check if evidence exists for a scenario
|
|
144
|
+
* Evidence can be in PR comments, test.log, or other sources
|
|
145
|
+
*/
|
|
146
|
+
function checkEvidenceForScenario(scenario: ValidationScenario, issueNumber: number): boolean {
|
|
147
|
+
// Extract key terms from scenario for searching
|
|
148
|
+
const scenarioKeywords = scenario.userScenario
|
|
149
|
+
.toLowerCase()
|
|
150
|
+
.replace(/\*\*/g, '')
|
|
151
|
+
.split(/\s+/)
|
|
152
|
+
.filter(w => w.length > 3 && !['agent', 'the', 'with', 'without'].includes(w));
|
|
153
|
+
|
|
154
|
+
const outcomeKeywords = scenario.expectedOutcome
|
|
155
|
+
.toLowerCase()
|
|
156
|
+
.replace(/\*\*/g, '')
|
|
157
|
+
.split(/\s+/)
|
|
158
|
+
.filter(w => w.length > 3);
|
|
159
|
+
|
|
160
|
+
// Method 1: Check test.log for scenario keywords
|
|
161
|
+
const testLogPath = path.join(process.cwd(), 'test.log');
|
|
162
|
+
if (fs.existsSync(testLogPath)) {
|
|
163
|
+
const testLogContent = fs.readFileSync(testLogPath, 'utf-8').toLowerCase();
|
|
164
|
+
// Check if multiple keywords from scenario appear
|
|
165
|
+
const matchingKeywords = scenarioKeywords.filter(kw => testLogContent.includes(kw));
|
|
166
|
+
if (matchingKeywords.length >= 2) {
|
|
167
|
+
return true;
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
// Method 2: Check for evidence file (created by agent with scenario validation)
|
|
172
|
+
const evidenceFiles = ['test-evidence.md', 'validation-evidence.md', 'implementation-evidence.md'];
|
|
173
|
+
for (const evidenceFile of evidenceFiles) {
|
|
174
|
+
const evidencePath = path.join(process.cwd(), evidenceFile);
|
|
175
|
+
if (fs.existsSync(evidencePath)) {
|
|
176
|
+
const evidenceContent = fs.readFileSync(evidencePath, 'utf-8').toLowerCase();
|
|
177
|
+
const matchingKeywords = scenarioKeywords.filter(kw => evidenceContent.includes(kw));
|
|
178
|
+
if (matchingKeywords.length >= 2) {
|
|
179
|
+
return true;
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
// Method 3: Check server.log for scenario-related activity
|
|
185
|
+
const serverLogPath = path.join(process.cwd(), 'server.log');
|
|
186
|
+
if (fs.existsSync(serverLogPath)) {
|
|
187
|
+
const serverLogContent = fs.readFileSync(serverLogPath, 'utf-8').toLowerCase();
|
|
188
|
+
// Look for validation-related keywords
|
|
189
|
+
if (scenarioKeywords.some(kw => serverLogContent.includes(kw)) ||
|
|
190
|
+
outcomeKeywords.some(kw => serverLogContent.includes('gate') && serverLogContent.includes(kw))) {
|
|
191
|
+
return true;
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
// Note: PR comment checking is done in the shell script (Gate 9)
|
|
196
|
+
// This TypeScript tool focuses on file-based evidence
|
|
197
|
+
|
|
198
|
+
return false; // Conservative: require evidence
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/**
|
|
202
|
+
* Main function
|
|
203
|
+
*/
|
|
204
|
+
function main(): void {
|
|
205
|
+
const issueNumberArg = process.argv[2];
|
|
206
|
+
const issueNumber = issueNumberArg ? parseInt(issueNumberArg) : extractIssueNumberFromBranch();
|
|
207
|
+
|
|
208
|
+
if (!issueNumber) {
|
|
209
|
+
console.error('ā ERROR: Could not determine issue number');
|
|
210
|
+
console.error(' Usage: npx tsx validate-coverage.ts <issue-number>');
|
|
211
|
+
console.error(' Or set GIT_BRANCH environment variable with feature/N-* format');
|
|
212
|
+
process.exit(1);
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
const specFile = findSpecOrDesignFile(issueNumber);
|
|
216
|
+
|
|
217
|
+
if (!specFile) {
|
|
218
|
+
// No spec/design found - Gate 9 warns but doesn't block
|
|
219
|
+
console.log('ā¹ļø No spec/design found for issue #' + issueNumber);
|
|
220
|
+
console.log(' Skipping validation plan coverage check');
|
|
221
|
+
console.log(' Gate 9: Validation Plan Coverage - SKIPPED (no spec/design)');
|
|
222
|
+
process.exit(0);
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
console.log(`Found spec/design: ${specFile}`);
|
|
226
|
+
|
|
227
|
+
const scenarios = extractValidationPlan(specFile);
|
|
228
|
+
|
|
229
|
+
if (scenarios.length === 0) {
|
|
230
|
+
console.log('ā¹ļø No validation plan table found in spec/design');
|
|
231
|
+
console.log(' Gate 9: Validation Plan Coverage - SKIPPED (no validation plan)');
|
|
232
|
+
process.exit(0);
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
console.log(`Found ${scenarios.length} validation scenario(s)`);
|
|
236
|
+
|
|
237
|
+
// Check evidence for each scenario
|
|
238
|
+
const covered: ValidationScenario[] = [];
|
|
239
|
+
const missing: ValidationScenario[] = [];
|
|
240
|
+
|
|
241
|
+
for (const scenario of scenarios) {
|
|
242
|
+
if (checkEvidenceForScenario(scenario, issueNumber)) {
|
|
243
|
+
covered.push(scenario);
|
|
244
|
+
} else {
|
|
245
|
+
missing.push(scenario);
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
const coverage: CoverageResult = {
|
|
250
|
+
complete: missing.length === 0,
|
|
251
|
+
totalScenarios: scenarios.length,
|
|
252
|
+
coveredScenarios: covered.length,
|
|
253
|
+
missingScenarios: missing,
|
|
254
|
+
coverage: `${covered.length}/${scenarios.length} scenarios (${Math.round(covered.length / scenarios.length * 100)}%)`
|
|
255
|
+
};
|
|
256
|
+
|
|
257
|
+
// Output results
|
|
258
|
+
console.log('\nš Validation Plan Coverage:');
|
|
259
|
+
console.log(` ${coverage.coverage}`);
|
|
260
|
+
|
|
261
|
+
if (coverage.complete) {
|
|
262
|
+
console.log('ā
PASSED: All validation scenarios have evidence');
|
|
263
|
+
console.log(' Gate 9: Validation Plan Coverage - PASSED');
|
|
264
|
+
process.exit(0);
|
|
265
|
+
} else {
|
|
266
|
+
console.log('ā FAILED: Missing evidence for validation scenarios:');
|
|
267
|
+
for (const scenario of missing) {
|
|
268
|
+
console.log(` - "${scenario.userScenario}"`);
|
|
269
|
+
console.log(` Expected: ${scenario.expectedOutcome}`);
|
|
270
|
+
console.log(` Method: ${scenario.validationMethod}`);
|
|
271
|
+
}
|
|
272
|
+
console.log('\n Required: Provide evidence for all validation scenarios before marking PR ready');
|
|
273
|
+
console.log(' Evidence can be: PR comment, test output, manual verification log');
|
|
274
|
+
console.log(' Gate 9: Validation Plan Coverage - FAILED');
|
|
275
|
+
process.exit(1);
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
// Run main function
|
|
280
|
+
main();
|
package/registry/stubs/workflows/customer-development/ai-coach-phases/phase1-customer-profiling.md
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# FRAIM Workflow: phase1-customer-profiling
|
|
2
|
+
|
|
3
|
+
> [!IMPORTANT]
|
|
4
|
+
> This is a **FRAIM-managed workflow stub**.
|
|
5
|
+
> To load the full context (rules, templates, and execution steps), ask your AI agent to:
|
|
6
|
+
> `@fraim get_fraim_workflow("phase1-customer-profiling")`
|
|
7
|
+
>
|
|
8
|
+
> DO NOT EXECUTE.
|
|
9
|
+
|
|
10
|
+
## Intent
|
|
11
|
+
No intent defined.
|
package/registry/stubs/workflows/customer-development/ai-coach-phases/phase1-survey-scoping.md
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# FRAIM Workflow: phase1-survey-scoping
|
|
2
|
+
|
|
3
|
+
> [!IMPORTANT]
|
|
4
|
+
> This is a **FRAIM-managed workflow stub**.
|
|
5
|
+
> To load the full context (rules, templates, and execution steps), ask your AI agent to:
|
|
6
|
+
> `@fraim get_fraim_workflow("phase1-survey-scoping")`
|
|
7
|
+
>
|
|
8
|
+
> DO NOT EXECUTE.
|
|
9
|
+
|
|
10
|
+
## Intent
|
|
11
|
+
No intent defined.
|
package/registry/stubs/workflows/customer-development/ai-coach-phases/phase2-platform-discovery.md
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# FRAIM Workflow: phase2-platform-discovery
|
|
2
|
+
|
|
3
|
+
> [!IMPORTANT]
|
|
4
|
+
> This is a **FRAIM-managed workflow stub**.
|
|
5
|
+
> To load the full context (rules, templates, and execution steps), ask your AI agent to:
|
|
6
|
+
> `@fraim get_fraim_workflow("phase2-platform-discovery")`
|
|
7
|
+
>
|
|
8
|
+
> DO NOT EXECUTE.
|
|
9
|
+
|
|
10
|
+
## Intent
|
|
11
|
+
No intent defined.
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# FRAIM Workflow: phase2-survey-build-linkedin
|
|
2
|
+
|
|
3
|
+
> [!IMPORTANT]
|
|
4
|
+
> This is a **FRAIM-managed workflow stub**.
|
|
5
|
+
> To load the full context (rules, templates, and execution steps), ask your AI agent to:
|
|
6
|
+
> `@fraim get_fraim_workflow("phase2-survey-build-linkedin")`
|
|
7
|
+
>
|
|
8
|
+
> DO NOT EXECUTE.
|
|
9
|
+
|
|
10
|
+
## Intent
|
|
11
|
+
No intent defined.
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# FRAIM Workflow: phase3-prospect-qualification
|
|
2
|
+
|
|
3
|
+
> [!IMPORTANT]
|
|
4
|
+
> This is a **FRAIM-managed workflow stub**.
|
|
5
|
+
> To load the full context (rules, templates, and execution steps), ask your AI agent to:
|
|
6
|
+
> `@fraim get_fraim_workflow("phase3-prospect-qualification")`
|
|
7
|
+
>
|
|
8
|
+
> DO NOT EXECUTE.
|
|
9
|
+
|
|
10
|
+
## Intent
|
|
11
|
+
No intent defined.
|
package/registry/stubs/workflows/customer-development/ai-coach-phases/phase3-survey-build-reddit.md
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# FRAIM Workflow: phase3-survey-build-reddit
|
|
2
|
+
|
|
3
|
+
> [!IMPORTANT]
|
|
4
|
+
> This is a **FRAIM-managed workflow stub**.
|
|
5
|
+
> To load the full context (rules, templates, and execution steps), ask your AI agent to:
|
|
6
|
+
> `@fraim get_fraim_workflow("phase3-survey-build-reddit")`
|
|
7
|
+
>
|
|
8
|
+
> DO NOT EXECUTE.
|
|
9
|
+
|
|
10
|
+
## Intent
|
|
11
|
+
No intent defined.
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# FRAIM Workflow: phase4-inventory-compilation
|
|
2
|
+
|
|
3
|
+
> [!IMPORTANT]
|
|
4
|
+
> This is a **FRAIM-managed workflow stub**.
|
|
5
|
+
> To load the full context (rules, templates, and execution steps), ask your AI agent to:
|
|
6
|
+
> `@fraim get_fraim_workflow("phase4-inventory-compilation")`
|
|
7
|
+
>
|
|
8
|
+
> DO NOT EXECUTE.
|
|
9
|
+
|
|
10
|
+
## Intent
|
|
11
|
+
No intent defined.
|
package/registry/stubs/workflows/customer-development/ai-coach-phases/phase4-survey-build-x.md
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# FRAIM Workflow: phase4-survey-build-x
|
|
2
|
+
|
|
3
|
+
> [!IMPORTANT]
|
|
4
|
+
> This is a **FRAIM-managed workflow stub**.
|
|
5
|
+
> To load the full context (rules, templates, and execution steps), ask your AI agent to:
|
|
6
|
+
> `@fraim get_fraim_workflow("phase4-survey-build-x")`
|
|
7
|
+
>
|
|
8
|
+
> DO NOT EXECUTE.
|
|
9
|
+
|
|
10
|
+
## Intent
|
|
11
|
+
No intent defined.
|