@aiready/testability 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +24 -0
- package/.turbo/turbo-test.log +16 -0
- package/dist/chunk-CYZ7DTWN.mjs +338 -0
- package/dist/cli.d.mts +1 -0
- package/dist/cli.d.ts +1 -0
- package/dist/cli.js +498 -0
- package/dist/cli.mjs +143 -0
- package/dist/index.d.mts +74 -0
- package/dist/index.d.ts +74 -0
- package/dist/index.js +365 -0
- package/dist/index.mjs +8 -0
- package/package.json +62 -0
- package/src/__tests__/analyzer.test.ts +84 -0
- package/src/analyzer.ts +381 -0
- package/src/cli.ts +160 -0
- package/src/index.ts +7 -0
- package/src/scoring.ts +59 -0
- package/src/types.ts +55 -0
- package/tsconfig.json +9 -0
package/src/analyzer.ts
ADDED
|
@@ -0,0 +1,381 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Testability analyzer.
|
|
3
|
+
*
|
|
4
|
+
* Walks the codebase and measures 5 structural dimensions that determine
|
|
5
|
+
* whether AI-generated changes can be safely verified:
|
|
6
|
+
* 1. Test file coverage ratio
|
|
7
|
+
* 2. Pure function prevalence
|
|
8
|
+
* 3. Dependency injection patterns
|
|
9
|
+
* 4. Interface focus (bloated interface detection)
|
|
10
|
+
* 5. Observability (return values vs. external state mutations)
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { readdirSync, statSync, readFileSync, existsSync } from 'fs';
|
|
14
|
+
import { join, extname, basename } from 'path';
|
|
15
|
+
import { parse } from '@typescript-eslint/typescript-estree';
|
|
16
|
+
import type { TSESTree } from '@typescript-eslint/types';
|
|
17
|
+
import { calculateTestabilityIndex } from '@aiready/core';
|
|
18
|
+
import type { TestabilityOptions, TestabilityIssue, TestabilityReport } from './types';
|
|
19
|
+
|
|
20
|
+
// ---------------------------------------------------------------------------
|
|
21
|
+
// File classification
|
|
22
|
+
// ---------------------------------------------------------------------------
|
|
23
|
+
|
|
24
|
+
const SRC_EXTENSIONS = new Set(['.ts', '.tsx', '.js', '.jsx']);
|
|
25
|
+
const DEFAULT_EXCLUDES = ['node_modules', 'dist', '.git', 'coverage', '.turbo', 'build'];
|
|
26
|
+
const TEST_PATTERNS = [
|
|
27
|
+
/\.(test|spec)\.(ts|tsx|js|jsx)$/,
|
|
28
|
+
/__tests__\//,
|
|
29
|
+
/\/tests?\//,
|
|
30
|
+
/\/e2e\//,
|
|
31
|
+
/\/fixtures\//,
|
|
32
|
+
];
|
|
33
|
+
|
|
34
|
+
function isTestFile(filePath: string, extra?: string[]): boolean {
|
|
35
|
+
if (TEST_PATTERNS.some(p => p.test(filePath))) return true;
|
|
36
|
+
if (extra) return extra.some(p => filePath.includes(p));
|
|
37
|
+
return false;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function isSourceFile(filePath: string): boolean {
|
|
41
|
+
return SRC_EXTENSIONS.has(extname(filePath));
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function collectFiles(
|
|
45
|
+
dir: string,
|
|
46
|
+
options: TestabilityOptions,
|
|
47
|
+
depth = 0,
|
|
48
|
+
): string[] {
|
|
49
|
+
if (depth > (options.maxDepth ?? 20)) return [];
|
|
50
|
+
const excludes = [...DEFAULT_EXCLUDES, ...(options.exclude ?? [])];
|
|
51
|
+
const files: string[] = [];
|
|
52
|
+
let entries: string[];
|
|
53
|
+
try {
|
|
54
|
+
entries = readdirSync(dir);
|
|
55
|
+
} catch {
|
|
56
|
+
return files;
|
|
57
|
+
}
|
|
58
|
+
for (const entry of entries) {
|
|
59
|
+
if (excludes.some(ex => entry === ex || entry.includes(ex))) continue;
|
|
60
|
+
const full = join(dir, entry);
|
|
61
|
+
let stat;
|
|
62
|
+
try { stat = statSync(full); } catch { continue; }
|
|
63
|
+
if (stat.isDirectory()) {
|
|
64
|
+
files.push(...collectFiles(full, options, depth + 1));
|
|
65
|
+
} else if (stat.isFile() && isSourceFile(full)) {
|
|
66
|
+
if (!options.include || options.include.some(p => full.includes(p))) {
|
|
67
|
+
files.push(full);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
return files;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
// ---------------------------------------------------------------------------
|
|
75
|
+
// Per-file analysis
|
|
76
|
+
// ---------------------------------------------------------------------------
|
|
77
|
+
|
|
78
|
+
interface FileAnalysis {
|
|
79
|
+
pureFunctions: number;
|
|
80
|
+
totalFunctions: number;
|
|
81
|
+
injectionPatterns: number;
|
|
82
|
+
totalClasses: number;
|
|
83
|
+
bloatedInterfaces: number;
|
|
84
|
+
totalInterfaces: number;
|
|
85
|
+
externalStateMutations: number;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
function countMethodsInInterface(node: TSESTree.TSInterfaceDeclaration | TSESTree.TSTypeAliasDeclaration): number {
|
|
89
|
+
// Count method signatures
|
|
90
|
+
if (node.type === 'TSInterfaceDeclaration') {
|
|
91
|
+
return node.body.body.filter(m =>
|
|
92
|
+
m.type === 'TSMethodSignature' || m.type === 'TSPropertySignature'
|
|
93
|
+
).length;
|
|
94
|
+
}
|
|
95
|
+
if (node.type === 'TSTypeAliasDeclaration' && node.typeAnnotation.type === 'TSTypeLiteral') {
|
|
96
|
+
return node.typeAnnotation.members.length;
|
|
97
|
+
}
|
|
98
|
+
return 0;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
function hasDependencyInjection(node: TSESTree.ClassDeclaration | TSESTree.ClassExpression): boolean {
|
|
102
|
+
// Look for a constructor with typed parameters (the most common DI pattern)
|
|
103
|
+
for (const member of node.body.body) {
|
|
104
|
+
if (
|
|
105
|
+
member.type === 'MethodDefinition' &&
|
|
106
|
+
member.key.type === 'Identifier' &&
|
|
107
|
+
member.key.name === 'constructor'
|
|
108
|
+
) {
|
|
109
|
+
const fn = member.value;
|
|
110
|
+
if (fn.params && fn.params.length > 0) {
|
|
111
|
+
// If constructor takes parameters that are typed class/interface references, that's DI
|
|
112
|
+
const typedParams = fn.params.filter(p => {
|
|
113
|
+
const param = p as any;
|
|
114
|
+
return param.typeAnnotation != null ||
|
|
115
|
+
param.parameter?.typeAnnotation != null;
|
|
116
|
+
});
|
|
117
|
+
if (typedParams.length > 0) return true;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
return false;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
function isPureFunction(
|
|
125
|
+
fn: TSESTree.FunctionDeclaration | TSESTree.FunctionExpression | TSESTree.ArrowFunctionExpression,
|
|
126
|
+
): boolean {
|
|
127
|
+
let hasReturn = false;
|
|
128
|
+
let hasSideEffect = false;
|
|
129
|
+
|
|
130
|
+
function walk(node: TSESTree.Node) {
|
|
131
|
+
if (node.type === 'ReturnStatement' && node.argument) hasReturn = true;
|
|
132
|
+
if (
|
|
133
|
+
node.type === 'AssignmentExpression' &&
|
|
134
|
+
node.left.type === 'MemberExpression'
|
|
135
|
+
) hasSideEffect = true;
|
|
136
|
+
// Calls to console, process, global objects
|
|
137
|
+
if (
|
|
138
|
+
node.type === 'CallExpression' &&
|
|
139
|
+
node.callee.type === 'MemberExpression' &&
|
|
140
|
+
node.callee.object.type === 'Identifier' &&
|
|
141
|
+
['console', 'process', 'window', 'document', 'fs'].includes(node.callee.object.name)
|
|
142
|
+
) hasSideEffect = true;
|
|
143
|
+
|
|
144
|
+
// Recurse
|
|
145
|
+
for (const key of Object.keys(node)) {
|
|
146
|
+
if (key === 'parent') continue;
|
|
147
|
+
const child = (node as any)[key];
|
|
148
|
+
if (child && typeof child === 'object') {
|
|
149
|
+
if (Array.isArray(child)) {
|
|
150
|
+
child.forEach(c => c?.type && walk(c));
|
|
151
|
+
} else if (child.type) {
|
|
152
|
+
walk(child);
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
if (fn.body?.type === 'BlockStatement') {
|
|
159
|
+
fn.body.body.forEach(s => walk(s));
|
|
160
|
+
} else if (fn.body) {
|
|
161
|
+
hasReturn = true; // arrow expression body
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
return hasReturn && !hasSideEffect;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
function hasExternalStateMutation(
|
|
168
|
+
fn: TSESTree.FunctionDeclaration | TSESTree.FunctionExpression | TSESTree.ArrowFunctionExpression,
|
|
169
|
+
): boolean {
|
|
170
|
+
let found = false;
|
|
171
|
+
function walk(node: TSESTree.Node) {
|
|
172
|
+
if (found) return;
|
|
173
|
+
if (
|
|
174
|
+
node.type === 'AssignmentExpression' &&
|
|
175
|
+
node.left.type === 'MemberExpression'
|
|
176
|
+
) found = true;
|
|
177
|
+
for (const key of Object.keys(node)) {
|
|
178
|
+
if (key === 'parent') continue;
|
|
179
|
+
const child = (node as any)[key];
|
|
180
|
+
if (child && typeof child === 'object') {
|
|
181
|
+
if (Array.isArray(child)) child.forEach(c => c?.type && walk(c));
|
|
182
|
+
else if (child.type) walk(child);
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
if (fn.body?.type === 'BlockStatement') fn.body.body.forEach(s => walk(s));
|
|
187
|
+
return found;
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
function analyzeFileTestability(filePath: string): FileAnalysis {
|
|
191
|
+
const result: FileAnalysis = {
|
|
192
|
+
pureFunctions: 0,
|
|
193
|
+
totalFunctions: 0,
|
|
194
|
+
injectionPatterns: 0,
|
|
195
|
+
totalClasses: 0,
|
|
196
|
+
bloatedInterfaces: 0,
|
|
197
|
+
totalInterfaces: 0,
|
|
198
|
+
externalStateMutations: 0,
|
|
199
|
+
};
|
|
200
|
+
|
|
201
|
+
let code: string;
|
|
202
|
+
try { code = readFileSync(filePath, 'utf-8'); } catch { return result; }
|
|
203
|
+
|
|
204
|
+
let ast: TSESTree.Program;
|
|
205
|
+
try {
|
|
206
|
+
ast = parse(code, {
|
|
207
|
+
jsx: filePath.endsWith('.tsx') || filePath.endsWith('.jsx'),
|
|
208
|
+
range: false,
|
|
209
|
+
loc: false,
|
|
210
|
+
});
|
|
211
|
+
} catch { return result; }
|
|
212
|
+
|
|
213
|
+
function visit(node: TSESTree.Node) {
|
|
214
|
+
if (
|
|
215
|
+
node.type === 'FunctionDeclaration' ||
|
|
216
|
+
node.type === 'FunctionExpression' ||
|
|
217
|
+
node.type === 'ArrowFunctionExpression'
|
|
218
|
+
) {
|
|
219
|
+
result.totalFunctions++;
|
|
220
|
+
if (isPureFunction(node)) result.pureFunctions++;
|
|
221
|
+
if (hasExternalStateMutation(node)) result.externalStateMutations++;
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
if (node.type === 'ClassDeclaration' || node.type === 'ClassExpression') {
|
|
225
|
+
result.totalClasses++;
|
|
226
|
+
if (hasDependencyInjection(node)) result.injectionPatterns++;
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
if (node.type === 'TSInterfaceDeclaration' || node.type === 'TSTypeAliasDeclaration') {
|
|
230
|
+
result.totalInterfaces++;
|
|
231
|
+
const methodCount = countMethodsInInterface(node as any);
|
|
232
|
+
if (methodCount > 10) result.bloatedInterfaces++;
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
// Recurse
|
|
236
|
+
for (const key of Object.keys(node)) {
|
|
237
|
+
if (key === 'parent') continue;
|
|
238
|
+
const child = (node as any)[key];
|
|
239
|
+
if (child && typeof child === 'object') {
|
|
240
|
+
if (Array.isArray(child)) child.forEach(c => c?.type && visit(c));
|
|
241
|
+
else if (child.type) visit(child);
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
ast.body.forEach(visit);
|
|
247
|
+
return result;
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
// ---------------------------------------------------------------------------
|
|
251
|
+
// Test framework detection
|
|
252
|
+
// ---------------------------------------------------------------------------
|
|
253
|
+
|
|
254
|
+
function detectTestFramework(rootDir: string): boolean {
|
|
255
|
+
const pkgPath = join(rootDir, 'package.json');
|
|
256
|
+
if (!existsSync(pkgPath)) return false;
|
|
257
|
+
try {
|
|
258
|
+
const pkg = JSON.parse(readFileSync(pkgPath, 'utf-8'));
|
|
259
|
+
const allDeps = {
|
|
260
|
+
...(pkg.dependencies ?? {}),
|
|
261
|
+
...(pkg.devDependencies ?? {}),
|
|
262
|
+
};
|
|
263
|
+
const testFrameworks = ['jest', 'vitest', 'mocha', 'jasmine', 'ava', 'tap', 'pytest', 'unittest'];
|
|
264
|
+
return testFrameworks.some(fw => allDeps[fw]);
|
|
265
|
+
} catch { return false; }
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
// ---------------------------------------------------------------------------
|
|
269
|
+
// Main analyzer
|
|
270
|
+
// ---------------------------------------------------------------------------
|
|
271
|
+
|
|
272
|
+
export async function analyzeTestability(
|
|
273
|
+
options: TestabilityOptions,
|
|
274
|
+
): Promise<TestabilityReport> {
|
|
275
|
+
const allFiles = collectFiles(options.rootDir, options);
|
|
276
|
+
|
|
277
|
+
const sourceFiles = allFiles.filter(f => !isTestFile(f, options.testPatterns));
|
|
278
|
+
const testFiles = allFiles.filter(f => isTestFile(f, options.testPatterns));
|
|
279
|
+
|
|
280
|
+
const aggregated: FileAnalysis = {
|
|
281
|
+
pureFunctions: 0,
|
|
282
|
+
totalFunctions: 0,
|
|
283
|
+
injectionPatterns: 0,
|
|
284
|
+
totalClasses: 0,
|
|
285
|
+
bloatedInterfaces: 0,
|
|
286
|
+
totalInterfaces: 0,
|
|
287
|
+
externalStateMutations: 0,
|
|
288
|
+
};
|
|
289
|
+
|
|
290
|
+
for (const f of sourceFiles) {
|
|
291
|
+
const a = analyzeFileTestability(f);
|
|
292
|
+
for (const key of Object.keys(aggregated) as Array<keyof FileAnalysis>) {
|
|
293
|
+
aggregated[key] += a[key];
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
const hasTestFramework = detectTestFramework(options.rootDir);
|
|
298
|
+
|
|
299
|
+
const indexResult = calculateTestabilityIndex({
|
|
300
|
+
testFiles: testFiles.length,
|
|
301
|
+
sourceFiles: sourceFiles.length,
|
|
302
|
+
pureFunctions: aggregated.pureFunctions,
|
|
303
|
+
totalFunctions: Math.max(1, aggregated.totalFunctions),
|
|
304
|
+
injectionPatterns: aggregated.injectionPatterns,
|
|
305
|
+
totalClasses: Math.max(1, aggregated.totalClasses),
|
|
306
|
+
bloatedInterfaces: aggregated.bloatedInterfaces,
|
|
307
|
+
totalInterfaces: Math.max(1, aggregated.totalInterfaces),
|
|
308
|
+
externalStateMutations: aggregated.externalStateMutations,
|
|
309
|
+
hasTestFramework,
|
|
310
|
+
});
|
|
311
|
+
|
|
312
|
+
// Build issues
|
|
313
|
+
const issues: TestabilityIssue[] = [];
|
|
314
|
+
const minCoverage = options.minCoverageRatio ?? 0.3;
|
|
315
|
+
const actualRatio = sourceFiles.length > 0 ? testFiles.length / sourceFiles.length : 0;
|
|
316
|
+
|
|
317
|
+
if (!hasTestFramework) {
|
|
318
|
+
issues.push({
|
|
319
|
+
type: 'low-testability',
|
|
320
|
+
dimension: 'framework',
|
|
321
|
+
severity: 'critical',
|
|
322
|
+
message: 'No testing framework detected in package.json — AI changes cannot be verified at all.',
|
|
323
|
+
location: { file: options.rootDir, line: 0 },
|
|
324
|
+
suggestion: 'Add Jest, Vitest, or another testing framework as a devDependency.',
|
|
325
|
+
});
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
if (actualRatio < minCoverage) {
|
|
329
|
+
const needed = Math.ceil(sourceFiles.length * minCoverage) - testFiles.length;
|
|
330
|
+
issues.push({
|
|
331
|
+
type: 'low-testability',
|
|
332
|
+
dimension: 'test-coverage',
|
|
333
|
+
severity: actualRatio === 0 ? 'critical' : 'major',
|
|
334
|
+
message: `Test ratio is ${Math.round(actualRatio * 100)}% (${testFiles.length} test files for ${sourceFiles.length} source files). Need at least ${Math.round(minCoverage * 100)}%.`,
|
|
335
|
+
location: { file: options.rootDir, line: 0 },
|
|
336
|
+
suggestion: `Add ~${needed} test file(s) to reach the ${Math.round(minCoverage * 100)}% minimum for safe AI assistance.`,
|
|
337
|
+
});
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
if (indexResult.dimensions.purityScore < 50) {
|
|
341
|
+
issues.push({
|
|
342
|
+
type: 'low-testability',
|
|
343
|
+
dimension: 'purity',
|
|
344
|
+
severity: 'major',
|
|
345
|
+
message: `Only ${indexResult.dimensions.purityScore}% of functions are pure — side-effectful functions require complex test setup.`,
|
|
346
|
+
location: { file: options.rootDir, line: 0 },
|
|
347
|
+
suggestion: 'Extract pure transformation logic from I/O and mutation code.',
|
|
348
|
+
});
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
if (indexResult.dimensions.observabilityScore < 50) {
|
|
352
|
+
issues.push({
|
|
353
|
+
type: 'low-testability',
|
|
354
|
+
dimension: 'observability',
|
|
355
|
+
severity: 'major',
|
|
356
|
+
message: `Many functions mutate external state directly — outputs are invisible to unit tests.`,
|
|
357
|
+
location: { file: options.rootDir, line: 0 },
|
|
358
|
+
suggestion: 'Prefer returning values over mutating shared state.',
|
|
359
|
+
});
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
return {
|
|
363
|
+
summary: {
|
|
364
|
+
sourceFiles: sourceFiles.length,
|
|
365
|
+
testFiles: testFiles.length,
|
|
366
|
+
coverageRatio: Math.round(actualRatio * 100) / 100,
|
|
367
|
+
score: indexResult.score,
|
|
368
|
+
rating: indexResult.rating,
|
|
369
|
+
aiChangeSafetyRating: indexResult.aiChangeSafetyRating,
|
|
370
|
+
dimensions: indexResult.dimensions,
|
|
371
|
+
},
|
|
372
|
+
issues,
|
|
373
|
+
rawData: {
|
|
374
|
+
sourceFiles: sourceFiles.length,
|
|
375
|
+
testFiles: testFiles.length,
|
|
376
|
+
...aggregated,
|
|
377
|
+
hasTestFramework,
|
|
378
|
+
},
|
|
379
|
+
recommendations: indexResult.recommendations,
|
|
380
|
+
};
|
|
381
|
+
}
|
package/src/cli.ts
ADDED
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
import { Command } from 'commander';
|
|
4
|
+
import { analyzeTestability } from './analyzer';
|
|
5
|
+
import { calculateTestabilityScore } from './scoring';
|
|
6
|
+
import type { TestabilityOptions } from './types';
|
|
7
|
+
import chalk from 'chalk';
|
|
8
|
+
import { writeFileSync, mkdirSync, existsSync } from 'fs';
|
|
9
|
+
import { dirname } from 'path';
|
|
10
|
+
import { loadConfig, mergeConfigWithDefaults, resolveOutputPath } from '@aiready/core';
|
|
11
|
+
|
|
12
|
+
const program = new Command();
|
|
13
|
+
|
|
14
|
+
program
|
|
15
|
+
.name('aiready-testability')
|
|
16
|
+
.description('Measure how safely AI-generated changes can be verified in your codebase')
|
|
17
|
+
.version('0.1.0')
|
|
18
|
+
.addHelpText('after', `
|
|
19
|
+
DIMENSIONS MEASURED:
|
|
20
|
+
Test Coverage Ratio of test files to source files
|
|
21
|
+
Function Purity Pure functions are trivially AI-testable
|
|
22
|
+
Dependency Inject. DI makes classes mockable and verifiable
|
|
23
|
+
Interface Focus Small interfaces are easier to mock
|
|
24
|
+
Observability Functions returning values > mutating state
|
|
25
|
+
|
|
26
|
+
AI CHANGE SAFETY RATINGS:
|
|
27
|
+
safe ✅ AI changes can be safely verified (≥50% coverage + score ≥70)
|
|
28
|
+
moderate-risk ⚠️ Some risk — partial test coverage
|
|
29
|
+
high-risk 🔴 Tests exist but insufficient — AI changes may slip through
|
|
30
|
+
blind-risk 💀 NO TESTS — AI changes cannot be verified at all
|
|
31
|
+
|
|
32
|
+
EXAMPLES:
|
|
33
|
+
aiready-testability . # Full analysis
|
|
34
|
+
aiready-testability src/ --output json # JSON report
|
|
35
|
+
aiready-testability . --min-coverage 0.5 # Stricter 50% threshold
|
|
36
|
+
`)
|
|
37
|
+
.argument('<directory>', 'Directory to analyze')
|
|
38
|
+
.option('--min-coverage <ratio>', 'Minimum acceptable test/source ratio (default: 0.3)', '0.3')
|
|
39
|
+
.option('--test-patterns <patterns>', 'Additional test file patterns (comma-separated)')
|
|
40
|
+
.option('--include <patterns>', 'File patterns to include (comma-separated)')
|
|
41
|
+
.option('--exclude <patterns>', 'File patterns to exclude (comma-separated)')
|
|
42
|
+
.option('-o, --output <format>', 'Output format: console|json', 'console')
|
|
43
|
+
.option('--output-file <path>', 'Output file path (for json)')
|
|
44
|
+
.action(async (directory, options) => {
|
|
45
|
+
console.log(chalk.blue('🧪 Analyzing testability...\n'));
|
|
46
|
+
const startTime = Date.now();
|
|
47
|
+
|
|
48
|
+
const config = await loadConfig(directory);
|
|
49
|
+
const mergedConfig = mergeConfigWithDefaults(config, {
|
|
50
|
+
minCoverageRatio: 0.3,
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
const finalOptions: TestabilityOptions = {
|
|
54
|
+
rootDir: directory,
|
|
55
|
+
minCoverageRatio: parseFloat(options.minCoverage ?? '0.3') || mergedConfig.minCoverageRatio,
|
|
56
|
+
testPatterns: options.testPatterns?.split(','),
|
|
57
|
+
include: options.include?.split(','),
|
|
58
|
+
exclude: options.exclude?.split(','),
|
|
59
|
+
};
|
|
60
|
+
|
|
61
|
+
const report = await analyzeTestability(finalOptions);
|
|
62
|
+
const scoring = calculateTestabilityScore(report);
|
|
63
|
+
const elapsed = ((Date.now() - startTime) / 1000).toFixed(2);
|
|
64
|
+
|
|
65
|
+
if (options.output === 'json') {
|
|
66
|
+
const payload = { report, score: scoring };
|
|
67
|
+
const outputPath = resolveOutputPath(
|
|
68
|
+
options.outputFile,
|
|
69
|
+
`testability-report-${new Date().toISOString().split('T')[0]}.json`,
|
|
70
|
+
directory,
|
|
71
|
+
);
|
|
72
|
+
const dir = dirname(outputPath);
|
|
73
|
+
if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
|
|
74
|
+
writeFileSync(outputPath, JSON.stringify(payload, null, 2));
|
|
75
|
+
console.log(chalk.green(`✓ Report saved to ${outputPath}`));
|
|
76
|
+
} else {
|
|
77
|
+
displayConsoleReport(report, scoring, elapsed);
|
|
78
|
+
}
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
program.parse();
|
|
82
|
+
|
|
83
|
+
function safetyColor(rating: string) {
|
|
84
|
+
switch (rating) {
|
|
85
|
+
case 'safe': return chalk.green;
|
|
86
|
+
case 'moderate-risk': return chalk.yellow;
|
|
87
|
+
case 'high-risk': return chalk.red;
|
|
88
|
+
case 'blind-risk': return chalk.bgRed.white;
|
|
89
|
+
default: return chalk.white;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function safetyIcon(rating: string) {
|
|
94
|
+
switch (rating) {
|
|
95
|
+
case 'safe': return '✅';
|
|
96
|
+
case 'moderate-risk': return '⚠️ ';
|
|
97
|
+
case 'high-risk': return '🔴';
|
|
98
|
+
case 'blind-risk': return '💀';
|
|
99
|
+
default: return '❓';
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
function scoreBar(val: number): string {
|
|
104
|
+
return '█'.repeat(Math.round(val / 10)).padEnd(10, '░');
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function displayConsoleReport(report: any, scoring: any, elapsed: string) {
|
|
108
|
+
const { summary, rawData, issues, recommendations } = report;
|
|
109
|
+
|
|
110
|
+
// The most important banner
|
|
111
|
+
const safetyRating = summary.aiChangeSafetyRating;
|
|
112
|
+
console.log(chalk.bold('\n🧪 Testability Analysis\n'));
|
|
113
|
+
|
|
114
|
+
if (safetyRating === 'blind-risk') {
|
|
115
|
+
console.log(chalk.bgRed.white.bold(
|
|
116
|
+
' 💀 BLIND RISK — NO TESTS DETECTED. AI-GENERATED CHANGES CANNOT BE VERIFIED. '
|
|
117
|
+
));
|
|
118
|
+
console.log();
|
|
119
|
+
} else if (safetyRating === 'high-risk') {
|
|
120
|
+
console.log(chalk.red.bold(` 🔴 HIGH RISK — Insufficient test coverage. AI changes may introduce silent bugs.`));
|
|
121
|
+
console.log();
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
console.log(`AI Change Safety: ${safetyColor(safetyRating)(`${safetyIcon(safetyRating)} ${safetyRating.toUpperCase()}`)}`);
|
|
125
|
+
console.log(`Score: ${chalk.bold(summary.score + '/100')} (${summary.rating})`);
|
|
126
|
+
console.log(`Source Files: ${chalk.cyan(rawData.sourceFiles)} Test Files: ${chalk.cyan(rawData.testFiles)}`);
|
|
127
|
+
console.log(`Coverage Ratio: ${chalk.bold(Math.round(summary.coverageRatio * 100) + '%')}`);
|
|
128
|
+
console.log(`Analysis Time: ${chalk.gray(elapsed + 's')}\n`);
|
|
129
|
+
|
|
130
|
+
console.log(chalk.bold('📐 Dimension Scores\n'));
|
|
131
|
+
const dims: [string, number][] = [
|
|
132
|
+
['Test Coverage', summary.dimensions.testCoverageRatio],
|
|
133
|
+
['Function Purity', summary.dimensions.purityScore],
|
|
134
|
+
['Dependency Injection', summary.dimensions.dependencyInjectionScore],
|
|
135
|
+
['Interface Focus', summary.dimensions.interfaceFocusScore],
|
|
136
|
+
['Observability', summary.dimensions.observabilityScore],
|
|
137
|
+
];
|
|
138
|
+
for (const [name, val] of dims) {
|
|
139
|
+
const color = val >= 70 ? chalk.green : val >= 50 ? chalk.yellow : chalk.red;
|
|
140
|
+
console.log(` ${name.padEnd(22)} ${color(scoreBar(val))} ${val}/100`);
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
if (issues.length > 0) {
|
|
144
|
+
console.log(chalk.bold('\n⚠️ Issues\n'));
|
|
145
|
+
for (const issue of issues) {
|
|
146
|
+
const sev = issue.severity === 'critical' ? chalk.red : issue.severity === 'major' ? chalk.yellow : chalk.blue;
|
|
147
|
+
console.log(`${sev(issue.severity.toUpperCase())} ${issue.message}`);
|
|
148
|
+
if (issue.suggestion) console.log(` ${chalk.dim('→')} ${chalk.italic(issue.suggestion)}`);
|
|
149
|
+
console.log();
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
if (recommendations.length > 0) {
|
|
154
|
+
console.log(chalk.bold('💡 Recommendations\n'));
|
|
155
|
+
recommendations.forEach((rec: string, i: number) => {
|
|
156
|
+
console.log(`${i + 1}. ${rec}`);
|
|
157
|
+
});
|
|
158
|
+
}
|
|
159
|
+
console.log();
|
|
160
|
+
}
|
package/src/index.ts
ADDED
package/src/scoring.ts
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { calculateTestabilityIndex } from '@aiready/core';
|
|
2
|
+
import type { ToolScoringOutput } from '@aiready/core';
|
|
3
|
+
import type { TestabilityReport } from './types';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Convert testability report into a ToolScoringOutput for the unified score.
|
|
7
|
+
*/
|
|
8
|
+
export function calculateTestabilityScore(report: TestabilityReport): ToolScoringOutput {
|
|
9
|
+
const { summary, rawData, recommendations } = report;
|
|
10
|
+
|
|
11
|
+
const factors: ToolScoringOutput['factors'] = [
|
|
12
|
+
{
|
|
13
|
+
name: 'Test Coverage',
|
|
14
|
+
impact: Math.round(summary.dimensions.testCoverageRatio - 50),
|
|
15
|
+
description: `${rawData.testFiles} test files / ${rawData.sourceFiles} source files (${Math.round(summary.coverageRatio * 100)}%)`,
|
|
16
|
+
},
|
|
17
|
+
{
|
|
18
|
+
name: 'Function Purity',
|
|
19
|
+
impact: Math.round(summary.dimensions.purityScore - 50),
|
|
20
|
+
description: `${rawData.pureFunctions}/${rawData.totalFunctions} functions are pure`,
|
|
21
|
+
},
|
|
22
|
+
{
|
|
23
|
+
name: 'Dependency Injection',
|
|
24
|
+
impact: Math.round(summary.dimensions.dependencyInjectionScore - 50),
|
|
25
|
+
description: `${rawData.injectionPatterns}/${rawData.totalClasses} classes use DI`,
|
|
26
|
+
},
|
|
27
|
+
{
|
|
28
|
+
name: 'Interface Focus',
|
|
29
|
+
impact: Math.round(summary.dimensions.interfaceFocusScore - 50),
|
|
30
|
+
description: `${rawData.bloatedInterfaces} interfaces have >10 methods`,
|
|
31
|
+
},
|
|
32
|
+
{
|
|
33
|
+
name: 'Observability',
|
|
34
|
+
impact: Math.round(summary.dimensions.observabilityScore - 50),
|
|
35
|
+
description: `${rawData.externalStateMutations} functions mutate external state`,
|
|
36
|
+
},
|
|
37
|
+
];
|
|
38
|
+
|
|
39
|
+
const recs: ToolScoringOutput['recommendations'] = recommendations.map(action => ({
|
|
40
|
+
action,
|
|
41
|
+
estimatedImpact: summary.aiChangeSafetyRating === 'blind-risk' ? 15 : 8,
|
|
42
|
+
priority: summary.aiChangeSafetyRating === 'blind-risk' || summary.aiChangeSafetyRating === 'high-risk'
|
|
43
|
+
? 'high'
|
|
44
|
+
: 'medium',
|
|
45
|
+
}));
|
|
46
|
+
|
|
47
|
+
return {
|
|
48
|
+
toolName: 'testability',
|
|
49
|
+
score: summary.score,
|
|
50
|
+
rawMetrics: {
|
|
51
|
+
...rawData,
|
|
52
|
+
rating: summary.rating,
|
|
53
|
+
aiChangeSafetyRating: summary.aiChangeSafetyRating,
|
|
54
|
+
coverageRatio: summary.coverageRatio,
|
|
55
|
+
},
|
|
56
|
+
factors,
|
|
57
|
+
recommendations: recs,
|
|
58
|
+
};
|
|
59
|
+
}
|
package/src/types.ts
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import type { ScanOptions, Issue } from '@aiready/core';
|
|
2
|
+
|
|
3
|
+
export interface TestabilityOptions {
|
|
4
|
+
/** Root directory to scan */
|
|
5
|
+
rootDir: string;
|
|
6
|
+
/** Minimum test-to-source ratio to consider acceptable (default: 0.3) */
|
|
7
|
+
minCoverageRatio?: number;
|
|
8
|
+
/** Custom test file patterns in addition to built-ins */
|
|
9
|
+
testPatterns?: string[];
|
|
10
|
+
/** Maximum scan depth */
|
|
11
|
+
maxDepth?: number;
|
|
12
|
+
/** File glob patterns to include */
|
|
13
|
+
include?: string[];
|
|
14
|
+
/** File glob patterns to exclude */
|
|
15
|
+
exclude?: string[];
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export interface TestabilityIssue extends Issue {
|
|
19
|
+
type: 'low-testability';
|
|
20
|
+
/** Category of testability barrier */
|
|
21
|
+
dimension: 'test-coverage' | 'purity' | 'dependency-injection' | 'interface-focus' | 'observability' | 'framework';
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export interface TestabilityReport {
|
|
25
|
+
summary: {
|
|
26
|
+
sourceFiles: number;
|
|
27
|
+
testFiles: number;
|
|
28
|
+
coverageRatio: number;
|
|
29
|
+
score: number;
|
|
30
|
+
rating: 'excellent' | 'good' | 'moderate' | 'poor' | 'unverifiable';
|
|
31
|
+
/** THE MOST IMPORTANT FIELD: is AI-generated code safe to ship? */
|
|
32
|
+
aiChangeSafetyRating: 'safe' | 'moderate-risk' | 'high-risk' | 'blind-risk';
|
|
33
|
+
dimensions: {
|
|
34
|
+
testCoverageRatio: number;
|
|
35
|
+
purityScore: number;
|
|
36
|
+
dependencyInjectionScore: number;
|
|
37
|
+
interfaceFocusScore: number;
|
|
38
|
+
observabilityScore: number;
|
|
39
|
+
};
|
|
40
|
+
};
|
|
41
|
+
issues: TestabilityIssue[];
|
|
42
|
+
rawData: {
|
|
43
|
+
sourceFiles: number;
|
|
44
|
+
testFiles: number;
|
|
45
|
+
pureFunctions: number;
|
|
46
|
+
totalFunctions: number;
|
|
47
|
+
injectionPatterns: number;
|
|
48
|
+
totalClasses: number;
|
|
49
|
+
bloatedInterfaces: number;
|
|
50
|
+
totalInterfaces: number;
|
|
51
|
+
externalStateMutations: number;
|
|
52
|
+
hasTestFramework: boolean;
|
|
53
|
+
};
|
|
54
|
+
recommendations: string[];
|
|
55
|
+
}
|