@aiready/testability 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +24 -0
- package/.turbo/turbo-test.log +16 -0
- package/dist/chunk-CYZ7DTWN.mjs +338 -0
- package/dist/cli.d.mts +1 -0
- package/dist/cli.d.ts +1 -0
- package/dist/cli.js +498 -0
- package/dist/cli.mjs +143 -0
- package/dist/index.d.mts +74 -0
- package/dist/index.d.ts +74 -0
- package/dist/index.js +365 -0
- package/dist/index.mjs +8 -0
- package/package.json +62 -0
- package/src/__tests__/analyzer.test.ts +84 -0
- package/src/analyzer.ts +381 -0
- package/src/cli.ts +160 -0
- package/src/index.ts +7 -0
- package/src/scoring.ts +59 -0
- package/src/types.ts +55 -0
- package/tsconfig.json +9 -0
package/dist/cli.js
ADDED
|
@@ -0,0 +1,498 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
"use strict";
|
|
3
|
+
var __create = Object.create;
|
|
4
|
+
var __defProp = Object.defineProperty;
|
|
5
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
6
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
7
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
8
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
9
|
+
var __copyProps = (to, from, except, desc) => {
|
|
10
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
11
|
+
for (let key of __getOwnPropNames(from))
|
|
12
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
13
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
14
|
+
}
|
|
15
|
+
return to;
|
|
16
|
+
};
|
|
17
|
+
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
18
|
+
// If the importer is in node compatibility mode or this is not an ESM
|
|
19
|
+
// file that has been converted to a CommonJS file using a Babel-
|
|
20
|
+
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
21
|
+
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
22
|
+
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
23
|
+
mod
|
|
24
|
+
));
|
|
25
|
+
|
|
26
|
+
// src/cli.ts
|
|
27
|
+
var import_commander = require("commander");
|
|
28
|
+
|
|
29
|
+
// src/analyzer.ts
|
|
30
|
+
var import_fs = require("fs");
|
|
31
|
+
var import_path = require("path");
|
|
32
|
+
var import_typescript_estree = require("@typescript-eslint/typescript-estree");
|
|
33
|
+
var import_core = require("@aiready/core");
|
|
34
|
+
var SRC_EXTENSIONS = /* @__PURE__ */ new Set([".ts", ".tsx", ".js", ".jsx"]);
|
|
35
|
+
var DEFAULT_EXCLUDES = ["node_modules", "dist", ".git", "coverage", ".turbo", "build"];
|
|
36
|
+
var TEST_PATTERNS = [
|
|
37
|
+
/\.(test|spec)\.(ts|tsx|js|jsx)$/,
|
|
38
|
+
/__tests__\//,
|
|
39
|
+
/\/tests?\//,
|
|
40
|
+
/\/e2e\//,
|
|
41
|
+
/\/fixtures\//
|
|
42
|
+
];
|
|
43
|
+
function isTestFile(filePath, extra) {
|
|
44
|
+
if (TEST_PATTERNS.some((p) => p.test(filePath))) return true;
|
|
45
|
+
if (extra) return extra.some((p) => filePath.includes(p));
|
|
46
|
+
return false;
|
|
47
|
+
}
|
|
48
|
+
function isSourceFile(filePath) {
|
|
49
|
+
return SRC_EXTENSIONS.has((0, import_path.extname)(filePath));
|
|
50
|
+
}
|
|
51
|
+
function collectFiles(dir, options, depth = 0) {
|
|
52
|
+
if (depth > (options.maxDepth ?? 20)) return [];
|
|
53
|
+
const excludes = [...DEFAULT_EXCLUDES, ...options.exclude ?? []];
|
|
54
|
+
const files = [];
|
|
55
|
+
let entries;
|
|
56
|
+
try {
|
|
57
|
+
entries = (0, import_fs.readdirSync)(dir);
|
|
58
|
+
} catch {
|
|
59
|
+
return files;
|
|
60
|
+
}
|
|
61
|
+
for (const entry of entries) {
|
|
62
|
+
if (excludes.some((ex) => entry === ex || entry.includes(ex))) continue;
|
|
63
|
+
const full = (0, import_path.join)(dir, entry);
|
|
64
|
+
let stat;
|
|
65
|
+
try {
|
|
66
|
+
stat = (0, import_fs.statSync)(full);
|
|
67
|
+
} catch {
|
|
68
|
+
continue;
|
|
69
|
+
}
|
|
70
|
+
if (stat.isDirectory()) {
|
|
71
|
+
files.push(...collectFiles(full, options, depth + 1));
|
|
72
|
+
} else if (stat.isFile() && isSourceFile(full)) {
|
|
73
|
+
if (!options.include || options.include.some((p) => full.includes(p))) {
|
|
74
|
+
files.push(full);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
return files;
|
|
79
|
+
}
|
|
80
|
+
function countMethodsInInterface(node) {
|
|
81
|
+
if (node.type === "TSInterfaceDeclaration") {
|
|
82
|
+
return node.body.body.filter(
|
|
83
|
+
(m) => m.type === "TSMethodSignature" || m.type === "TSPropertySignature"
|
|
84
|
+
).length;
|
|
85
|
+
}
|
|
86
|
+
if (node.type === "TSTypeAliasDeclaration" && node.typeAnnotation.type === "TSTypeLiteral") {
|
|
87
|
+
return node.typeAnnotation.members.length;
|
|
88
|
+
}
|
|
89
|
+
return 0;
|
|
90
|
+
}
|
|
91
|
+
function hasDependencyInjection(node) {
|
|
92
|
+
for (const member of node.body.body) {
|
|
93
|
+
if (member.type === "MethodDefinition" && member.key.type === "Identifier" && member.key.name === "constructor") {
|
|
94
|
+
const fn = member.value;
|
|
95
|
+
if (fn.params && fn.params.length > 0) {
|
|
96
|
+
const typedParams = fn.params.filter((p) => {
|
|
97
|
+
const param = p;
|
|
98
|
+
return param.typeAnnotation != null || param.parameter?.typeAnnotation != null;
|
|
99
|
+
});
|
|
100
|
+
if (typedParams.length > 0) return true;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
return false;
|
|
105
|
+
}
|
|
106
|
+
function isPureFunction(fn) {
|
|
107
|
+
let hasReturn = false;
|
|
108
|
+
let hasSideEffect = false;
|
|
109
|
+
function walk(node) {
|
|
110
|
+
if (node.type === "ReturnStatement" && node.argument) hasReturn = true;
|
|
111
|
+
if (node.type === "AssignmentExpression" && node.left.type === "MemberExpression") hasSideEffect = true;
|
|
112
|
+
if (node.type === "CallExpression" && node.callee.type === "MemberExpression" && node.callee.object.type === "Identifier" && ["console", "process", "window", "document", "fs"].includes(node.callee.object.name)) hasSideEffect = true;
|
|
113
|
+
for (const key of Object.keys(node)) {
|
|
114
|
+
if (key === "parent") continue;
|
|
115
|
+
const child = node[key];
|
|
116
|
+
if (child && typeof child === "object") {
|
|
117
|
+
if (Array.isArray(child)) {
|
|
118
|
+
child.forEach((c) => c?.type && walk(c));
|
|
119
|
+
} else if (child.type) {
|
|
120
|
+
walk(child);
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
if (fn.body?.type === "BlockStatement") {
|
|
126
|
+
fn.body.body.forEach((s) => walk(s));
|
|
127
|
+
} else if (fn.body) {
|
|
128
|
+
hasReturn = true;
|
|
129
|
+
}
|
|
130
|
+
return hasReturn && !hasSideEffect;
|
|
131
|
+
}
|
|
132
|
+
function hasExternalStateMutation(fn) {
|
|
133
|
+
let found = false;
|
|
134
|
+
function walk(node) {
|
|
135
|
+
if (found) return;
|
|
136
|
+
if (node.type === "AssignmentExpression" && node.left.type === "MemberExpression") found = true;
|
|
137
|
+
for (const key of Object.keys(node)) {
|
|
138
|
+
if (key === "parent") continue;
|
|
139
|
+
const child = node[key];
|
|
140
|
+
if (child && typeof child === "object") {
|
|
141
|
+
if (Array.isArray(child)) child.forEach((c) => c?.type && walk(c));
|
|
142
|
+
else if (child.type) walk(child);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
if (fn.body?.type === "BlockStatement") fn.body.body.forEach((s) => walk(s));
|
|
147
|
+
return found;
|
|
148
|
+
}
|
|
149
|
+
function analyzeFileTestability(filePath) {
|
|
150
|
+
const result = {
|
|
151
|
+
pureFunctions: 0,
|
|
152
|
+
totalFunctions: 0,
|
|
153
|
+
injectionPatterns: 0,
|
|
154
|
+
totalClasses: 0,
|
|
155
|
+
bloatedInterfaces: 0,
|
|
156
|
+
totalInterfaces: 0,
|
|
157
|
+
externalStateMutations: 0
|
|
158
|
+
};
|
|
159
|
+
let code;
|
|
160
|
+
try {
|
|
161
|
+
code = (0, import_fs.readFileSync)(filePath, "utf-8");
|
|
162
|
+
} catch {
|
|
163
|
+
return result;
|
|
164
|
+
}
|
|
165
|
+
let ast;
|
|
166
|
+
try {
|
|
167
|
+
ast = (0, import_typescript_estree.parse)(code, {
|
|
168
|
+
jsx: filePath.endsWith(".tsx") || filePath.endsWith(".jsx"),
|
|
169
|
+
range: false,
|
|
170
|
+
loc: false
|
|
171
|
+
});
|
|
172
|
+
} catch {
|
|
173
|
+
return result;
|
|
174
|
+
}
|
|
175
|
+
function visit(node) {
|
|
176
|
+
if (node.type === "FunctionDeclaration" || node.type === "FunctionExpression" || node.type === "ArrowFunctionExpression") {
|
|
177
|
+
result.totalFunctions++;
|
|
178
|
+
if (isPureFunction(node)) result.pureFunctions++;
|
|
179
|
+
if (hasExternalStateMutation(node)) result.externalStateMutations++;
|
|
180
|
+
}
|
|
181
|
+
if (node.type === "ClassDeclaration" || node.type === "ClassExpression") {
|
|
182
|
+
result.totalClasses++;
|
|
183
|
+
if (hasDependencyInjection(node)) result.injectionPatterns++;
|
|
184
|
+
}
|
|
185
|
+
if (node.type === "TSInterfaceDeclaration" || node.type === "TSTypeAliasDeclaration") {
|
|
186
|
+
result.totalInterfaces++;
|
|
187
|
+
const methodCount = countMethodsInInterface(node);
|
|
188
|
+
if (methodCount > 10) result.bloatedInterfaces++;
|
|
189
|
+
}
|
|
190
|
+
for (const key of Object.keys(node)) {
|
|
191
|
+
if (key === "parent") continue;
|
|
192
|
+
const child = node[key];
|
|
193
|
+
if (child && typeof child === "object") {
|
|
194
|
+
if (Array.isArray(child)) child.forEach((c) => c?.type && visit(c));
|
|
195
|
+
else if (child.type) visit(child);
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
ast.body.forEach(visit);
|
|
200
|
+
return result;
|
|
201
|
+
}
|
|
202
|
+
function detectTestFramework(rootDir) {
|
|
203
|
+
const pkgPath = (0, import_path.join)(rootDir, "package.json");
|
|
204
|
+
if (!(0, import_fs.existsSync)(pkgPath)) return false;
|
|
205
|
+
try {
|
|
206
|
+
const pkg = JSON.parse((0, import_fs.readFileSync)(pkgPath, "utf-8"));
|
|
207
|
+
const allDeps = {
|
|
208
|
+
...pkg.dependencies ?? {},
|
|
209
|
+
...pkg.devDependencies ?? {}
|
|
210
|
+
};
|
|
211
|
+
const testFrameworks = ["jest", "vitest", "mocha", "jasmine", "ava", "tap", "pytest", "unittest"];
|
|
212
|
+
return testFrameworks.some((fw) => allDeps[fw]);
|
|
213
|
+
} catch {
|
|
214
|
+
return false;
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
async function analyzeTestability(options) {
|
|
218
|
+
const allFiles = collectFiles(options.rootDir, options);
|
|
219
|
+
const sourceFiles = allFiles.filter((f) => !isTestFile(f, options.testPatterns));
|
|
220
|
+
const testFiles = allFiles.filter((f) => isTestFile(f, options.testPatterns));
|
|
221
|
+
const aggregated = {
|
|
222
|
+
pureFunctions: 0,
|
|
223
|
+
totalFunctions: 0,
|
|
224
|
+
injectionPatterns: 0,
|
|
225
|
+
totalClasses: 0,
|
|
226
|
+
bloatedInterfaces: 0,
|
|
227
|
+
totalInterfaces: 0,
|
|
228
|
+
externalStateMutations: 0
|
|
229
|
+
};
|
|
230
|
+
for (const f of sourceFiles) {
|
|
231
|
+
const a = analyzeFileTestability(f);
|
|
232
|
+
for (const key of Object.keys(aggregated)) {
|
|
233
|
+
aggregated[key] += a[key];
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
const hasTestFramework = detectTestFramework(options.rootDir);
|
|
237
|
+
const indexResult = (0, import_core.calculateTestabilityIndex)({
|
|
238
|
+
testFiles: testFiles.length,
|
|
239
|
+
sourceFiles: sourceFiles.length,
|
|
240
|
+
pureFunctions: aggregated.pureFunctions,
|
|
241
|
+
totalFunctions: Math.max(1, aggregated.totalFunctions),
|
|
242
|
+
injectionPatterns: aggregated.injectionPatterns,
|
|
243
|
+
totalClasses: Math.max(1, aggregated.totalClasses),
|
|
244
|
+
bloatedInterfaces: aggregated.bloatedInterfaces,
|
|
245
|
+
totalInterfaces: Math.max(1, aggregated.totalInterfaces),
|
|
246
|
+
externalStateMutations: aggregated.externalStateMutations,
|
|
247
|
+
hasTestFramework
|
|
248
|
+
});
|
|
249
|
+
const issues = [];
|
|
250
|
+
const minCoverage = options.minCoverageRatio ?? 0.3;
|
|
251
|
+
const actualRatio = sourceFiles.length > 0 ? testFiles.length / sourceFiles.length : 0;
|
|
252
|
+
if (!hasTestFramework) {
|
|
253
|
+
issues.push({
|
|
254
|
+
type: "low-testability",
|
|
255
|
+
dimension: "framework",
|
|
256
|
+
severity: "critical",
|
|
257
|
+
message: "No testing framework detected in package.json \u2014 AI changes cannot be verified at all.",
|
|
258
|
+
location: { file: options.rootDir, line: 0 },
|
|
259
|
+
suggestion: "Add Jest, Vitest, or another testing framework as a devDependency."
|
|
260
|
+
});
|
|
261
|
+
}
|
|
262
|
+
if (actualRatio < minCoverage) {
|
|
263
|
+
const needed = Math.ceil(sourceFiles.length * minCoverage) - testFiles.length;
|
|
264
|
+
issues.push({
|
|
265
|
+
type: "low-testability",
|
|
266
|
+
dimension: "test-coverage",
|
|
267
|
+
severity: actualRatio === 0 ? "critical" : "major",
|
|
268
|
+
message: `Test ratio is ${Math.round(actualRatio * 100)}% (${testFiles.length} test files for ${sourceFiles.length} source files). Need at least ${Math.round(minCoverage * 100)}%.`,
|
|
269
|
+
location: { file: options.rootDir, line: 0 },
|
|
270
|
+
suggestion: `Add ~${needed} test file(s) to reach the ${Math.round(minCoverage * 100)}% minimum for safe AI assistance.`
|
|
271
|
+
});
|
|
272
|
+
}
|
|
273
|
+
if (indexResult.dimensions.purityScore < 50) {
|
|
274
|
+
issues.push({
|
|
275
|
+
type: "low-testability",
|
|
276
|
+
dimension: "purity",
|
|
277
|
+
severity: "major",
|
|
278
|
+
message: `Only ${indexResult.dimensions.purityScore}% of functions are pure \u2014 side-effectful functions require complex test setup.`,
|
|
279
|
+
location: { file: options.rootDir, line: 0 },
|
|
280
|
+
suggestion: "Extract pure transformation logic from I/O and mutation code."
|
|
281
|
+
});
|
|
282
|
+
}
|
|
283
|
+
if (indexResult.dimensions.observabilityScore < 50) {
|
|
284
|
+
issues.push({
|
|
285
|
+
type: "low-testability",
|
|
286
|
+
dimension: "observability",
|
|
287
|
+
severity: "major",
|
|
288
|
+
message: `Many functions mutate external state directly \u2014 outputs are invisible to unit tests.`,
|
|
289
|
+
location: { file: options.rootDir, line: 0 },
|
|
290
|
+
suggestion: "Prefer returning values over mutating shared state."
|
|
291
|
+
});
|
|
292
|
+
}
|
|
293
|
+
return {
|
|
294
|
+
summary: {
|
|
295
|
+
sourceFiles: sourceFiles.length,
|
|
296
|
+
testFiles: testFiles.length,
|
|
297
|
+
coverageRatio: Math.round(actualRatio * 100) / 100,
|
|
298
|
+
score: indexResult.score,
|
|
299
|
+
rating: indexResult.rating,
|
|
300
|
+
aiChangeSafetyRating: indexResult.aiChangeSafetyRating,
|
|
301
|
+
dimensions: indexResult.dimensions
|
|
302
|
+
},
|
|
303
|
+
issues,
|
|
304
|
+
rawData: {
|
|
305
|
+
sourceFiles: sourceFiles.length,
|
|
306
|
+
testFiles: testFiles.length,
|
|
307
|
+
...aggregated,
|
|
308
|
+
hasTestFramework
|
|
309
|
+
},
|
|
310
|
+
recommendations: indexResult.recommendations
|
|
311
|
+
};
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
// src/scoring.ts
|
|
315
|
+
function calculateTestabilityScore(report) {
|
|
316
|
+
const { summary, rawData, recommendations } = report;
|
|
317
|
+
const factors = [
|
|
318
|
+
{
|
|
319
|
+
name: "Test Coverage",
|
|
320
|
+
impact: Math.round(summary.dimensions.testCoverageRatio - 50),
|
|
321
|
+
description: `${rawData.testFiles} test files / ${rawData.sourceFiles} source files (${Math.round(summary.coverageRatio * 100)}%)`
|
|
322
|
+
},
|
|
323
|
+
{
|
|
324
|
+
name: "Function Purity",
|
|
325
|
+
impact: Math.round(summary.dimensions.purityScore - 50),
|
|
326
|
+
description: `${rawData.pureFunctions}/${rawData.totalFunctions} functions are pure`
|
|
327
|
+
},
|
|
328
|
+
{
|
|
329
|
+
name: "Dependency Injection",
|
|
330
|
+
impact: Math.round(summary.dimensions.dependencyInjectionScore - 50),
|
|
331
|
+
description: `${rawData.injectionPatterns}/${rawData.totalClasses} classes use DI`
|
|
332
|
+
},
|
|
333
|
+
{
|
|
334
|
+
name: "Interface Focus",
|
|
335
|
+
impact: Math.round(summary.dimensions.interfaceFocusScore - 50),
|
|
336
|
+
description: `${rawData.bloatedInterfaces} interfaces have >10 methods`
|
|
337
|
+
},
|
|
338
|
+
{
|
|
339
|
+
name: "Observability",
|
|
340
|
+
impact: Math.round(summary.dimensions.observabilityScore - 50),
|
|
341
|
+
description: `${rawData.externalStateMutations} functions mutate external state`
|
|
342
|
+
}
|
|
343
|
+
];
|
|
344
|
+
const recs = recommendations.map((action) => ({
|
|
345
|
+
action,
|
|
346
|
+
estimatedImpact: summary.aiChangeSafetyRating === "blind-risk" ? 15 : 8,
|
|
347
|
+
priority: summary.aiChangeSafetyRating === "blind-risk" || summary.aiChangeSafetyRating === "high-risk" ? "high" : "medium"
|
|
348
|
+
}));
|
|
349
|
+
return {
|
|
350
|
+
toolName: "testability",
|
|
351
|
+
score: summary.score,
|
|
352
|
+
rawMetrics: {
|
|
353
|
+
...rawData,
|
|
354
|
+
rating: summary.rating,
|
|
355
|
+
aiChangeSafetyRating: summary.aiChangeSafetyRating,
|
|
356
|
+
coverageRatio: summary.coverageRatio
|
|
357
|
+
},
|
|
358
|
+
factors,
|
|
359
|
+
recommendations: recs
|
|
360
|
+
};
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
// src/cli.ts
|
|
364
|
+
var import_chalk = __toESM(require("chalk"));
|
|
365
|
+
var import_fs2 = require("fs");
|
|
366
|
+
var import_path2 = require("path");
|
|
367
|
+
var import_core2 = require("@aiready/core");
|
|
368
|
+
var program = new import_commander.Command();
|
|
369
|
+
program.name("aiready-testability").description("Measure how safely AI-generated changes can be verified in your codebase").version("0.1.0").addHelpText("after", `
|
|
370
|
+
DIMENSIONS MEASURED:
|
|
371
|
+
Test Coverage Ratio of test files to source files
|
|
372
|
+
Function Purity Pure functions are trivially AI-testable
|
|
373
|
+
Dependency Inject. DI makes classes mockable and verifiable
|
|
374
|
+
Interface Focus Small interfaces are easier to mock
|
|
375
|
+
Observability Functions returning values > mutating state
|
|
376
|
+
|
|
377
|
+
AI CHANGE SAFETY RATINGS:
|
|
378
|
+
safe \u2705 AI changes can be safely verified (\u226550% coverage + score \u226570)
|
|
379
|
+
moderate-risk \u26A0\uFE0F Some risk \u2014 partial test coverage
|
|
380
|
+
high-risk \u{1F534} Tests exist but insufficient \u2014 AI changes may slip through
|
|
381
|
+
blind-risk \u{1F480} NO TESTS \u2014 AI changes cannot be verified at all
|
|
382
|
+
|
|
383
|
+
EXAMPLES:
|
|
384
|
+
aiready-testability . # Full analysis
|
|
385
|
+
aiready-testability src/ --output json # JSON report
|
|
386
|
+
aiready-testability . --min-coverage 0.5 # Stricter 50% threshold
|
|
387
|
+
`).argument("<directory>", "Directory to analyze").option("--min-coverage <ratio>", "Minimum acceptable test/source ratio (default: 0.3)", "0.3").option("--test-patterns <patterns>", "Additional test file patterns (comma-separated)").option("--include <patterns>", "File patterns to include (comma-separated)").option("--exclude <patterns>", "File patterns to exclude (comma-separated)").option("-o, --output <format>", "Output format: console|json", "console").option("--output-file <path>", "Output file path (for json)").action(async (directory, options) => {
|
|
388
|
+
console.log(import_chalk.default.blue("\u{1F9EA} Analyzing testability...\n"));
|
|
389
|
+
const startTime = Date.now();
|
|
390
|
+
const config = await (0, import_core2.loadConfig)(directory);
|
|
391
|
+
const mergedConfig = (0, import_core2.mergeConfigWithDefaults)(config, {
|
|
392
|
+
minCoverageRatio: 0.3
|
|
393
|
+
});
|
|
394
|
+
const finalOptions = {
|
|
395
|
+
rootDir: directory,
|
|
396
|
+
minCoverageRatio: parseFloat(options.minCoverage ?? "0.3") || mergedConfig.minCoverageRatio,
|
|
397
|
+
testPatterns: options.testPatterns?.split(","),
|
|
398
|
+
include: options.include?.split(","),
|
|
399
|
+
exclude: options.exclude?.split(",")
|
|
400
|
+
};
|
|
401
|
+
const report = await analyzeTestability(finalOptions);
|
|
402
|
+
const scoring = calculateTestabilityScore(report);
|
|
403
|
+
const elapsed = ((Date.now() - startTime) / 1e3).toFixed(2);
|
|
404
|
+
if (options.output === "json") {
|
|
405
|
+
const payload = { report, score: scoring };
|
|
406
|
+
const outputPath = (0, import_core2.resolveOutputPath)(
|
|
407
|
+
options.outputFile,
|
|
408
|
+
`testability-report-${(/* @__PURE__ */ new Date()).toISOString().split("T")[0]}.json`,
|
|
409
|
+
directory
|
|
410
|
+
);
|
|
411
|
+
const dir = (0, import_path2.dirname)(outputPath);
|
|
412
|
+
if (!(0, import_fs2.existsSync)(dir)) (0, import_fs2.mkdirSync)(dir, { recursive: true });
|
|
413
|
+
(0, import_fs2.writeFileSync)(outputPath, JSON.stringify(payload, null, 2));
|
|
414
|
+
console.log(import_chalk.default.green(`\u2713 Report saved to ${outputPath}`));
|
|
415
|
+
} else {
|
|
416
|
+
displayConsoleReport(report, scoring, elapsed);
|
|
417
|
+
}
|
|
418
|
+
});
|
|
419
|
+
program.parse();
|
|
420
|
+
function safetyColor(rating) {
|
|
421
|
+
switch (rating) {
|
|
422
|
+
case "safe":
|
|
423
|
+
return import_chalk.default.green;
|
|
424
|
+
case "moderate-risk":
|
|
425
|
+
return import_chalk.default.yellow;
|
|
426
|
+
case "high-risk":
|
|
427
|
+
return import_chalk.default.red;
|
|
428
|
+
case "blind-risk":
|
|
429
|
+
return import_chalk.default.bgRed.white;
|
|
430
|
+
default:
|
|
431
|
+
return import_chalk.default.white;
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
function safetyIcon(rating) {
|
|
435
|
+
switch (rating) {
|
|
436
|
+
case "safe":
|
|
437
|
+
return "\u2705";
|
|
438
|
+
case "moderate-risk":
|
|
439
|
+
return "\u26A0\uFE0F ";
|
|
440
|
+
case "high-risk":
|
|
441
|
+
return "\u{1F534}";
|
|
442
|
+
case "blind-risk":
|
|
443
|
+
return "\u{1F480}";
|
|
444
|
+
default:
|
|
445
|
+
return "\u2753";
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
function scoreBar(val) {
|
|
449
|
+
return "\u2588".repeat(Math.round(val / 10)).padEnd(10, "\u2591");
|
|
450
|
+
}
|
|
451
|
+
function displayConsoleReport(report, scoring, elapsed) {
|
|
452
|
+
const { summary, rawData, issues, recommendations } = report;
|
|
453
|
+
const safetyRating = summary.aiChangeSafetyRating;
|
|
454
|
+
console.log(import_chalk.default.bold("\n\u{1F9EA} Testability Analysis\n"));
|
|
455
|
+
if (safetyRating === "blind-risk") {
|
|
456
|
+
console.log(import_chalk.default.bgRed.white.bold(
|
|
457
|
+
" \u{1F480} BLIND RISK \u2014 NO TESTS DETECTED. AI-GENERATED CHANGES CANNOT BE VERIFIED. "
|
|
458
|
+
));
|
|
459
|
+
console.log();
|
|
460
|
+
} else if (safetyRating === "high-risk") {
|
|
461
|
+
console.log(import_chalk.default.red.bold(` \u{1F534} HIGH RISK \u2014 Insufficient test coverage. AI changes may introduce silent bugs.`));
|
|
462
|
+
console.log();
|
|
463
|
+
}
|
|
464
|
+
console.log(`AI Change Safety: ${safetyColor(safetyRating)(`${safetyIcon(safetyRating)} ${safetyRating.toUpperCase()}`)}`);
|
|
465
|
+
console.log(`Score: ${import_chalk.default.bold(summary.score + "/100")} (${summary.rating})`);
|
|
466
|
+
console.log(`Source Files: ${import_chalk.default.cyan(rawData.sourceFiles)} Test Files: ${import_chalk.default.cyan(rawData.testFiles)}`);
|
|
467
|
+
console.log(`Coverage Ratio: ${import_chalk.default.bold(Math.round(summary.coverageRatio * 100) + "%")}`);
|
|
468
|
+
console.log(`Analysis Time: ${import_chalk.default.gray(elapsed + "s")}
|
|
469
|
+
`);
|
|
470
|
+
console.log(import_chalk.default.bold("\u{1F4D0} Dimension Scores\n"));
|
|
471
|
+
const dims = [
|
|
472
|
+
["Test Coverage", summary.dimensions.testCoverageRatio],
|
|
473
|
+
["Function Purity", summary.dimensions.purityScore],
|
|
474
|
+
["Dependency Injection", summary.dimensions.dependencyInjectionScore],
|
|
475
|
+
["Interface Focus", summary.dimensions.interfaceFocusScore],
|
|
476
|
+
["Observability", summary.dimensions.observabilityScore]
|
|
477
|
+
];
|
|
478
|
+
for (const [name, val] of dims) {
|
|
479
|
+
const color = val >= 70 ? import_chalk.default.green : val >= 50 ? import_chalk.default.yellow : import_chalk.default.red;
|
|
480
|
+
console.log(` ${name.padEnd(22)} ${color(scoreBar(val))} ${val}/100`);
|
|
481
|
+
}
|
|
482
|
+
if (issues.length > 0) {
|
|
483
|
+
console.log(import_chalk.default.bold("\n\u26A0\uFE0F Issues\n"));
|
|
484
|
+
for (const issue of issues) {
|
|
485
|
+
const sev = issue.severity === "critical" ? import_chalk.default.red : issue.severity === "major" ? import_chalk.default.yellow : import_chalk.default.blue;
|
|
486
|
+
console.log(`${sev(issue.severity.toUpperCase())} ${issue.message}`);
|
|
487
|
+
if (issue.suggestion) console.log(` ${import_chalk.default.dim("\u2192")} ${import_chalk.default.italic(issue.suggestion)}`);
|
|
488
|
+
console.log();
|
|
489
|
+
}
|
|
490
|
+
}
|
|
491
|
+
if (recommendations.length > 0) {
|
|
492
|
+
console.log(import_chalk.default.bold("\u{1F4A1} Recommendations\n"));
|
|
493
|
+
recommendations.forEach((rec, i) => {
|
|
494
|
+
console.log(`${i + 1}. ${rec}`);
|
|
495
|
+
});
|
|
496
|
+
}
|
|
497
|
+
console.log();
|
|
498
|
+
}
|
package/dist/cli.mjs
ADDED
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import {
|
|
3
|
+
analyzeTestability,
|
|
4
|
+
calculateTestabilityScore
|
|
5
|
+
} from "./chunk-CYZ7DTWN.mjs";
|
|
6
|
+
|
|
7
|
+
// src/cli.ts
|
|
8
|
+
import { Command } from "commander";
|
|
9
|
+
import chalk from "chalk";
|
|
10
|
+
import { writeFileSync, mkdirSync, existsSync } from "fs";
|
|
11
|
+
import { dirname } from "path";
|
|
12
|
+
import { loadConfig, mergeConfigWithDefaults, resolveOutputPath } from "@aiready/core";
|
|
13
|
+
var program = new Command();
|
|
14
|
+
program.name("aiready-testability").description("Measure how safely AI-generated changes can be verified in your codebase").version("0.1.0").addHelpText("after", `
|
|
15
|
+
DIMENSIONS MEASURED:
|
|
16
|
+
Test Coverage Ratio of test files to source files
|
|
17
|
+
Function Purity Pure functions are trivially AI-testable
|
|
18
|
+
Dependency Inject. DI makes classes mockable and verifiable
|
|
19
|
+
Interface Focus Small interfaces are easier to mock
|
|
20
|
+
Observability Functions returning values > mutating state
|
|
21
|
+
|
|
22
|
+
AI CHANGE SAFETY RATINGS:
|
|
23
|
+
safe \u2705 AI changes can be safely verified (\u226550% coverage + score \u226570)
|
|
24
|
+
moderate-risk \u26A0\uFE0F Some risk \u2014 partial test coverage
|
|
25
|
+
high-risk \u{1F534} Tests exist but insufficient \u2014 AI changes may slip through
|
|
26
|
+
blind-risk \u{1F480} NO TESTS \u2014 AI changes cannot be verified at all
|
|
27
|
+
|
|
28
|
+
EXAMPLES:
|
|
29
|
+
aiready-testability . # Full analysis
|
|
30
|
+
aiready-testability src/ --output json # JSON report
|
|
31
|
+
aiready-testability . --min-coverage 0.5 # Stricter 50% threshold
|
|
32
|
+
`).argument("<directory>", "Directory to analyze").option("--min-coverage <ratio>", "Minimum acceptable test/source ratio (default: 0.3)", "0.3").option("--test-patterns <patterns>", "Additional test file patterns (comma-separated)").option("--include <patterns>", "File patterns to include (comma-separated)").option("--exclude <patterns>", "File patterns to exclude (comma-separated)").option("-o, --output <format>", "Output format: console|json", "console").option("--output-file <path>", "Output file path (for json)").action(async (directory, options) => {
|
|
33
|
+
console.log(chalk.blue("\u{1F9EA} Analyzing testability...\n"));
|
|
34
|
+
const startTime = Date.now();
|
|
35
|
+
const config = await loadConfig(directory);
|
|
36
|
+
const mergedConfig = mergeConfigWithDefaults(config, {
|
|
37
|
+
minCoverageRatio: 0.3
|
|
38
|
+
});
|
|
39
|
+
const finalOptions = {
|
|
40
|
+
rootDir: directory,
|
|
41
|
+
minCoverageRatio: parseFloat(options.minCoverage ?? "0.3") || mergedConfig.minCoverageRatio,
|
|
42
|
+
testPatterns: options.testPatterns?.split(","),
|
|
43
|
+
include: options.include?.split(","),
|
|
44
|
+
exclude: options.exclude?.split(",")
|
|
45
|
+
};
|
|
46
|
+
const report = await analyzeTestability(finalOptions);
|
|
47
|
+
const scoring = calculateTestabilityScore(report);
|
|
48
|
+
const elapsed = ((Date.now() - startTime) / 1e3).toFixed(2);
|
|
49
|
+
if (options.output === "json") {
|
|
50
|
+
const payload = { report, score: scoring };
|
|
51
|
+
const outputPath = resolveOutputPath(
|
|
52
|
+
options.outputFile,
|
|
53
|
+
`testability-report-${(/* @__PURE__ */ new Date()).toISOString().split("T")[0]}.json`,
|
|
54
|
+
directory
|
|
55
|
+
);
|
|
56
|
+
const dir = dirname(outputPath);
|
|
57
|
+
if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
|
|
58
|
+
writeFileSync(outputPath, JSON.stringify(payload, null, 2));
|
|
59
|
+
console.log(chalk.green(`\u2713 Report saved to ${outputPath}`));
|
|
60
|
+
} else {
|
|
61
|
+
displayConsoleReport(report, scoring, elapsed);
|
|
62
|
+
}
|
|
63
|
+
});
|
|
64
|
+
program.parse();
|
|
65
|
+
function safetyColor(rating) {
|
|
66
|
+
switch (rating) {
|
|
67
|
+
case "safe":
|
|
68
|
+
return chalk.green;
|
|
69
|
+
case "moderate-risk":
|
|
70
|
+
return chalk.yellow;
|
|
71
|
+
case "high-risk":
|
|
72
|
+
return chalk.red;
|
|
73
|
+
case "blind-risk":
|
|
74
|
+
return chalk.bgRed.white;
|
|
75
|
+
default:
|
|
76
|
+
return chalk.white;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
function safetyIcon(rating) {
|
|
80
|
+
switch (rating) {
|
|
81
|
+
case "safe":
|
|
82
|
+
return "\u2705";
|
|
83
|
+
case "moderate-risk":
|
|
84
|
+
return "\u26A0\uFE0F ";
|
|
85
|
+
case "high-risk":
|
|
86
|
+
return "\u{1F534}";
|
|
87
|
+
case "blind-risk":
|
|
88
|
+
return "\u{1F480}";
|
|
89
|
+
default:
|
|
90
|
+
return "\u2753";
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
function scoreBar(val) {
|
|
94
|
+
return "\u2588".repeat(Math.round(val / 10)).padEnd(10, "\u2591");
|
|
95
|
+
}
|
|
96
|
+
function displayConsoleReport(report, scoring, elapsed) {
|
|
97
|
+
const { summary, rawData, issues, recommendations } = report;
|
|
98
|
+
const safetyRating = summary.aiChangeSafetyRating;
|
|
99
|
+
console.log(chalk.bold("\n\u{1F9EA} Testability Analysis\n"));
|
|
100
|
+
if (safetyRating === "blind-risk") {
|
|
101
|
+
console.log(chalk.bgRed.white.bold(
|
|
102
|
+
" \u{1F480} BLIND RISK \u2014 NO TESTS DETECTED. AI-GENERATED CHANGES CANNOT BE VERIFIED. "
|
|
103
|
+
));
|
|
104
|
+
console.log();
|
|
105
|
+
} else if (safetyRating === "high-risk") {
|
|
106
|
+
console.log(chalk.red.bold(` \u{1F534} HIGH RISK \u2014 Insufficient test coverage. AI changes may introduce silent bugs.`));
|
|
107
|
+
console.log();
|
|
108
|
+
}
|
|
109
|
+
console.log(`AI Change Safety: ${safetyColor(safetyRating)(`${safetyIcon(safetyRating)} ${safetyRating.toUpperCase()}`)}`);
|
|
110
|
+
console.log(`Score: ${chalk.bold(summary.score + "/100")} (${summary.rating})`);
|
|
111
|
+
console.log(`Source Files: ${chalk.cyan(rawData.sourceFiles)} Test Files: ${chalk.cyan(rawData.testFiles)}`);
|
|
112
|
+
console.log(`Coverage Ratio: ${chalk.bold(Math.round(summary.coverageRatio * 100) + "%")}`);
|
|
113
|
+
console.log(`Analysis Time: ${chalk.gray(elapsed + "s")}
|
|
114
|
+
`);
|
|
115
|
+
console.log(chalk.bold("\u{1F4D0} Dimension Scores\n"));
|
|
116
|
+
const dims = [
|
|
117
|
+
["Test Coverage", summary.dimensions.testCoverageRatio],
|
|
118
|
+
["Function Purity", summary.dimensions.purityScore],
|
|
119
|
+
["Dependency Injection", summary.dimensions.dependencyInjectionScore],
|
|
120
|
+
["Interface Focus", summary.dimensions.interfaceFocusScore],
|
|
121
|
+
["Observability", summary.dimensions.observabilityScore]
|
|
122
|
+
];
|
|
123
|
+
for (const [name, val] of dims) {
|
|
124
|
+
const color = val >= 70 ? chalk.green : val >= 50 ? chalk.yellow : chalk.red;
|
|
125
|
+
console.log(` ${name.padEnd(22)} ${color(scoreBar(val))} ${val}/100`);
|
|
126
|
+
}
|
|
127
|
+
if (issues.length > 0) {
|
|
128
|
+
console.log(chalk.bold("\n\u26A0\uFE0F Issues\n"));
|
|
129
|
+
for (const issue of issues) {
|
|
130
|
+
const sev = issue.severity === "critical" ? chalk.red : issue.severity === "major" ? chalk.yellow : chalk.blue;
|
|
131
|
+
console.log(`${sev(issue.severity.toUpperCase())} ${issue.message}`);
|
|
132
|
+
if (issue.suggestion) console.log(` ${chalk.dim("\u2192")} ${chalk.italic(issue.suggestion)}`);
|
|
133
|
+
console.log();
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
if (recommendations.length > 0) {
|
|
137
|
+
console.log(chalk.bold("\u{1F4A1} Recommendations\n"));
|
|
138
|
+
recommendations.forEach((rec, i) => {
|
|
139
|
+
console.log(`${i + 1}. ${rec}`);
|
|
140
|
+
});
|
|
141
|
+
}
|
|
142
|
+
console.log();
|
|
143
|
+
}
|