@rigour-labs/core 2.21.2 → 2.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/gates/base.d.ts +2 -2
- package/dist/gates/base.js +2 -1
- package/dist/gates/content.js +1 -1
- package/dist/gates/context-window-artifacts.d.ts +33 -0
- package/dist/gates/context-window-artifacts.js +211 -0
- package/dist/gates/context.js +3 -3
- package/dist/gates/duplication-drift.d.ts +32 -0
- package/dist/gates/duplication-drift.js +187 -0
- package/dist/gates/file.js +1 -1
- package/dist/gates/hallucinated-imports.d.ts +44 -0
- package/dist/gates/hallucinated-imports.js +292 -0
- package/dist/gates/inconsistent-error-handling.d.ts +38 -0
- package/dist/gates/inconsistent-error-handling.js +222 -0
- package/dist/gates/runner.js +29 -1
- package/dist/gates/security-patterns.js +1 -1
- package/dist/templates/index.js +26 -0
- package/dist/types/fix-packet.d.ts +4 -4
- package/dist/types/index.d.ts +277 -0
- package/dist/types/index.js +38 -0
- package/package.json +1 -1
- package/src/gates/base.ts +3 -2
- package/src/gates/content.ts +2 -1
- package/src/gates/context-window-artifacts.ts +277 -0
- package/src/gates/context.ts +6 -3
- package/src/gates/duplication-drift.ts +231 -0
- package/src/gates/file.ts +5 -1
- package/src/gates/hallucinated-imports.ts +361 -0
- package/src/gates/inconsistent-error-handling.ts +254 -0
- package/src/gates/runner.ts +34 -2
- package/src/gates/security-patterns.ts +2 -1
- package/src/templates/index.ts +26 -0
- package/src/types/index.ts +41 -0
|
@@ -0,0 +1,361 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Hallucinated Imports Gate
|
|
3
|
+
*
|
|
4
|
+
* Detects imports that reference modules which don't exist in the project.
|
|
5
|
+
* This is an AI-specific failure mode — LLMs confidently generate import
|
|
6
|
+
* statements for packages, files, or modules that were never installed
|
|
7
|
+
* or created.
|
|
8
|
+
*
|
|
9
|
+
* Detection strategy:
|
|
10
|
+
* 1. Parse all import/require statements
|
|
11
|
+
* 2. For relative imports: verify the target file exists
|
|
12
|
+
* 3. For package imports: verify the package exists in node_modules or package.json
|
|
13
|
+
* 4. For Python imports: verify the module exists in the project or site-packages
|
|
14
|
+
*
|
|
15
|
+
* @since v2.16.0
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
import { Gate, GateContext } from './base.js';
|
|
19
|
+
import { Failure } from '../types/index.js';
|
|
20
|
+
import { FileScanner } from '../utils/scanner.js';
|
|
21
|
+
import { Logger } from '../utils/logger.js';
|
|
22
|
+
import fs from 'fs-extra';
|
|
23
|
+
import path from 'path';
|
|
24
|
+
|
|
25
|
+
export interface HallucinatedImport {
|
|
26
|
+
file: string;
|
|
27
|
+
line: number;
|
|
28
|
+
importPath: string;
|
|
29
|
+
type: 'relative' | 'package' | 'python';
|
|
30
|
+
reason: string;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export interface HallucinatedImportsConfig {
|
|
34
|
+
enabled?: boolean;
|
|
35
|
+
check_relative?: boolean; // Check relative imports resolve to real files
|
|
36
|
+
check_packages?: boolean; // Check npm/pip packages exist
|
|
37
|
+
ignore_patterns?: string[]; // Import patterns to ignore (e.g. asset imports)
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export class HallucinatedImportsGate extends Gate {
|
|
41
|
+
private config: Required<Omit<HallucinatedImportsConfig, 'ignore_patterns'>> & { ignore_patterns: string[] };
|
|
42
|
+
|
|
43
|
+
constructor(config: HallucinatedImportsConfig = {}) {
|
|
44
|
+
super('hallucinated-imports', 'Hallucinated Import Detection');
|
|
45
|
+
this.config = {
|
|
46
|
+
enabled: config.enabled ?? true,
|
|
47
|
+
check_relative: config.check_relative ?? true,
|
|
48
|
+
check_packages: config.check_packages ?? true,
|
|
49
|
+
ignore_patterns: config.ignore_patterns ?? [
|
|
50
|
+
'\\.css$', '\\.scss$', '\\.less$', '\\.svg$', '\\.png$', '\\.jpg$',
|
|
51
|
+
'\\.json$', '\\.wasm$', '\\.graphql$', '\\.gql$',
|
|
52
|
+
],
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
async run(context: GateContext): Promise<Failure[]> {
|
|
57
|
+
if (!this.config.enabled) return [];
|
|
58
|
+
|
|
59
|
+
const failures: Failure[] = [];
|
|
60
|
+
const hallucinated: HallucinatedImport[] = [];
|
|
61
|
+
|
|
62
|
+
const files = await FileScanner.findFiles({
|
|
63
|
+
cwd: context.cwd,
|
|
64
|
+
patterns: ['**/*.{ts,js,tsx,jsx,py}'],
|
|
65
|
+
ignore: [...(context.ignore || []), '**/node_modules/**', '**/dist/**', '**/build/**', '**/.venv/**'],
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
Logger.info(`Hallucinated Imports: Scanning ${files.length} files`);
|
|
69
|
+
|
|
70
|
+
// Build lookup sets for fast resolution
|
|
71
|
+
const projectFiles = new Set(files.map(f => f.replace(/\\/g, '/')));
|
|
72
|
+
const packageJson = await this.loadPackageJson(context.cwd);
|
|
73
|
+
const allDeps = new Set([
|
|
74
|
+
...Object.keys(packageJson?.dependencies || {}),
|
|
75
|
+
...Object.keys(packageJson?.devDependencies || {}),
|
|
76
|
+
...Object.keys(packageJson?.peerDependencies || {}),
|
|
77
|
+
]);
|
|
78
|
+
|
|
79
|
+
// Check if node_modules exists (for package verification)
|
|
80
|
+
const hasNodeModules = await fs.pathExists(path.join(context.cwd, 'node_modules'));
|
|
81
|
+
|
|
82
|
+
for (const file of files) {
|
|
83
|
+
try {
|
|
84
|
+
const fullPath = path.join(context.cwd, file);
|
|
85
|
+
const content = await fs.readFile(fullPath, 'utf-8');
|
|
86
|
+
const ext = path.extname(file);
|
|
87
|
+
|
|
88
|
+
if (['.ts', '.js', '.tsx', '.jsx'].includes(ext)) {
|
|
89
|
+
await this.checkJSImports(content, file, context.cwd, projectFiles, allDeps, hasNodeModules, hallucinated);
|
|
90
|
+
} else if (ext === '.py') {
|
|
91
|
+
await this.checkPyImports(content, file, context.cwd, projectFiles, hallucinated);
|
|
92
|
+
}
|
|
93
|
+
} catch (e) { }
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
// Group hallucinated imports by file for cleaner output
|
|
97
|
+
const byFile = new Map<string, HallucinatedImport[]>();
|
|
98
|
+
for (const h of hallucinated) {
|
|
99
|
+
const existing = byFile.get(h.file) || [];
|
|
100
|
+
existing.push(h);
|
|
101
|
+
byFile.set(h.file, existing);
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
for (const [file, imports] of byFile) {
|
|
105
|
+
const details = imports.map(i => ` L${i.line}: import '${i.importPath}' — ${i.reason}`).join('\n');
|
|
106
|
+
|
|
107
|
+
failures.push(this.createFailure(
|
|
108
|
+
`Hallucinated imports in ${file}:\n${details}`,
|
|
109
|
+
[file],
|
|
110
|
+
`These imports reference modules that don't exist. Remove or replace with real modules. AI models often "hallucinate" package names or file paths.`,
|
|
111
|
+
'Hallucinated Imports',
|
|
112
|
+
imports[0].line,
|
|
113
|
+
undefined,
|
|
114
|
+
'critical'
|
|
115
|
+
));
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
return failures;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
private async checkJSImports(
|
|
122
|
+
content: string,
|
|
123
|
+
file: string,
|
|
124
|
+
cwd: string,
|
|
125
|
+
projectFiles: Set<string>,
|
|
126
|
+
allDeps: Set<string>,
|
|
127
|
+
hasNodeModules: boolean,
|
|
128
|
+
hallucinated: HallucinatedImport[]
|
|
129
|
+
): Promise<void> {
|
|
130
|
+
const lines = content.split('\n');
|
|
131
|
+
|
|
132
|
+
// Match: import ... from '...', require('...'), import('...')
|
|
133
|
+
const importPatterns = [
|
|
134
|
+
/import\s+(?:{[^}]*}|\*\s+as\s+\w+|\w+(?:\s*,\s*{[^}]*})?)\s+from\s+['"]([^'"]+)['"]/g,
|
|
135
|
+
/require\s*\(\s*['"]([^'"]+)['"]\s*\)/g,
|
|
136
|
+
/import\s*\(\s*['"]([^'"]+)['"]\s*\)/g,
|
|
137
|
+
/export\s+(?:{[^}]*}|\*)\s+from\s+['"]([^'"]+)['"]/g,
|
|
138
|
+
];
|
|
139
|
+
|
|
140
|
+
for (let i = 0; i < lines.length; i++) {
|
|
141
|
+
const line = lines[i];
|
|
142
|
+
|
|
143
|
+
for (const pattern of importPatterns) {
|
|
144
|
+
pattern.lastIndex = 0;
|
|
145
|
+
let match;
|
|
146
|
+
while ((match = pattern.exec(line)) !== null) {
|
|
147
|
+
const importPath = match[1];
|
|
148
|
+
|
|
149
|
+
// Skip ignored patterns (assets, etc.)
|
|
150
|
+
if (this.shouldIgnore(importPath)) continue;
|
|
151
|
+
|
|
152
|
+
if (importPath.startsWith('.')) {
|
|
153
|
+
// Relative import — check file exists
|
|
154
|
+
if (this.config.check_relative) {
|
|
155
|
+
const resolved = this.resolveRelativeImport(file, importPath, projectFiles);
|
|
156
|
+
if (!resolved) {
|
|
157
|
+
hallucinated.push({
|
|
158
|
+
file, line: i + 1, importPath, type: 'relative',
|
|
159
|
+
reason: `File not found: ${importPath}`,
|
|
160
|
+
});
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
} else {
|
|
164
|
+
// Package import — check it exists
|
|
165
|
+
if (this.config.check_packages) {
|
|
166
|
+
const pkgName = this.extractPackageName(importPath);
|
|
167
|
+
|
|
168
|
+
// Skip Node.js built-ins
|
|
169
|
+
if (this.isNodeBuiltin(pkgName)) continue;
|
|
170
|
+
|
|
171
|
+
if (!allDeps.has(pkgName)) {
|
|
172
|
+
// Double-check node_modules if available
|
|
173
|
+
if (hasNodeModules) {
|
|
174
|
+
const pkgPath = path.join(cwd, 'node_modules', pkgName);
|
|
175
|
+
if (await fs.pathExists(pkgPath)) continue;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
hallucinated.push({
|
|
179
|
+
file, line: i + 1, importPath, type: 'package',
|
|
180
|
+
reason: `Package '${pkgName}' not in package.json dependencies`,
|
|
181
|
+
});
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
private async checkPyImports(
|
|
191
|
+
content: string,
|
|
192
|
+
file: string,
|
|
193
|
+
cwd: string,
|
|
194
|
+
projectFiles: Set<string>,
|
|
195
|
+
hallucinated: HallucinatedImport[]
|
|
196
|
+
): Promise<void> {
|
|
197
|
+
const lines = content.split('\n');
|
|
198
|
+
|
|
199
|
+
for (let i = 0; i < lines.length; i++) {
|
|
200
|
+
const line = lines[i].trim();
|
|
201
|
+
|
|
202
|
+
// Match: from X import Y, import X
|
|
203
|
+
const fromMatch = line.match(/^from\s+([\w.]+)\s+import/);
|
|
204
|
+
const importMatch = line.match(/^import\s+([\w.]+)/);
|
|
205
|
+
|
|
206
|
+
const modulePath = fromMatch?.[1] || importMatch?.[1];
|
|
207
|
+
if (!modulePath) continue;
|
|
208
|
+
|
|
209
|
+
// Skip standard library modules
|
|
210
|
+
if (this.isPythonStdlib(modulePath)) continue;
|
|
211
|
+
|
|
212
|
+
// Check if it's a relative project import
|
|
213
|
+
if (modulePath.startsWith('.')) {
|
|
214
|
+
// Relative Python import
|
|
215
|
+
const pyFile = modulePath.replace(/\./g, '/') + '.py';
|
|
216
|
+
const pyInit = modulePath.replace(/\./g, '/') + '/__init__.py';
|
|
217
|
+
const fileDir = path.dirname(file);
|
|
218
|
+
const resolved1 = path.join(fileDir, pyFile).replace(/\\/g, '/');
|
|
219
|
+
const resolved2 = path.join(fileDir, pyInit).replace(/\\/g, '/');
|
|
220
|
+
|
|
221
|
+
if (!projectFiles.has(resolved1) && !projectFiles.has(resolved2)) {
|
|
222
|
+
hallucinated.push({
|
|
223
|
+
file, line: i + 1, importPath: modulePath, type: 'python',
|
|
224
|
+
reason: `Relative module '${modulePath}' not found in project`,
|
|
225
|
+
});
|
|
226
|
+
}
|
|
227
|
+
} else {
|
|
228
|
+
// Absolute import — check if it's a project module
|
|
229
|
+
const topLevel = modulePath.split('.')[0];
|
|
230
|
+
const pyFile = topLevel + '.py';
|
|
231
|
+
const pyInit = topLevel + '/__init__.py';
|
|
232
|
+
|
|
233
|
+
// If it matches a project file, it's a local import — verify it exists
|
|
234
|
+
const isLocalModule = projectFiles.has(pyFile) || projectFiles.has(pyInit) ||
|
|
235
|
+
[...projectFiles].some(f => f.startsWith(topLevel + '/'));
|
|
236
|
+
|
|
237
|
+
// If not local and not stdlib, we can't easily verify pip packages
|
|
238
|
+
// without a requirements.txt or pyproject.toml check
|
|
239
|
+
if (isLocalModule) {
|
|
240
|
+
// It's referencing a local module — verify the full path
|
|
241
|
+
const fullModulePath = modulePath.replace(/\./g, '/');
|
|
242
|
+
const candidates = [
|
|
243
|
+
fullModulePath + '.py',
|
|
244
|
+
fullModulePath + '/__init__.py',
|
|
245
|
+
];
|
|
246
|
+
const exists = candidates.some(c => projectFiles.has(c));
|
|
247
|
+
if (!exists && modulePath.includes('.')) {
|
|
248
|
+
// Only flag deep module paths that partially resolve
|
|
249
|
+
hallucinated.push({
|
|
250
|
+
file, line: i + 1, importPath: modulePath, type: 'python',
|
|
251
|
+
reason: `Module '${modulePath}' partially resolves but target not found`,
|
|
252
|
+
});
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
private resolveRelativeImport(fromFile: string, importPath: string, projectFiles: Set<string>): boolean {
|
|
260
|
+
const dir = path.dirname(fromFile);
|
|
261
|
+
const resolved = path.join(dir, importPath).replace(/\\/g, '/');
|
|
262
|
+
|
|
263
|
+
// Try exact match, then common extensions
|
|
264
|
+
const extensions = ['', '.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs'];
|
|
265
|
+
const indexFiles = extensions.map(ext => `${resolved}/index${ext}`);
|
|
266
|
+
|
|
267
|
+
const candidates = [
|
|
268
|
+
...extensions.map(ext => resolved + ext),
|
|
269
|
+
...indexFiles,
|
|
270
|
+
];
|
|
271
|
+
|
|
272
|
+
return candidates.some(c => projectFiles.has(c));
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
private extractPackageName(importPath: string): string {
|
|
276
|
+
// Scoped packages: @scope/package/... → @scope/package
|
|
277
|
+
if (importPath.startsWith('@')) {
|
|
278
|
+
const parts = importPath.split('/');
|
|
279
|
+
return parts.length >= 2 ? `${parts[0]}/${parts[1]}` : importPath;
|
|
280
|
+
}
|
|
281
|
+
// Regular packages: package/... → package
|
|
282
|
+
return importPath.split('/')[0];
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
private shouldIgnore(importPath: string): boolean {
|
|
286
|
+
return this.config.ignore_patterns.some(pattern => new RegExp(pattern).test(importPath));
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
private isNodeBuiltin(name: string): boolean {
|
|
290
|
+
const builtins = new Set([
|
|
291
|
+
'assert', 'buffer', 'child_process', 'cluster', 'console', 'constants',
|
|
292
|
+
'crypto', 'dgram', 'dns', 'domain', 'events', 'fs', 'http', 'http2',
|
|
293
|
+
'https', 'inspector', 'module', 'net', 'os', 'path', 'perf_hooks',
|
|
294
|
+
'process', 'punycode', 'querystring', 'readline', 'repl', 'stream',
|
|
295
|
+
'string_decoder', 'sys', 'timers', 'tls', 'trace_events', 'tty',
|
|
296
|
+
'url', 'util', 'v8', 'vm', 'wasi', 'worker_threads', 'zlib',
|
|
297
|
+
'node:assert', 'node:buffer', 'node:child_process', 'node:cluster',
|
|
298
|
+
'node:console', 'node:constants', 'node:crypto', 'node:dgram',
|
|
299
|
+
'node:dns', 'node:domain', 'node:events', 'node:fs', 'node:http',
|
|
300
|
+
'node:http2', 'node:https', 'node:inspector', 'node:module', 'node:net',
|
|
301
|
+
'node:os', 'node:path', 'node:perf_hooks', 'node:process',
|
|
302
|
+
'node:punycode', 'node:querystring', 'node:readline', 'node:repl',
|
|
303
|
+
'node:stream', 'node:string_decoder', 'node:sys', 'node:timers',
|
|
304
|
+
'node:tls', 'node:trace_events', 'node:tty', 'node:url', 'node:util',
|
|
305
|
+
'node:v8', 'node:vm', 'node:wasi', 'node:worker_threads', 'node:zlib',
|
|
306
|
+
'fs-extra', // common enough to skip
|
|
307
|
+
]);
|
|
308
|
+
return builtins.has(name) || name.startsWith('node:');
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
private isPythonStdlib(modulePath: string): boolean {
|
|
312
|
+
const topLevel = modulePath.split('.')[0];
|
|
313
|
+
const stdlibs = new Set([
|
|
314
|
+
'abc', 'aifc', 'argparse', 'array', 'ast', 'asyncio', 'atexit',
|
|
315
|
+
'base64', 'bdb', 'binascii', 'binhex', 'bisect', 'builtins',
|
|
316
|
+
'calendar', 'cgi', 'cgitb', 'chunk', 'cmath', 'cmd', 'code',
|
|
317
|
+
'codecs', 'codeop', 'collections', 'colorsys', 'compileall',
|
|
318
|
+
'concurrent', 'configparser', 'contextlib', 'contextvars', 'copy',
|
|
319
|
+
'copyreg', 'cProfile', 'csv', 'ctypes', 'curses', 'dataclasses',
|
|
320
|
+
'datetime', 'dbm', 'decimal', 'difflib', 'dis', 'distutils',
|
|
321
|
+
'doctest', 'email', 'encodings', 'enum', 'errno', 'faulthandler',
|
|
322
|
+
'fcntl', 'filecmp', 'fileinput', 'fnmatch', 'fractions', 'ftplib',
|
|
323
|
+
'functools', 'gc', 'getopt', 'getpass', 'gettext', 'glob', 'grp',
|
|
324
|
+
'gzip', 'hashlib', 'heapq', 'hmac', 'html', 'http', 'idlelib',
|
|
325
|
+
'imaplib', 'imghdr', 'importlib', 'inspect', 'io', 'ipaddress',
|
|
326
|
+
'itertools', 'json', 'keyword', 'lib2to3', 'linecache', 'locale',
|
|
327
|
+
'logging', 'lzma', 'mailbox', 'mailcap', 'marshal', 'math',
|
|
328
|
+
'mimetypes', 'mmap', 'modulefinder', 'multiprocessing', 'netrc',
|
|
329
|
+
'nis', 'nntplib', 'numbers', 'operator', 'optparse', 'os',
|
|
330
|
+
'ossaudiodev', 'parser', 'pathlib', 'pdb', 'pickle', 'pickletools',
|
|
331
|
+
'pipes', 'pkgutil', 'platform', 'plistlib', 'poplib', 'posix',
|
|
332
|
+
'posixpath', 'pprint', 'profile', 'pstats', 'pty', 'pwd', 'py_compile',
|
|
333
|
+
'pyclbr', 'pydoc', 'queue', 'quopri', 'random', 're', 'readline',
|
|
334
|
+
'reprlib', 'resource', 'rlcompleter', 'runpy', 'sched', 'secrets',
|
|
335
|
+
'select', 'selectors', 'shelve', 'shlex', 'shutil', 'signal',
|
|
336
|
+
'site', 'smtpd', 'smtplib', 'sndhdr', 'socket', 'socketserver',
|
|
337
|
+
'spwd', 'sqlite3', 'sre_compile', 'sre_constants', 'sre_parse',
|
|
338
|
+
'ssl', 'stat', 'statistics', 'string', 'stringprep', 'struct',
|
|
339
|
+
'subprocess', 'sunau', 'symtable', 'sys', 'sysconfig', 'syslog',
|
|
340
|
+
'tabnanny', 'tarfile', 'telnetlib', 'tempfile', 'termios', 'test',
|
|
341
|
+
'textwrap', 'threading', 'time', 'timeit', 'tkinter', 'token',
|
|
342
|
+
'tokenize', 'tomllib', 'trace', 'traceback', 'tracemalloc', 'tty',
|
|
343
|
+
'turtle', 'turtledemo', 'types', 'typing', 'unicodedata', 'unittest',
|
|
344
|
+
'urllib', 'uu', 'uuid', 'venv', 'warnings', 'wave', 'weakref',
|
|
345
|
+
'webbrowser', 'winreg', 'winsound', 'wsgiref', 'xdrlib', 'xml',
|
|
346
|
+
'xmlrpc', 'zipapp', 'zipfile', 'zipimport', 'zlib',
|
|
347
|
+
'_thread', '__future__', '__main__',
|
|
348
|
+
]);
|
|
349
|
+
return stdlibs.has(topLevel);
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
private async loadPackageJson(cwd: string): Promise<any> {
|
|
353
|
+
try {
|
|
354
|
+
const pkgPath = path.join(cwd, 'package.json');
|
|
355
|
+
if (await fs.pathExists(pkgPath)) {
|
|
356
|
+
return await fs.readJson(pkgPath);
|
|
357
|
+
}
|
|
358
|
+
} catch (e) { }
|
|
359
|
+
return null;
|
|
360
|
+
}
|
|
361
|
+
}
|
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Inconsistent Error Handling Gate
|
|
3
|
+
*
|
|
4
|
+
* Detects when the same error type is handled differently across the codebase.
|
|
5
|
+
* This is an AI-specific failure mode — each agent session writes error handling
|
|
6
|
+
* from scratch, leading to 4 different patterns for the same error type.
|
|
7
|
+
*
|
|
8
|
+
* Detection strategy:
|
|
9
|
+
* 1. Extract all try-catch blocks and error handling patterns
|
|
10
|
+
* 2. Cluster by caught error type (e.g. "Error", "TypeError", custom errors)
|
|
11
|
+
* 3. Compare handling strategies within each cluster
|
|
12
|
+
* 4. Flag types with >2 distinct handling patterns across files
|
|
13
|
+
*
|
|
14
|
+
* Examples of inconsistency:
|
|
15
|
+
* - File A: catch(e) { console.log(e) }
|
|
16
|
+
* - File B: catch(e) { throw new AppError(e) }
|
|
17
|
+
* - File C: catch(e) { return null }
|
|
18
|
+
* - File D: catch(e) { [empty] }
|
|
19
|
+
*
|
|
20
|
+
* @since v2.16.0
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
import { Gate, GateContext } from './base.js';
|
|
24
|
+
import { Failure } from '../types/index.js';
|
|
25
|
+
import { FileScanner } from '../utils/scanner.js';
|
|
26
|
+
import { Logger } from '../utils/logger.js';
|
|
27
|
+
import fs from 'fs-extra';
|
|
28
|
+
import path from 'path';
|
|
29
|
+
|
|
30
|
+
interface ErrorHandler {
|
|
31
|
+
file: string;
|
|
32
|
+
line: number;
|
|
33
|
+
errorType: string; // What's caught: 'Error', 'TypeError', 'any', etc.
|
|
34
|
+
strategy: string; // Classified handling strategy
|
|
35
|
+
rawPattern: string; // First line of catch body for display
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export interface InconsistentErrorHandlingConfig {
|
|
39
|
+
enabled?: boolean;
|
|
40
|
+
max_strategies_per_type?: number; // Flag if more than N strategies, default 2
|
|
41
|
+
min_occurrences?: number; // Minimum catch blocks to analyze, default 3
|
|
42
|
+
ignore_empty_catches?: boolean; // Whether to count empty catches as a strategy
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export class InconsistentErrorHandlingGate extends Gate {
|
|
46
|
+
private config: Required<InconsistentErrorHandlingConfig>;
|
|
47
|
+
|
|
48
|
+
constructor(config: InconsistentErrorHandlingConfig = {}) {
|
|
49
|
+
super('inconsistent-error-handling', 'Inconsistent Error Handling Detection');
|
|
50
|
+
this.config = {
|
|
51
|
+
enabled: config.enabled ?? true,
|
|
52
|
+
max_strategies_per_type: config.max_strategies_per_type ?? 2,
|
|
53
|
+
min_occurrences: config.min_occurrences ?? 3,
|
|
54
|
+
ignore_empty_catches: config.ignore_empty_catches ?? false,
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
async run(context: GateContext): Promise<Failure[]> {
|
|
59
|
+
if (!this.config.enabled) return [];
|
|
60
|
+
|
|
61
|
+
const failures: Failure[] = [];
|
|
62
|
+
const handlers: ErrorHandler[] = [];
|
|
63
|
+
|
|
64
|
+
const files = await FileScanner.findFiles({
|
|
65
|
+
cwd: context.cwd,
|
|
66
|
+
patterns: ['**/*.{ts,js,tsx,jsx}'],
|
|
67
|
+
ignore: [...(context.ignore || []), '**/node_modules/**', '**/dist/**', '**/*.test.*', '**/*.spec.*'],
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
Logger.info(`Inconsistent Error Handling: Scanning ${files.length} files`);
|
|
71
|
+
|
|
72
|
+
for (const file of files) {
|
|
73
|
+
try {
|
|
74
|
+
const content = await fs.readFile(path.join(context.cwd, file), 'utf-8');
|
|
75
|
+
this.extractErrorHandlers(content, file, handlers);
|
|
76
|
+
} catch (e) { }
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
// Group by error type
|
|
80
|
+
const byType = new Map<string, ErrorHandler[]>();
|
|
81
|
+
for (const handler of handlers) {
|
|
82
|
+
const existing = byType.get(handler.errorType) || [];
|
|
83
|
+
existing.push(handler);
|
|
84
|
+
byType.set(handler.errorType, existing);
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
// Analyze each error type for inconsistent handling
|
|
88
|
+
for (const [errorType, typeHandlers] of byType) {
|
|
89
|
+
if (typeHandlers.length < this.config.min_occurrences) continue;
|
|
90
|
+
|
|
91
|
+
// Count unique strategies
|
|
92
|
+
const strategies = new Map<string, ErrorHandler[]>();
|
|
93
|
+
for (const handler of typeHandlers) {
|
|
94
|
+
const existing = strategies.get(handler.strategy) || [];
|
|
95
|
+
existing.push(handler);
|
|
96
|
+
strategies.set(handler.strategy, existing);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
// Filter out empty catches if configured
|
|
100
|
+
const activeStrategies = this.config.ignore_empty_catches
|
|
101
|
+
? new Map([...strategies].filter(([k]) => k !== 'swallow'))
|
|
102
|
+
: strategies;
|
|
103
|
+
|
|
104
|
+
if (activeStrategies.size > this.config.max_strategies_per_type) {
|
|
105
|
+
// Only flag if handlers span multiple files
|
|
106
|
+
const uniqueFiles = new Set(typeHandlers.map(h => h.file));
|
|
107
|
+
if (uniqueFiles.size < 2) continue;
|
|
108
|
+
|
|
109
|
+
const strategyBreakdown = [...activeStrategies.entries()]
|
|
110
|
+
.map(([strategy, handlers]) => {
|
|
111
|
+
const files = [...new Set(handlers.map(h => h.file))].slice(0, 3);
|
|
112
|
+
return ` • ${strategy} (${handlers.length}x): ${files.join(', ')}`;
|
|
113
|
+
})
|
|
114
|
+
.join('\n');
|
|
115
|
+
|
|
116
|
+
failures.push(this.createFailure(
|
|
117
|
+
`Inconsistent error handling for '${errorType}': ${activeStrategies.size} different strategies found across ${uniqueFiles.size} files:\n${strategyBreakdown}`,
|
|
118
|
+
[...uniqueFiles].slice(0, 5),
|
|
119
|
+
`Standardize error handling for '${errorType}'. Create a shared error handler or establish a project convention. AI agents often write error handling from scratch each session, leading to divergent patterns.`,
|
|
120
|
+
'Inconsistent Error Handling',
|
|
121
|
+
typeHandlers[0].line,
|
|
122
|
+
undefined,
|
|
123
|
+
'high'
|
|
124
|
+
));
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
return failures;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
private extractErrorHandlers(content: string, file: string, handlers: ErrorHandler[]): void {
|
|
132
|
+
const lines = content.split('\n');
|
|
133
|
+
|
|
134
|
+
for (let i = 0; i < lines.length; i++) {
|
|
135
|
+
// Match catch clauses: catch (e), catch (error: Error), catch (e: TypeError)
|
|
136
|
+
const catchMatch = lines[i].match(/\bcatch\s*\(\s*(\w+)(?:\s*:\s*(\w+))?\s*\)/);
|
|
137
|
+
if (!catchMatch) continue;
|
|
138
|
+
|
|
139
|
+
const varName = catchMatch[1];
|
|
140
|
+
const explicitType = catchMatch[2] || 'any';
|
|
141
|
+
|
|
142
|
+
// Extract catch body (up to next closing brace at same level)
|
|
143
|
+
const body = this.extractCatchBody(lines, i);
|
|
144
|
+
if (!body) continue;
|
|
145
|
+
|
|
146
|
+
const strategy = this.classifyStrategy(body, varName);
|
|
147
|
+
const rawPattern = body.split('\n')[0]?.trim() || '';
|
|
148
|
+
|
|
149
|
+
handlers.push({
|
|
150
|
+
file,
|
|
151
|
+
line: i + 1,
|
|
152
|
+
errorType: explicitType,
|
|
153
|
+
strategy,
|
|
154
|
+
rawPattern,
|
|
155
|
+
});
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
// Also detect .catch() promise patterns
|
|
159
|
+
for (let i = 0; i < lines.length; i++) {
|
|
160
|
+
const catchMatch = lines[i].match(/\.catch\s*\(\s*(?:async\s+)?(?:\(\s*)?(\w+)/);
|
|
161
|
+
if (!catchMatch) continue;
|
|
162
|
+
|
|
163
|
+
const varName = catchMatch[1];
|
|
164
|
+
|
|
165
|
+
// Extract the callback body
|
|
166
|
+
const body = this.extractCatchCallbackBody(lines, i);
|
|
167
|
+
if (!body) continue;
|
|
168
|
+
|
|
169
|
+
const strategy = this.classifyStrategy(body, varName);
|
|
170
|
+
|
|
171
|
+
handlers.push({
|
|
172
|
+
file,
|
|
173
|
+
line: i + 1,
|
|
174
|
+
errorType: 'Promise',
|
|
175
|
+
strategy,
|
|
176
|
+
rawPattern: body.split('\n')[0]?.trim() || '',
|
|
177
|
+
});
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
private classifyStrategy(body: string, varName: string): string {
|
|
182
|
+
const trimmed = body.trim();
|
|
183
|
+
|
|
184
|
+
// Empty catch (swallow)
|
|
185
|
+
if (!trimmed || trimmed === '{}' || trimmed === '') {
|
|
186
|
+
return 'swallow';
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
// Re-throw
|
|
190
|
+
if (/\bthrow\b/.test(trimmed)) {
|
|
191
|
+
if (/\bthrow\s+new\b/.test(trimmed)) return 'wrap-and-throw';
|
|
192
|
+
if (new RegExp(`\\bthrow\\s+${varName}\\b`).test(trimmed)) return 'rethrow';
|
|
193
|
+
return 'throw-new';
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
// Return patterns
|
|
197
|
+
if (/\breturn\s+null\b/.test(trimmed)) return 'return-null';
|
|
198
|
+
if (/\breturn\s+undefined\b/.test(trimmed) || /\breturn\s*;/.test(trimmed)) return 'return-undefined';
|
|
199
|
+
if (/\breturn\s+\[\s*\]/.test(trimmed)) return 'return-empty-array';
|
|
200
|
+
if (/\breturn\s+\{\s*\}/.test(trimmed)) return 'return-empty-object';
|
|
201
|
+
if (/\breturn\s+false\b/.test(trimmed)) return 'return-false';
|
|
202
|
+
if (/\breturn\s+/.test(trimmed)) return 'return-value';
|
|
203
|
+
|
|
204
|
+
// Logging patterns
|
|
205
|
+
if (/console\.(error|warn)\b/.test(trimmed)) return 'log-error';
|
|
206
|
+
if (/console\.log\b/.test(trimmed)) return 'log-info';
|
|
207
|
+
if (/\blogger\b/i.test(trimmed) || /\blog\b/i.test(trimmed)) return 'log-custom';
|
|
208
|
+
|
|
209
|
+
// Process.exit
|
|
210
|
+
if (/process\.exit\b/.test(trimmed)) return 'exit';
|
|
211
|
+
|
|
212
|
+
// Response patterns (Express/HTTP)
|
|
213
|
+
if (/\bres\s*\.\s*status\b/.test(trimmed) || /\bres\s*\.\s*json\b/.test(trimmed)) return 'http-response';
|
|
214
|
+
|
|
215
|
+
// Notification patterns
|
|
216
|
+
if (/\bnotif|toast|alert|modal\b/i.test(trimmed)) return 'user-notification';
|
|
217
|
+
|
|
218
|
+
return 'other';
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
private extractCatchBody(lines: string[], catchLine: number): string | null {
|
|
222
|
+
let braceDepth = 0;
|
|
223
|
+
let started = false;
|
|
224
|
+
const body: string[] = [];
|
|
225
|
+
|
|
226
|
+
for (let i = catchLine; i < lines.length; i++) {
|
|
227
|
+
for (const ch of lines[i]) {
|
|
228
|
+
if (ch === '{') { braceDepth++; started = true; }
|
|
229
|
+
if (ch === '}') braceDepth--;
|
|
230
|
+
}
|
|
231
|
+
if (started && i > catchLine) body.push(lines[i]);
|
|
232
|
+
if (started && braceDepth === 0) break;
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
return body.length > 0 ? body.join('\n') : null;
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
private extractCatchCallbackBody(lines: string[], startLine: number): string | null {
|
|
239
|
+
let depth = 0;
|
|
240
|
+
let started = false;
|
|
241
|
+
const body: string[] = [];
|
|
242
|
+
|
|
243
|
+
for (let i = startLine; i < Math.min(startLine + 20, lines.length); i++) {
|
|
244
|
+
for (const ch of lines[i]) {
|
|
245
|
+
if (ch === '{' || ch === '(') { depth++; started = true; }
|
|
246
|
+
if (ch === '}' || ch === ')') depth--;
|
|
247
|
+
}
|
|
248
|
+
if (started && i > startLine) body.push(lines[i]);
|
|
249
|
+
if (started && depth <= 0) break;
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
return body.length > 0 ? body.join('\n') : null;
|
|
253
|
+
}
|
|
254
|
+
}
|