@aiready/pattern-detect 0.7.7 → 0.7.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +130 -0
- package/dist/chunk-262N2JB7.mjs +497 -0
- package/dist/chunk-GSJFORRO.mjs +504 -0
- package/dist/chunk-M4PQMW34.mjs +480 -0
- package/dist/chunk-R2S73CVG.mjs +503 -0
- package/dist/chunk-S2KQFII2.mjs +491 -0
- package/dist/chunk-TJKDLVLN.mjs +503 -0
- package/dist/cli.js +57 -48
- package/dist/cli.mjs +16 -2
- package/dist/index.d.mts +5 -1
- package/dist/index.d.ts +5 -1
- package/dist/index.js +46 -49
- package/dist/index.mjs +5 -3
- package/package.json +2 -2
|
@@ -0,0 +1,503 @@
|
|
|
1
|
+
// src/index.ts
|
|
2
|
+
import { readFileContent } from "@aiready/core";
|
|
3
|
+
|
|
4
|
+
// src/detector.ts
|
|
5
|
+
import { estimateTokens } from "@aiready/core";
|
|
6
|
+
function categorizePattern(code) {
|
|
7
|
+
const lower = code.toLowerCase();
|
|
8
|
+
if (lower.includes("request") && lower.includes("response") || lower.includes("router.") || lower.includes("app.get") || lower.includes("app.post") || lower.includes("express") || lower.includes("ctx.body")) {
|
|
9
|
+
return "api-handler";
|
|
10
|
+
}
|
|
11
|
+
if (lower.includes("validate") || lower.includes("schema") || lower.includes("zod") || lower.includes("yup") || lower.includes("if") && lower.includes("throw")) {
|
|
12
|
+
return "validator";
|
|
13
|
+
}
|
|
14
|
+
if (lower.includes("return (") || lower.includes("jsx") || lower.includes("component") || lower.includes("props")) {
|
|
15
|
+
return "component";
|
|
16
|
+
}
|
|
17
|
+
if (lower.includes("class ") || lower.includes("this.")) {
|
|
18
|
+
return "class-method";
|
|
19
|
+
}
|
|
20
|
+
if (lower.includes("return ") && !lower.includes("this") && !lower.includes("new ")) {
|
|
21
|
+
return "utility";
|
|
22
|
+
}
|
|
23
|
+
if (lower.includes("function") || lower.includes("=>")) {
|
|
24
|
+
return "function";
|
|
25
|
+
}
|
|
26
|
+
return "unknown";
|
|
27
|
+
}
|
|
28
|
+
function extractCodeBlocks(content, minLines) {
|
|
29
|
+
const lines = content.split("\n");
|
|
30
|
+
const blocks = [];
|
|
31
|
+
let currentBlock = [];
|
|
32
|
+
let blockStart = 0;
|
|
33
|
+
let braceDepth = 0;
|
|
34
|
+
let inFunction = false;
|
|
35
|
+
for (let i = 0; i < lines.length; i++) {
|
|
36
|
+
const line = lines[i];
|
|
37
|
+
const trimmed = line.trim();
|
|
38
|
+
if (!inFunction && (trimmed.includes("function ") || trimmed.includes("=>") || trimmed.includes("async ") || /^(export\s+)?(async\s+)?function\s+/.test(trimmed) || /^(export\s+)?const\s+\w+\s*=\s*(async\s*)?\(/.test(trimmed))) {
|
|
39
|
+
inFunction = true;
|
|
40
|
+
blockStart = i;
|
|
41
|
+
}
|
|
42
|
+
for (const char of line) {
|
|
43
|
+
if (char === "{") braceDepth++;
|
|
44
|
+
if (char === "}") braceDepth--;
|
|
45
|
+
}
|
|
46
|
+
if (inFunction) {
|
|
47
|
+
currentBlock.push(line);
|
|
48
|
+
}
|
|
49
|
+
if (inFunction && braceDepth === 0 && currentBlock.length >= minLines) {
|
|
50
|
+
const blockContent = currentBlock.join("\n");
|
|
51
|
+
const linesOfCode = currentBlock.filter(
|
|
52
|
+
(l) => l.trim() && !l.trim().startsWith("//")
|
|
53
|
+
).length;
|
|
54
|
+
blocks.push({
|
|
55
|
+
content: blockContent,
|
|
56
|
+
startLine: blockStart + 1,
|
|
57
|
+
endLine: i + 1,
|
|
58
|
+
patternType: categorizePattern(blockContent),
|
|
59
|
+
linesOfCode
|
|
60
|
+
});
|
|
61
|
+
currentBlock = [];
|
|
62
|
+
inFunction = false;
|
|
63
|
+
} else if (inFunction && braceDepth === 0) {
|
|
64
|
+
currentBlock = [];
|
|
65
|
+
inFunction = false;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
return blocks;
|
|
69
|
+
}
|
|
70
|
+
function normalizeCode(code) {
|
|
71
|
+
return code.replace(/\/\/.*$/gm, "").replace(/\/\*[\s\S]*?\*\//g, "").replace(/"[^"]*"/g, '"STR"').replace(/'[^']*'/g, "'STR'").replace(/`[^`]*`/g, "`STR`").replace(/\b\d+\b/g, "NUM").replace(/\s+/g, " ").trim();
|
|
72
|
+
}
|
|
73
|
+
function jaccardSimilarity(tokens1, tokens2) {
|
|
74
|
+
const set1 = new Set(tokens1);
|
|
75
|
+
const set2 = new Set(tokens2);
|
|
76
|
+
let intersection = 0;
|
|
77
|
+
for (const token of set1) {
|
|
78
|
+
if (set2.has(token)) intersection++;
|
|
79
|
+
}
|
|
80
|
+
const union = set1.size + set2.size - intersection;
|
|
81
|
+
return union === 0 ? 0 : intersection / union;
|
|
82
|
+
}
|
|
83
|
+
async function detectDuplicatePatterns(files, options) {
|
|
84
|
+
const {
|
|
85
|
+
minSimilarity,
|
|
86
|
+
minLines,
|
|
87
|
+
batchSize = 100,
|
|
88
|
+
approx = true,
|
|
89
|
+
minSharedTokens = 8,
|
|
90
|
+
maxCandidatesPerBlock = 100,
|
|
91
|
+
streamResults = false
|
|
92
|
+
} = options;
|
|
93
|
+
const duplicates = [];
|
|
94
|
+
const maxComparisons = approx ? Infinity : 5e5;
|
|
95
|
+
const allBlocks = files.flatMap(
|
|
96
|
+
(file) => extractCodeBlocks(file.content, minLines).map((block) => ({
|
|
97
|
+
content: block.content,
|
|
98
|
+
startLine: block.startLine,
|
|
99
|
+
endLine: block.endLine,
|
|
100
|
+
file: file.file,
|
|
101
|
+
normalized: normalizeCode(block.content),
|
|
102
|
+
patternType: block.patternType,
|
|
103
|
+
tokenCost: estimateTokens(block.content),
|
|
104
|
+
linesOfCode: block.linesOfCode
|
|
105
|
+
}))
|
|
106
|
+
);
|
|
107
|
+
console.log(`Extracted ${allBlocks.length} code blocks for analysis`);
|
|
108
|
+
if (!approx && allBlocks.length > 500) {
|
|
109
|
+
console.log(`\u26A0\uFE0F Using --no-approx mode with ${allBlocks.length} blocks may be slow (O(B\xB2) complexity).`);
|
|
110
|
+
console.log(` Consider using approximate mode (default) for better performance.`);
|
|
111
|
+
}
|
|
112
|
+
const stopwords = /* @__PURE__ */ new Set([
|
|
113
|
+
"return",
|
|
114
|
+
"const",
|
|
115
|
+
"let",
|
|
116
|
+
"var",
|
|
117
|
+
"function",
|
|
118
|
+
"class",
|
|
119
|
+
"new",
|
|
120
|
+
"if",
|
|
121
|
+
"else",
|
|
122
|
+
"for",
|
|
123
|
+
"while",
|
|
124
|
+
"async",
|
|
125
|
+
"await",
|
|
126
|
+
"try",
|
|
127
|
+
"catch",
|
|
128
|
+
"switch",
|
|
129
|
+
"case",
|
|
130
|
+
"default",
|
|
131
|
+
"import",
|
|
132
|
+
"export",
|
|
133
|
+
"from",
|
|
134
|
+
"true",
|
|
135
|
+
"false",
|
|
136
|
+
"null",
|
|
137
|
+
"undefined",
|
|
138
|
+
"this"
|
|
139
|
+
]);
|
|
140
|
+
const tokenize = (norm) => norm.split(/[\s(){}\[\];,\.]+/).filter((t) => t && t.length >= 3 && !stopwords.has(t.toLowerCase()));
|
|
141
|
+
const blockTokens = allBlocks.map((b) => tokenize(b.normalized));
|
|
142
|
+
const invertedIndex = /* @__PURE__ */ new Map();
|
|
143
|
+
if (approx) {
|
|
144
|
+
for (let i = 0; i < blockTokens.length; i++) {
|
|
145
|
+
for (const tok of blockTokens[i]) {
|
|
146
|
+
let arr = invertedIndex.get(tok);
|
|
147
|
+
if (!arr) {
|
|
148
|
+
arr = [];
|
|
149
|
+
invertedIndex.set(tok, arr);
|
|
150
|
+
}
|
|
151
|
+
arr.push(i);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
const totalComparisons = approx ? void 0 : allBlocks.length * (allBlocks.length - 1) / 2;
|
|
156
|
+
if (totalComparisons !== void 0) {
|
|
157
|
+
console.log(`Processing ${totalComparisons.toLocaleString()} comparisons in batches...`);
|
|
158
|
+
} else {
|
|
159
|
+
console.log(`Using approximate candidate selection to reduce comparisons...`);
|
|
160
|
+
}
|
|
161
|
+
let comparisonsProcessed = 0;
|
|
162
|
+
let comparisonsBudgetExhausted = false;
|
|
163
|
+
const startTime = Date.now();
|
|
164
|
+
for (let i = 0; i < allBlocks.length; i++) {
|
|
165
|
+
if (maxComparisons && comparisonsProcessed >= maxComparisons) {
|
|
166
|
+
comparisonsBudgetExhausted = true;
|
|
167
|
+
break;
|
|
168
|
+
}
|
|
169
|
+
if (i % batchSize === 0 && i > 0) {
|
|
170
|
+
const elapsed = ((Date.now() - startTime) / 1e3).toFixed(1);
|
|
171
|
+
const duplicatesFound = duplicates.length;
|
|
172
|
+
if (totalComparisons !== void 0) {
|
|
173
|
+
const progress = (comparisonsProcessed / totalComparisons * 100).toFixed(1);
|
|
174
|
+
const remaining = totalComparisons - comparisonsProcessed;
|
|
175
|
+
const rate = comparisonsProcessed / parseFloat(elapsed);
|
|
176
|
+
const eta = remaining > 0 ? (remaining / rate).toFixed(0) : 0;
|
|
177
|
+
console.log(` ${progress}% (${comparisonsProcessed.toLocaleString()}/${totalComparisons.toLocaleString()} comparisons, ${elapsed}s elapsed, ~${eta}s remaining, ${duplicatesFound} duplicates)`);
|
|
178
|
+
} else {
|
|
179
|
+
console.log(` Processed ${i.toLocaleString()}/${allBlocks.length} blocks (${elapsed}s elapsed, ${duplicatesFound} duplicates)`);
|
|
180
|
+
}
|
|
181
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
182
|
+
}
|
|
183
|
+
const block1 = allBlocks[i];
|
|
184
|
+
let candidates = null;
|
|
185
|
+
if (approx) {
|
|
186
|
+
const counts = /* @__PURE__ */ new Map();
|
|
187
|
+
for (const tok of blockTokens[i]) {
|
|
188
|
+
const ids = invertedIndex.get(tok);
|
|
189
|
+
if (!ids) continue;
|
|
190
|
+
for (const j of ids) {
|
|
191
|
+
if (j <= i) continue;
|
|
192
|
+
if (allBlocks[j].file === block1.file) continue;
|
|
193
|
+
counts.set(j, (counts.get(j) || 0) + 1);
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
candidates = Array.from(counts.entries()).filter(([, shared]) => shared >= minSharedTokens).sort((a, b) => b[1] - a[1]).slice(0, maxCandidatesPerBlock).map(([j, shared]) => ({ j, shared }));
|
|
197
|
+
}
|
|
198
|
+
if (approx && candidates) {
|
|
199
|
+
for (const { j } of candidates) {
|
|
200
|
+
if (!approx && maxComparisons !== Infinity && comparisonsProcessed >= maxComparisons) {
|
|
201
|
+
console.log(`\u26A0\uFE0F Comparison safety limit reached (${maxComparisons.toLocaleString()} comparisons in --no-approx mode).`);
|
|
202
|
+
console.log(` This prevents excessive runtime on large repos. Consider using approximate mode (default) or --min-lines to reduce blocks.`);
|
|
203
|
+
break;
|
|
204
|
+
}
|
|
205
|
+
comparisonsProcessed++;
|
|
206
|
+
const block2 = allBlocks[j];
|
|
207
|
+
const similarity = jaccardSimilarity(blockTokens[i], blockTokens[j]);
|
|
208
|
+
if (similarity >= minSimilarity) {
|
|
209
|
+
const duplicate = {
|
|
210
|
+
file1: block1.file,
|
|
211
|
+
file2: block2.file,
|
|
212
|
+
line1: block1.startLine,
|
|
213
|
+
line2: block2.startLine,
|
|
214
|
+
endLine1: block1.endLine,
|
|
215
|
+
endLine2: block2.endLine,
|
|
216
|
+
similarity,
|
|
217
|
+
snippet: block1.content.split("\n").slice(0, 5).join("\n") + "\n...",
|
|
218
|
+
patternType: block1.patternType,
|
|
219
|
+
tokenCost: block1.tokenCost + block2.tokenCost,
|
|
220
|
+
linesOfCode: block1.linesOfCode
|
|
221
|
+
};
|
|
222
|
+
duplicates.push(duplicate);
|
|
223
|
+
if (streamResults) {
|
|
224
|
+
console.log(`
|
|
225
|
+
\u2705 Found: ${duplicate.patternType} ${Math.round(similarity * 100)}% similar`);
|
|
226
|
+
console.log(` ${duplicate.file1}:${duplicate.line1}-${duplicate.endLine1} \u21D4 ${duplicate.file2}:${duplicate.line2}-${duplicate.endLine2}`);
|
|
227
|
+
console.log(` Token cost: ${duplicate.tokenCost.toLocaleString()}`);
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
} else {
|
|
232
|
+
for (let j = i + 1; j < allBlocks.length; j++) {
|
|
233
|
+
if (maxComparisons && comparisonsProcessed >= maxComparisons) break;
|
|
234
|
+
comparisonsProcessed++;
|
|
235
|
+
const block2 = allBlocks[j];
|
|
236
|
+
if (block1.file === block2.file) continue;
|
|
237
|
+
const similarity = jaccardSimilarity(blockTokens[i], blockTokens[j]);
|
|
238
|
+
if (similarity >= minSimilarity) {
|
|
239
|
+
const duplicate = {
|
|
240
|
+
file1: block1.file,
|
|
241
|
+
file2: block2.file,
|
|
242
|
+
line1: block1.startLine,
|
|
243
|
+
line2: block2.startLine,
|
|
244
|
+
endLine1: block1.endLine,
|
|
245
|
+
endLine2: block2.endLine,
|
|
246
|
+
similarity,
|
|
247
|
+
snippet: block1.content.split("\n").slice(0, 5).join("\n") + "\n...",
|
|
248
|
+
patternType: block1.patternType,
|
|
249
|
+
tokenCost: block1.tokenCost + block2.tokenCost,
|
|
250
|
+
linesOfCode: block1.linesOfCode
|
|
251
|
+
};
|
|
252
|
+
duplicates.push(duplicate);
|
|
253
|
+
if (streamResults) {
|
|
254
|
+
console.log(`
|
|
255
|
+
\u2705 Found: ${duplicate.patternType} ${Math.round(similarity * 100)}% similar`);
|
|
256
|
+
console.log(` ${duplicate.file1}:${duplicate.line1}-${duplicate.endLine1} \u21D4 ${duplicate.file2}:${duplicate.line2}-${duplicate.endLine2}`);
|
|
257
|
+
console.log(` Token cost: ${duplicate.tokenCost.toLocaleString()}`);
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
if (comparisonsBudgetExhausted) {
|
|
264
|
+
console.log(`\u26A0\uFE0F Comparison budget exhausted (${maxComparisons.toLocaleString()} comparisons). Use --max-comparisons to increase.`);
|
|
265
|
+
}
|
|
266
|
+
return duplicates.sort(
|
|
267
|
+
(a, b) => b.similarity - a.similarity || b.tokenCost - a.tokenCost
|
|
268
|
+
);
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
// src/index.ts
|
|
272
|
+
function getRefactoringSuggestion(patternType, similarity) {
|
|
273
|
+
const baseMessages = {
|
|
274
|
+
"api-handler": "Extract common middleware or create a base handler class",
|
|
275
|
+
validator: "Consolidate validation logic into shared schema validators (Zod/Yup)",
|
|
276
|
+
utility: "Move to a shared utilities file and reuse across modules",
|
|
277
|
+
"class-method": "Consider inheritance or composition to share behavior",
|
|
278
|
+
component: "Extract shared logic into a custom hook or HOC",
|
|
279
|
+
function: "Extract into a shared helper function",
|
|
280
|
+
unknown: "Extract common logic into a reusable module"
|
|
281
|
+
};
|
|
282
|
+
const urgency = similarity > 0.95 ? " (CRITICAL: Nearly identical code)" : similarity > 0.9 ? " (HIGH: Very similar, refactor soon)" : "";
|
|
283
|
+
return baseMessages[patternType] + urgency;
|
|
284
|
+
}
|
|
285
|
+
async function getSmartDefaults(directory, userOptions) {
|
|
286
|
+
if (userOptions.useSmartDefaults === false) {
|
|
287
|
+
return {};
|
|
288
|
+
}
|
|
289
|
+
const scanOptions = {
|
|
290
|
+
rootDir: directory,
|
|
291
|
+
include: userOptions.include || ["**/*.{ts,tsx,js,jsx,py,java}"],
|
|
292
|
+
exclude: userOptions.exclude || [
|
|
293
|
+
"**/node_modules/**",
|
|
294
|
+
"**/dist/**",
|
|
295
|
+
"**/build/**",
|
|
296
|
+
"**/coverage/**",
|
|
297
|
+
"**/.git/**",
|
|
298
|
+
"**/.turbo/**"
|
|
299
|
+
]
|
|
300
|
+
};
|
|
301
|
+
const { scanFiles: scanFiles2 } = await import("@aiready/core");
|
|
302
|
+
const files = await scanFiles2(scanOptions);
|
|
303
|
+
const estimatedBlocks = files.length * 3;
|
|
304
|
+
let defaults;
|
|
305
|
+
if (estimatedBlocks < 1e3) {
|
|
306
|
+
defaults = {
|
|
307
|
+
rootDir: directory,
|
|
308
|
+
minSimilarity: 0.4,
|
|
309
|
+
minLines: 5,
|
|
310
|
+
batchSize: 100,
|
|
311
|
+
approx: true,
|
|
312
|
+
minSharedTokens: 8,
|
|
313
|
+
maxCandidatesPerBlock: 100,
|
|
314
|
+
streamResults: false,
|
|
315
|
+
severity: "all",
|
|
316
|
+
includeTests: false
|
|
317
|
+
};
|
|
318
|
+
} else if (estimatedBlocks < 5e3) {
|
|
319
|
+
defaults = {
|
|
320
|
+
rootDir: directory,
|
|
321
|
+
minSimilarity: 0.5,
|
|
322
|
+
minLines: 6,
|
|
323
|
+
batchSize: 200,
|
|
324
|
+
approx: true,
|
|
325
|
+
minSharedTokens: 12,
|
|
326
|
+
maxCandidatesPerBlock: 30,
|
|
327
|
+
streamResults: false,
|
|
328
|
+
severity: "all",
|
|
329
|
+
includeTests: false
|
|
330
|
+
};
|
|
331
|
+
} else {
|
|
332
|
+
defaults = {
|
|
333
|
+
rootDir: directory,
|
|
334
|
+
minSimilarity: 0.6,
|
|
335
|
+
minLines: 8,
|
|
336
|
+
batchSize: 200,
|
|
337
|
+
approx: true,
|
|
338
|
+
minSharedTokens: 12,
|
|
339
|
+
maxCandidatesPerBlock: 25,
|
|
340
|
+
streamResults: false,
|
|
341
|
+
severity: "high",
|
|
342
|
+
includeTests: false
|
|
343
|
+
};
|
|
344
|
+
}
|
|
345
|
+
const result = { ...defaults };
|
|
346
|
+
for (const [key, value] of Object.entries(defaults)) {
|
|
347
|
+
if (key in userOptions && userOptions[key] !== void 0) {
|
|
348
|
+
result[key] = userOptions[key];
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
return result;
|
|
352
|
+
}
|
|
353
|
+
function logConfiguration(config, estimatedBlocks) {
|
|
354
|
+
console.log("\u{1F4CB} Configuration:");
|
|
355
|
+
console.log(` Repository size: ~${estimatedBlocks} code blocks`);
|
|
356
|
+
console.log(` Similarity threshold: ${config.minSimilarity}`);
|
|
357
|
+
console.log(` Minimum lines: ${config.minLines}`);
|
|
358
|
+
console.log(` Approximate mode: ${config.approx ? "enabled" : "disabled"}`);
|
|
359
|
+
console.log(` Max candidates per block: ${config.maxCandidatesPerBlock}`);
|
|
360
|
+
console.log(` Min shared tokens: ${config.minSharedTokens}`);
|
|
361
|
+
console.log(` Severity filter: ${config.severity}`);
|
|
362
|
+
console.log(` Include tests: ${config.includeTests}`);
|
|
363
|
+
console.log("");
|
|
364
|
+
}
|
|
365
|
+
async function analyzePatterns(options) {
|
|
366
|
+
const smartDefaults = await getSmartDefaults(options.rootDir || ".", options);
|
|
367
|
+
const finalOptions = { ...smartDefaults, ...options };
|
|
368
|
+
const {
|
|
369
|
+
minSimilarity = 0.4,
|
|
370
|
+
minLines = 5,
|
|
371
|
+
batchSize = 100,
|
|
372
|
+
approx = true,
|
|
373
|
+
minSharedTokens = 8,
|
|
374
|
+
maxCandidatesPerBlock = 100,
|
|
375
|
+
streamResults = false,
|
|
376
|
+
severity = "all",
|
|
377
|
+
includeTests = false,
|
|
378
|
+
...scanOptions
|
|
379
|
+
} = finalOptions;
|
|
380
|
+
const { scanFiles: scanFiles2 } = await import("@aiready/core");
|
|
381
|
+
const files = await scanFiles2(scanOptions);
|
|
382
|
+
const estimatedBlocks = files.length * 3;
|
|
383
|
+
logConfiguration(finalOptions, estimatedBlocks);
|
|
384
|
+
const results = [];
|
|
385
|
+
const fileContents = await Promise.all(
|
|
386
|
+
files.map(async (file) => ({
|
|
387
|
+
file,
|
|
388
|
+
content: await readFileContent(file)
|
|
389
|
+
}))
|
|
390
|
+
);
|
|
391
|
+
const duplicates = await detectDuplicatePatterns(fileContents, {
|
|
392
|
+
minSimilarity,
|
|
393
|
+
minLines,
|
|
394
|
+
batchSize,
|
|
395
|
+
approx,
|
|
396
|
+
minSharedTokens,
|
|
397
|
+
maxCandidatesPerBlock,
|
|
398
|
+
streamResults
|
|
399
|
+
});
|
|
400
|
+
for (const file of files) {
|
|
401
|
+
const fileDuplicates = duplicates.filter(
|
|
402
|
+
(dup) => dup.file1 === file || dup.file2 === file
|
|
403
|
+
);
|
|
404
|
+
const issues = fileDuplicates.map((dup) => {
|
|
405
|
+
const otherFile = dup.file1 === file ? dup.file2 : dup.file1;
|
|
406
|
+
const severity2 = dup.similarity > 0.95 ? "critical" : dup.similarity > 0.9 ? "major" : "minor";
|
|
407
|
+
return {
|
|
408
|
+
type: "duplicate-pattern",
|
|
409
|
+
severity: severity2,
|
|
410
|
+
message: `${dup.patternType} pattern ${Math.round(dup.similarity * 100)}% similar to ${otherFile} (${dup.tokenCost} tokens wasted)`,
|
|
411
|
+
location: {
|
|
412
|
+
file,
|
|
413
|
+
line: dup.file1 === file ? dup.line1 : dup.line2
|
|
414
|
+
},
|
|
415
|
+
suggestion: getRefactoringSuggestion(dup.patternType, dup.similarity)
|
|
416
|
+
};
|
|
417
|
+
});
|
|
418
|
+
let filteredIssues = issues;
|
|
419
|
+
if (severity !== "all") {
|
|
420
|
+
const severityMap = {
|
|
421
|
+
critical: ["critical"],
|
|
422
|
+
high: ["critical", "major"],
|
|
423
|
+
medium: ["critical", "major", "minor"]
|
|
424
|
+
};
|
|
425
|
+
const allowedSeverities = severityMap[severity] || ["critical", "major", "minor"];
|
|
426
|
+
filteredIssues = issues.filter((issue) => allowedSeverities.includes(issue.severity));
|
|
427
|
+
}
|
|
428
|
+
const totalTokenCost = fileDuplicates.reduce(
|
|
429
|
+
(sum, dup) => sum + dup.tokenCost,
|
|
430
|
+
0
|
|
431
|
+
);
|
|
432
|
+
results.push({
|
|
433
|
+
fileName: file,
|
|
434
|
+
issues: filteredIssues,
|
|
435
|
+
metrics: {
|
|
436
|
+
tokenCost: totalTokenCost,
|
|
437
|
+
consistencyScore: Math.max(0, 1 - fileDuplicates.length * 0.1)
|
|
438
|
+
}
|
|
439
|
+
});
|
|
440
|
+
}
|
|
441
|
+
return { results, duplicates, files };
|
|
442
|
+
}
|
|
443
|
+
function generateSummary(results) {
|
|
444
|
+
const allIssues = results.flatMap((r) => r.issues);
|
|
445
|
+
const totalTokenCost = results.reduce(
|
|
446
|
+
(sum, r) => sum + (r.metrics.tokenCost || 0),
|
|
447
|
+
0
|
|
448
|
+
);
|
|
449
|
+
const patternsByType = {
|
|
450
|
+
"api-handler": 0,
|
|
451
|
+
validator: 0,
|
|
452
|
+
utility: 0,
|
|
453
|
+
"class-method": 0,
|
|
454
|
+
component: 0,
|
|
455
|
+
function: 0,
|
|
456
|
+
unknown: 0
|
|
457
|
+
};
|
|
458
|
+
allIssues.forEach((issue) => {
|
|
459
|
+
const match = issue.message.match(/^(\S+(?:-\S+)*) pattern/);
|
|
460
|
+
if (match) {
|
|
461
|
+
const type = match[1];
|
|
462
|
+
patternsByType[type] = (patternsByType[type] || 0) + 1;
|
|
463
|
+
}
|
|
464
|
+
});
|
|
465
|
+
const topDuplicates = allIssues.slice(0, 10).map((issue) => {
|
|
466
|
+
const similarityMatch = issue.message.match(/(\d+)% similar/);
|
|
467
|
+
const tokenMatch = issue.message.match(/\((\d+) tokens/);
|
|
468
|
+
const typeMatch = issue.message.match(/^(\S+(?:-\S+)*) pattern/);
|
|
469
|
+
const fileMatch = issue.message.match(/similar to (.+?) \(/);
|
|
470
|
+
return {
|
|
471
|
+
files: [
|
|
472
|
+
{
|
|
473
|
+
path: issue.location.file,
|
|
474
|
+
startLine: issue.location.line,
|
|
475
|
+
endLine: 0
|
|
476
|
+
// Not available from Issue
|
|
477
|
+
},
|
|
478
|
+
{
|
|
479
|
+
path: fileMatch?.[1] || "unknown",
|
|
480
|
+
startLine: 0,
|
|
481
|
+
// Not available from Issue
|
|
482
|
+
endLine: 0
|
|
483
|
+
// Not available from Issue
|
|
484
|
+
}
|
|
485
|
+
],
|
|
486
|
+
similarity: similarityMatch ? parseInt(similarityMatch[1]) / 100 : 0,
|
|
487
|
+
patternType: typeMatch?.[1] || "unknown",
|
|
488
|
+
tokenCost: tokenMatch ? parseInt(tokenMatch[1]) : 0
|
|
489
|
+
};
|
|
490
|
+
});
|
|
491
|
+
return {
|
|
492
|
+
totalPatterns: allIssues.length,
|
|
493
|
+
totalTokenCost,
|
|
494
|
+
patternsByType,
|
|
495
|
+
topDuplicates
|
|
496
|
+
};
|
|
497
|
+
}
|
|
498
|
+
|
|
499
|
+
export {
|
|
500
|
+
detectDuplicatePatterns,
|
|
501
|
+
analyzePatterns,
|
|
502
|
+
generateSummary
|
|
503
|
+
};
|
package/dist/cli.js
CHANGED
|
@@ -212,7 +212,13 @@ async function detectDuplicatePatterns(files, options) {
|
|
|
212
212
|
let candidates = null;
|
|
213
213
|
if (approx) {
|
|
214
214
|
const counts = /* @__PURE__ */ new Map();
|
|
215
|
-
|
|
215
|
+
const block1Tokens = new Set(blockTokens[i]);
|
|
216
|
+
const block1Size = block1Tokens.size;
|
|
217
|
+
const rareTokens = blockTokens[i].filter((tok) => {
|
|
218
|
+
const blocksWithToken = invertedIndex.get(tok)?.length || 0;
|
|
219
|
+
return blocksWithToken < allBlocks.length * 0.1;
|
|
220
|
+
});
|
|
221
|
+
for (const tok of rareTokens) {
|
|
216
222
|
const ids = invertedIndex.get(tok);
|
|
217
223
|
if (!ids) continue;
|
|
218
224
|
for (const j of ids) {
|
|
@@ -221,7 +227,13 @@ async function detectDuplicatePatterns(files, options) {
|
|
|
221
227
|
counts.set(j, (counts.get(j) || 0) + 1);
|
|
222
228
|
}
|
|
223
229
|
}
|
|
224
|
-
candidates = Array.from(counts.entries()).filter(([
|
|
230
|
+
candidates = Array.from(counts.entries()).filter(([j, shared]) => {
|
|
231
|
+
const block2Tokens = blockTokens[j];
|
|
232
|
+
const block2Size = block2Tokens.length;
|
|
233
|
+
const minSize = Math.min(block1Size, block2Size);
|
|
234
|
+
const sharedPercentage = shared / minSize;
|
|
235
|
+
return shared >= minSharedTokens && sharedPercentage >= 0.3;
|
|
236
|
+
}).sort((a, b) => b[1] - a[1]).slice(0, Math.min(maxCandidatesPerBlock, 5)).map(([j, shared]) => ({ j, shared }));
|
|
225
237
|
}
|
|
226
238
|
if (approx && candidates) {
|
|
227
239
|
for (const { j } of candidates) {
|
|
@@ -314,12 +326,12 @@ async function getSmartDefaults(directory, userOptions) {
|
|
|
314
326
|
if (userOptions.useSmartDefaults === false) {
|
|
315
327
|
return {
|
|
316
328
|
rootDir: directory,
|
|
317
|
-
minSimilarity: 0.
|
|
318
|
-
minLines:
|
|
329
|
+
minSimilarity: 0.6,
|
|
330
|
+
minLines: 8,
|
|
319
331
|
batchSize: 100,
|
|
320
332
|
approx: true,
|
|
321
|
-
minSharedTokens:
|
|
322
|
-
maxCandidatesPerBlock:
|
|
333
|
+
minSharedTokens: 12,
|
|
334
|
+
maxCandidatesPerBlock: 5,
|
|
323
335
|
streamResults: false,
|
|
324
336
|
severity: "all",
|
|
325
337
|
includeTests: false
|
|
@@ -340,48 +352,31 @@ async function getSmartDefaults(directory, userOptions) {
|
|
|
340
352
|
const { scanFiles: scanFiles2 } = await import("@aiready/core");
|
|
341
353
|
const files = await scanFiles2(scanOptions);
|
|
342
354
|
const estimatedBlocks = files.length * 3;
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
streamResults: false,
|
|
367
|
-
severity: "all",
|
|
368
|
-
includeTests: false
|
|
369
|
-
};
|
|
370
|
-
} else {
|
|
371
|
-
defaults = {
|
|
372
|
-
rootDir: directory,
|
|
373
|
-
minSimilarity: 0.6,
|
|
374
|
-
minLines: 8,
|
|
375
|
-
batchSize: 200,
|
|
376
|
-
approx: true,
|
|
377
|
-
minSharedTokens: 12,
|
|
378
|
-
maxCandidatesPerBlock: 25,
|
|
379
|
-
streamResults: false,
|
|
380
|
-
severity: "high",
|
|
381
|
-
includeTests: false
|
|
382
|
-
};
|
|
355
|
+
const maxCandidatesPerBlock = Math.max(3, Math.min(10, Math.floor(3e4 / estimatedBlocks)));
|
|
356
|
+
const minSimilarity = Math.min(0.75, 0.5 + estimatedBlocks / 1e4 * 0.25);
|
|
357
|
+
const minLines = Math.max(6, Math.min(12, 6 + Math.floor(estimatedBlocks / 2e3)));
|
|
358
|
+
const minSharedTokens = Math.max(10, Math.min(20, 10 + Math.floor(estimatedBlocks / 2e3)));
|
|
359
|
+
const batchSize = estimatedBlocks > 1e3 ? 200 : 100;
|
|
360
|
+
const severity = estimatedBlocks > 5e3 ? "high" : "all";
|
|
361
|
+
let defaults = {
|
|
362
|
+
rootDir: directory,
|
|
363
|
+
minSimilarity,
|
|
364
|
+
minLines,
|
|
365
|
+
batchSize,
|
|
366
|
+
approx: true,
|
|
367
|
+
minSharedTokens,
|
|
368
|
+
maxCandidatesPerBlock,
|
|
369
|
+
streamResults: false,
|
|
370
|
+
severity,
|
|
371
|
+
includeTests: false
|
|
372
|
+
};
|
|
373
|
+
const result = { ...defaults };
|
|
374
|
+
for (const [key, value] of Object.entries(defaults)) {
|
|
375
|
+
if (key in userOptions && userOptions[key] !== void 0) {
|
|
376
|
+
result[key] = userOptions[key];
|
|
377
|
+
}
|
|
383
378
|
}
|
|
384
|
-
return
|
|
379
|
+
return result;
|
|
385
380
|
}
|
|
386
381
|
function logConfiguration(config, estimatedBlocks) {
|
|
387
382
|
console.log("\u{1F4CB} Configuration:");
|
|
@@ -535,7 +530,7 @@ var import_fs = require("fs");
|
|
|
535
530
|
var import_path = require("path");
|
|
536
531
|
var import_core3 = require("@aiready/core");
|
|
537
532
|
var program = new import_commander.Command();
|
|
538
|
-
program.name("aiready-patterns").description("Detect duplicate patterns in your codebase").version("0.1.0").addHelpText("after", "\nCONFIGURATION:\n Supports config files: aiready.json, aiready.config.json, .aiready.json, .aireadyrc.json, aiready.config.js, .aireadyrc.js\n CLI options override config file settings").argument("<directory>", "Directory to analyze").option("-s, --similarity <number>", "Minimum similarity score (0-1)").option("-l, --min-lines <number>", "Minimum lines to consider").option("--batch-size <number>", "Batch size for comparisons").option("--no-approx", "Disable approximate candidate selection
|
|
533
|
+
program.name("aiready-patterns").description("Detect duplicate patterns in your codebase").version("0.1.0").addHelpText("after", "\nCONFIGURATION:\n Supports config files: aiready.json, aiready.config.json, .aiready.json, .aireadyrc.json, aiready.config.js, .aireadyrc.js\n CLI options override config file settings\n\nPARAMETER TUNING:\n If you get too few results: decrease --similarity, --min-lines, or --min-shared-tokens\n If analysis is too slow: increase --min-lines, --min-shared-tokens, or decrease --max-candidates\n If you get too many false positives: increase --similarity or --min-lines\n\nEXAMPLES:\n aiready-patterns . # Basic analysis with smart defaults\n aiready-patterns . --similarity 0.3 --min-lines 3 # More sensitive detection\n aiready-patterns . --max-candidates 50 --no-approx # Slower but more thorough\n aiready-patterns . --output json > report.json # JSON export").argument("<directory>", "Directory to analyze").option("-s, --similarity <number>", "Minimum similarity score (0-1). Lower = more results, higher = fewer but more accurate. Default: 0.4").option("-l, --min-lines <number>", "Minimum lines to consider. Lower = more results, higher = faster analysis. Default: 5").option("--batch-size <number>", "Batch size for comparisons. Higher = faster but more memory. Default: 100").option("--no-approx", "Disable approximate candidate selection. Slower but more thorough on small repos").option("--min-shared-tokens <number>", "Minimum shared tokens to consider a candidate. Higher = faster, fewer results. Default: 8").option("--max-candidates <number>", "Maximum candidates per block. Higher = more thorough but slower. Default: 100").option("--no-stream-results", "Disable incremental output (default: enabled)").option("--include <patterns>", "File patterns to include (comma-separated)").option("--exclude <patterns>", "File patterns to exclude (comma-separated)").option("--severity <level>", "Filter by severity: critical|high|medium|all. Default: all").option("--include-tests", "Include test files in analysis (excluded by default)").option("--max-results <number>", "Maximum number of results to show in console output. Default: 10").option(
|
|
539
534
|
"-o, --output <format>",
|
|
540
535
|
"Output format: console, json, html",
|
|
541
536
|
"console"
|
|
@@ -687,6 +682,20 @@ program.name("aiready-patterns").description("Detect duplicate patterns in your
|
|
|
687
682
|
}
|
|
688
683
|
if (totalIssues === 0) {
|
|
689
684
|
console.log(import_chalk.default.green("\n\u2728 Great! No duplicate patterns detected.\n"));
|
|
685
|
+
console.log(import_chalk.default.yellow("\u{1F4A1} If you expected to find duplicates, try adjusting parameters:"));
|
|
686
|
+
console.log(import_chalk.default.dim(" \u2022 Lower similarity threshold: --similarity 0.3"));
|
|
687
|
+
console.log(import_chalk.default.dim(" \u2022 Reduce minimum lines: --min-lines 3"));
|
|
688
|
+
console.log(import_chalk.default.dim(" \u2022 Include test files: --include-tests"));
|
|
689
|
+
console.log(import_chalk.default.dim(" \u2022 Lower shared tokens threshold: --min-shared-tokens 5"));
|
|
690
|
+
console.log("");
|
|
691
|
+
}
|
|
692
|
+
if (totalIssues > 0 && totalIssues < 5) {
|
|
693
|
+
console.log(import_chalk.default.yellow("\n\u{1F4A1} Few results found. To find more duplicates, try:"));
|
|
694
|
+
console.log(import_chalk.default.dim(" \u2022 Lower similarity threshold: --similarity 0.3"));
|
|
695
|
+
console.log(import_chalk.default.dim(" \u2022 Reduce minimum lines: --min-lines 3"));
|
|
696
|
+
console.log(import_chalk.default.dim(" \u2022 Include test files: --include-tests"));
|
|
697
|
+
console.log(import_chalk.default.dim(" \u2022 Lower shared tokens threshold: --min-shared-tokens 5"));
|
|
698
|
+
console.log("");
|
|
690
699
|
}
|
|
691
700
|
console.log(import_chalk.default.cyan(divider));
|
|
692
701
|
if (totalIssues > 0) {
|