@aiready/pattern-detect 0.1.2 → 0.2.0

package/dist/index.d.ts

@@ -5,6 +5,8 @@ interface DuplicatePattern {
     file2: string;
     line1: number;
     line2: number;
+    endLine1: number;
+    endLine2: number;
     similarity: number;
     snippet: string;
     patternType: PatternType;
@@ -19,23 +21,38 @@ interface FileContent {
 interface DetectionOptions {
     minSimilarity: number;
     minLines: number;
+    maxBlocks?: number;
+    batchSize?: number;
+    approx?: boolean;
+    minSharedTokens?: number;
+    maxCandidatesPerBlock?: number;
+    maxComparisons?: number;
+    streamResults?: boolean;
 }
 /**
  * Detect duplicate patterns across files with enhanced analysis
  */
-declare function detectDuplicatePatterns(files: FileContent[], options: DetectionOptions): DuplicatePattern[];
+declare function detectDuplicatePatterns(files: FileContent[], options: DetectionOptions): Promise<DuplicatePattern[]>;
 
 interface PatternDetectOptions extends ScanOptions {
     minSimilarity?: number;
     minLines?: number;
+    batchSize?: number;
+    approx?: boolean;
+    minSharedTokens?: number;
+    maxCandidatesPerBlock?: number;
+    streamResults?: boolean;
 }
 interface PatternSummary {
     totalPatterns: number;
     totalTokenCost: number;
     patternsByType: Record<PatternType, number>;
     topDuplicates: Array<{
-        file1: string;
-        file2: string;
+        files: Array<{
+            path: string;
+            startLine: number;
+            endLine: number;
+        }>;
         similarity: number;
         patternType: PatternType;
         tokenCost: number;

package/dist/index.js

@@ -80,6 +80,7 @@ function extractCodeBlocks(content, minLines) {
       blocks.push({
         content: blockContent,
         startLine: blockStart + 1,
+        endLine: i + 1,
         patternType: categorizePattern(blockContent),
         linesOfCode
       });
@@ -95,47 +96,198 @@ function extractCodeBlocks(content, minLines) {
 function normalizeCode(code) {
   return code.replace(/\/\/.*$/gm, "").replace(/\/\*[\s\S]*?\*\//g, "").replace(/"[^"]*"/g, '"STR"').replace(/'[^']*'/g, "'STR'").replace(/`[^`]*`/g, "`STR`").replace(/\b\d+\b/g, "NUM").replace(/\s+/g, " ").trim();
 }
-function calculateSimilarity(block1, block2) {
-  const norm1 = normalizeCode(block1);
-  const norm2 = normalizeCode(block2);
-  const baseSimilarity = (0, import_core.similarityScore)(norm1, norm2);
-  const tokens1 = norm1.split(/[\s(){}[\];,]+/).filter(Boolean);
-  const tokens2 = norm2.split(/[\s(){}[\];,]+/).filter(Boolean);
-  const tokenSimilarity = (0, import_core.similarityScore)(tokens1.join(" "), tokens2.join(" "));
-  return baseSimilarity * 0.4 + tokenSimilarity * 0.6;
+function jaccardSimilarity(tokens1, tokens2) {
+  const set1 = new Set(tokens1);
+  const set2 = new Set(tokens2);
+  let intersection = 0;
+  for (const token of set1) {
+    if (set2.has(token)) intersection++;
+  }
+  const union = set1.size + set2.size - intersection;
+  return union === 0 ? 0 : intersection / union;
 }
-function detectDuplicatePatterns(files, options) {
-  const { minSimilarity, minLines } = options;
+async function detectDuplicatePatterns(files, options) {
+  const {
+    minSimilarity,
+    minLines,
+    batchSize = 100,
+    approx = true,
+    minSharedTokens = 8,
+    maxCandidatesPerBlock = 100,
+    streamResults = false
+  } = options;
   const duplicates = [];
+  const maxComparisons = approx ? Infinity : 5e5;
   const allBlocks = files.flatMap(
     (file) => extractCodeBlocks(file.content, minLines).map((block) => ({
-      ...block,
+      content: block.content,
+      startLine: block.startLine,
+      endLine: block.endLine,
       file: file.file,
       normalized: normalizeCode(block.content),
-      tokenCost: (0, import_core.estimateTokens)(block.content)
+      patternType: block.patternType,
+      tokenCost: (0, import_core.estimateTokens)(block.content),
+      linesOfCode: block.linesOfCode
     }))
   );
   console.log(`Extracted ${allBlocks.length} code blocks for analysis`);
+  if (!approx && allBlocks.length > 500) {
+    console.log(`\u26A0\uFE0F  Using --no-approx mode with ${allBlocks.length} blocks may be slow (O(B\xB2) complexity).`);
+    console.log(`   Consider using approximate mode (default) for better performance.`);
+  }
+  const stopwords = /* @__PURE__ */ new Set([
+    "return",
+    "const",
+    "let",
+    "var",
+    "function",
+    "class",
+    "new",
+    "if",
+    "else",
+    "for",
+    "while",
+    "async",
+    "await",
+    "try",
+    "catch",
+    "switch",
+    "case",
+    "default",
+    "import",
+    "export",
+    "from",
+    "true",
+    "false",
+    "null",
+    "undefined",
+    "this"
+  ]);
+  const tokenize = (norm) => norm.split(/[\s(){}\[\];,\.]+/).filter((t) => t && t.length >= 3 && !stopwords.has(t.toLowerCase()));
+  const blockTokens = allBlocks.map((b) => tokenize(b.normalized));
+  const invertedIndex = /* @__PURE__ */ new Map();
+  if (approx) {
+    for (let i = 0; i < blockTokens.length; i++) {
+      for (const tok of blockTokens[i]) {
+        let arr = invertedIndex.get(tok);
+        if (!arr) {
+          arr = [];
+          invertedIndex.set(tok, arr);
+        }
+        arr.push(i);
+      }
+    }
+  }
+  const totalComparisons = approx ? void 0 : allBlocks.length * (allBlocks.length - 1) / 2;
+  if (totalComparisons !== void 0) {
+    console.log(`Processing ${totalComparisons.toLocaleString()} comparisons in batches...`);
+  } else {
+    console.log(`Using approximate candidate selection to reduce comparisons...`);
+  }
+  let comparisonsProcessed = 0;
+  let comparisonsBudgetExhausted = false;
+  const startTime = Date.now();
   for (let i = 0; i < allBlocks.length; i++) {
-    for (let j = i + 1; j < allBlocks.length; j++) {
-      const block1 = allBlocks[i];
-      const block2 = allBlocks[j];
-      if (block1.file === block2.file) continue;
-      const similarity = calculateSimilarity(block1.content, block2.content);
-      if (similarity >= minSimilarity) {
-        duplicates.push({
-          file1: block1.file,
-          file2: block2.file,
-          line1: block1.startLine,
-          line2: block2.startLine,
-          similarity,
-          snippet: block1.content.split("\n").slice(0, 5).join("\n") + "\n...",
-          patternType: block1.patternType,
-          tokenCost: block1.tokenCost + block2.tokenCost,
-          linesOfCode: block1.linesOfCode
-        });
+    if (maxComparisons && comparisonsProcessed >= maxComparisons) {
+      comparisonsBudgetExhausted = true;
+      break;
+    }
+    if (i % batchSize === 0 && i > 0) {
+      const elapsed = ((Date.now() - startTime) / 1e3).toFixed(1);
+      const duplicatesFound = duplicates.length;
+      if (totalComparisons !== void 0) {
+        const progress = (comparisonsProcessed / totalComparisons * 100).toFixed(1);
+        const remaining = totalComparisons - comparisonsProcessed;
+        const rate = comparisonsProcessed / parseFloat(elapsed);
+        const eta = remaining > 0 ? (remaining / rate).toFixed(0) : 0;
+        console.log(`   ${progress}% (${comparisonsProcessed.toLocaleString()}/${totalComparisons.toLocaleString()} comparisons, ${elapsed}s elapsed, ~${eta}s remaining, ${duplicatesFound} duplicates)`);
+      } else {
+        console.log(`   Processed ${i.toLocaleString()}/${allBlocks.length} blocks (${elapsed}s elapsed, ${duplicatesFound} duplicates)`);
       }
+      await new Promise((resolve) => setImmediate(resolve));
     }
+    const block1 = allBlocks[i];
+    let candidates = null;
+    if (approx) {
+      const counts = /* @__PURE__ */ new Map();
+      for (const tok of blockTokens[i]) {
+        const ids = invertedIndex.get(tok);
+        if (!ids) continue;
+        for (const j of ids) {
+          if (j <= i) continue;
+          if (allBlocks[j].file === block1.file) continue;
+          counts.set(j, (counts.get(j) || 0) + 1);
+        }
+      }
+      candidates = Array.from(counts.entries()).filter(([, shared]) => shared >= minSharedTokens).sort((a, b) => b[1] - a[1]).slice(0, maxCandidatesPerBlock).map(([j, shared]) => ({ j, shared }));
+    }
+    if (approx && candidates) {
+      for (const { j } of candidates) {
+        if (!approx && maxComparisons !== Infinity && comparisonsProcessed >= maxComparisons) {
+          console.log(`\u26A0\uFE0F  Comparison safety limit reached (${maxComparisons.toLocaleString()} comparisons in --no-approx mode).`);
+          console.log(`   This prevents excessive runtime on large repos. Consider using approximate mode (default) or --min-lines to reduce blocks.`);
+          break;
+        }
+        comparisonsProcessed++;
+        const block2 = allBlocks[j];
+        const similarity = jaccardSimilarity(blockTokens[i], blockTokens[j]);
+        if (similarity >= minSimilarity) {
+          const duplicate = {
+            file1: block1.file,
+            file2: block2.file,
+            line1: block1.startLine,
+            line2: block2.startLine,
+            endLine1: block1.endLine,
+            endLine2: block2.endLine,
+            similarity,
+            snippet: block1.content.split("\n").slice(0, 5).join("\n") + "\n...",
+            patternType: block1.patternType,
+            tokenCost: block1.tokenCost + block2.tokenCost,
+            linesOfCode: block1.linesOfCode
+          };
+          duplicates.push(duplicate);
+          if (streamResults) {
+            console.log(`
+   \u2705 Found: ${duplicate.patternType} ${Math.round(similarity * 100)}% similar`);
+            console.log(`      ${duplicate.file1}:${duplicate.line1}-${duplicate.endLine1} \u21D4 ${duplicate.file2}:${duplicate.line2}-${duplicate.endLine2}`);
+            console.log(`      Token cost: ${duplicate.tokenCost.toLocaleString()}`);
+          }
+        }
+      }
+    } else {
+      for (let j = i + 1; j < allBlocks.length; j++) {
+        if (maxComparisons && comparisonsProcessed >= maxComparisons) break;
+        comparisonsProcessed++;
+        const block2 = allBlocks[j];
+        if (block1.file === block2.file) continue;
+        const similarity = jaccardSimilarity(blockTokens[i], blockTokens[j]);
+        if (similarity >= minSimilarity) {
+          const duplicate = {
+            file1: block1.file,
+            file2: block2.file,
+            line1: block1.startLine,
+            line2: block2.startLine,
+            endLine1: block1.endLine,
+            endLine2: block2.endLine,
+            similarity,
+            snippet: block1.content.split("\n").slice(0, 5).join("\n") + "\n...",
+            patternType: block1.patternType,
+            tokenCost: block1.tokenCost + block2.tokenCost,
+            linesOfCode: block1.linesOfCode
+          };
+          duplicates.push(duplicate);
+          if (streamResults) {
+            console.log(`
+   \u2705 Found: ${duplicate.patternType} ${Math.round(similarity * 100)}% similar`);
+            console.log(`      ${duplicate.file1}:${duplicate.line1}-${duplicate.endLine1} \u21D4 ${duplicate.file2}:${duplicate.line2}-${duplicate.endLine2}`);
+            console.log(`      Token cost: ${duplicate.tokenCost.toLocaleString()}`);
+          }
+        }
+      }
+    }
+  }
+  if (comparisonsBudgetExhausted) {
+    console.log(`\u26A0\uFE0F  Comparison budget exhausted (${maxComparisons.toLocaleString()} comparisons). Use --max-comparisons to increase.`);
   }
   return duplicates.sort(
     (a, b) => b.similarity - a.similarity || b.tokenCost - a.tokenCost
@@ -157,7 +309,17 @@ function getRefactoringSuggestion(patternType, similarity) {
   return baseMessages[patternType] + urgency;
 }
 async function analyzePatterns(options) {
-  const { minSimilarity = 0.85, minLines = 5, ...scanOptions } = options;
+  const {
+    minSimilarity = 0.4,
+    // Jaccard similarity default (40% threshold)
+    minLines = 5,
+    batchSize = 100,
+    approx = true,
+    minSharedTokens = 8,
+    maxCandidatesPerBlock = 100,
+    streamResults = false,
+    ...scanOptions
+  } = options;
   const files = await (0, import_core2.scanFiles)(scanOptions);
   const results = [];
   const fileContents = await Promise.all(
@@ -166,9 +328,14 @@ async function analyzePatterns(options) {
       content: await (0, import_core2.readFileContent)(file)
     }))
   );
-  const duplicates = detectDuplicatePatterns(fileContents, {
+  const duplicates = await detectDuplicatePatterns(fileContents, {
     minSimilarity,
-    minLines
+    minLines,
+    batchSize,
+    approx,
+    minSharedTokens,
+    maxCandidatesPerBlock,
+    streamResults
   });
   for (const file of files) {
     const fileDuplicates = duplicates.filter(
@@ -231,8 +398,21 @@ function generateSummary(results) {
     const typeMatch = issue.message.match(/^(\S+(?:-\S+)*) pattern/);
     const fileMatch = issue.message.match(/similar to (.+?) \(/);
     return {
-      file1: issue.location.file,
-      file2: fileMatch?.[1] || "unknown",
+      files: [
+        {
+          path: issue.location.file,
+          startLine: issue.location.line,
+          endLine: 0
+          // Not available from Issue
+        },
+        {
+          path: fileMatch?.[1] || "unknown",
+          startLine: 0,
+          // Not available from Issue
+          endLine: 0
+          // Not available from Issue
+        }
+      ],
       similarity: similarityMatch ? parseInt(similarityMatch[1]) / 100 : 0,
       patternType: typeMatch?.[1] || "unknown",
       tokenCost: tokenMatch ? parseInt(tokenMatch[1]) : 0

package/dist/index.mjs

@@ -2,7 +2,7 @@ import {
   analyzePatterns,
   detectDuplicatePatterns,
   generateSummary
-} from "./chunk-RLWJXASG.mjs";
+} from "./chunk-JKVKOXYR.mjs";
 export {
   analyzePatterns,
   detectDuplicatePatterns,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aiready/pattern-detect",
-  "version": "0.1.2",
+  "version": "0.2.0",
   "description": "Semantic duplicate pattern detection for AI-generated code - finds similar implementations that waste AI context tokens",
   "main": "./dist/index.js",
   "module": "./dist/index.mjs",
@@ -15,6 +15,13 @@
       "import": "./dist/index.mjs"
     }
   },
+  "scripts": {
+    "build": "tsup src/index.ts src/cli.ts --format cjs,esm --dts",
+    "dev": "tsup src/index.ts src/cli.ts --format cjs,esm --dts --watch",
+    "test": "vitest run",
+    "lint": "eslint src",
+    "clean": "rm -rf dist"
+  },
   "keywords": [
     "aiready",
     "duplicate-detection",
@@ -43,9 +50,9 @@
     "url": "https://github.com/caopengau/aiready-pattern-detect/issues"
   },
   "dependencies": {
+    "@aiready/core": "workspace:*",
     "commander": "^12.1.0",
-    "chalk": "^5.3.0",
-    "@aiready/core": "0.1.2"
+    "chalk": "^5.3.0"
   },
   "devDependencies": {
     "tsup": "^8.3.5",
@@ -62,12 +69,5 @@
   },
   "publishConfig": {
     "access": "public"
-  },
-  "scripts": {
-    "build": "tsup src/index.ts src/cli.ts --format cjs,esm --dts",
-    "dev": "tsup src/index.ts src/cli.ts --format cjs,esm --dts --watch",
-    "test": "vitest run",
-    "lint": "eslint src",
-    "clean": "rm -rf dist"
   }
-}
+}

package/dist/chunk-K5O2HVB5.mjs

@@ -1,114 +0,0 @@
-// src/index.ts
-import { scanFiles, readFileContent } from "@aiready/core";
-
-// src/detector.ts
-import { similarityScore } from "@aiready/core";
-function extractCodeBlocks(content, minLines) {
-  const lines = content.split("\n");
-  const blocks = [];
-  let currentBlock = [];
-  let blockStart = 0;
-  let braceDepth = 0;
-  for (let i = 0; i < lines.length; i++) {
-    const line = lines[i];
-    for (const char of line) {
-      if (char === "{") braceDepth++;
-      if (char === "}") braceDepth--;
-    }
-    currentBlock.push(line);
-    if (braceDepth === 0 && currentBlock.length >= minLines) {
-      blocks.push({
-        content: currentBlock.join("\n"),
-        startLine: blockStart + 1
-      });
-      currentBlock = [];
-      blockStart = i + 1;
-    } else if (braceDepth === 0) {
-      currentBlock = [];
-      blockStart = i + 1;
-    }
-  }
-  return blocks;
-}
-function normalizeCode(code) {
-  return code.replace(/\/\/.*$/gm, "").replace(/\/\*[\s\S]*?\*\//g, "").replace(/\s+/g, " ").trim();
-}
-function detectDuplicatePatterns(files, options) {
-  const { minSimilarity, minLines } = options;
-  const duplicates = [];
-  const allBlocks = files.flatMap(
-    (file) => extractCodeBlocks(file.content, minLines).map((block) => ({
-      ...block,
-      file: file.file,
-      normalized: normalizeCode(block.content)
-    }))
-  );
-  for (let i = 0; i < allBlocks.length; i++) {
-    for (let j = i + 1; j < allBlocks.length; j++) {
-      const block1 = allBlocks[i];
-      const block2 = allBlocks[j];
-      if (block1.file === block2.file) continue;
-      const similarity = similarityScore(block1.normalized, block2.normalized);
-      if (similarity >= minSimilarity) {
-        duplicates.push({
-          file1: block1.file,
-          file2: block2.file,
-          line1: block1.startLine,
-          line2: block2.startLine,
-          similarity,
-          snippet: block1.content.split("\n").slice(0, 3).join("\n") + "..."
-        });
-      }
-    }
-  }
-  return duplicates.sort((a, b) => b.similarity - a.similarity);
-}
-
-// src/index.ts
-async function analyzePatterns(options) {
-  const {
-    minSimilarity = 0.85,
-    minLines = 5,
-    ...scanOptions
-  } = options;
-  const files = await scanFiles(scanOptions);
-  const results = [];
-  const fileContents = await Promise.all(
-    files.map(async (file) => ({
-      file,
-      content: await readFileContent(file)
-    }))
-  );
-  const duplicates = detectDuplicatePatterns(fileContents, {
-    minSimilarity,
-    minLines
-  });
-  for (const file of files) {
-    const fileDuplicates = duplicates.filter(
-      (dup) => dup.file1 === file || dup.file2 === file
-    );
-    const issues = fileDuplicates.map((dup) => ({
-      type: "duplicate-pattern",
-      severity: dup.similarity > 0.95 ? "critical" : "major",
-      message: `Similar pattern found in ${dup.file1 === file ? dup.file2 : dup.file1}`,
-      location: {
-        file,
-        line: dup.file1 === file ? dup.line1 : dup.line2
-      },
-      suggestion: "Consider extracting common logic into a shared utility"
-    }));
-    results.push({
-      fileName: file,
-      issues,
-      metrics: {
-        consistencyScore: 1 - fileDuplicates.length * 0.1
-      }
-    });
-  }
-  return results;
-}
-
-export {
-  detectDuplicatePatterns,
-  analyzePatterns
-};