@aiready/pattern-detect 0.1.3 → 0.2.0

package/dist/index.d.ts

@@ -26,7 +26,6 @@ interface DetectionOptions {
     approx?: boolean;
     minSharedTokens?: number;
     maxCandidatesPerBlock?: number;
-    fastMode?: boolean;
     maxComparisons?: number;
     streamResults?: boolean;
 }
@@ -38,13 +37,10 @@ declare function detectDuplicatePatterns(files: FileContent[], options: Detectio
 interface PatternDetectOptions extends ScanOptions {
     minSimilarity?: number;
     minLines?: number;
-    maxBlocks?: number;
     batchSize?: number;
     approx?: boolean;
     minSharedTokens?: number;
     maxCandidatesPerBlock?: number;
-    fastMode?: boolean;
-    maxComparisons?: number;
     streamResults?: boolean;
 }
 interface PatternSummary {
@@ -52,12 +48,11 @@ interface PatternSummary {
     totalTokenCost: number;
     patternsByType: Record<PatternType, number>;
     topDuplicates: Array<{
-        file1: string;
-        file2: string;
-        line1: number;
-        line2: number;
-        endLine1: number;
-        endLine2: number;
+        files: Array<{
+            path: string;
+            startLine: number;
+            endLine: number;
+        }>;
         similarity: number;
         patternType: PatternType;
         tokenCost: number;

package/dist/index.js

@@ -106,31 +106,19 @@ function jaccardSimilarity(tokens1, tokens2) {
   const union = set1.size + set2.size - intersection;
   return union === 0 ? 0 : intersection / union;
 }
-function calculateSimilarity(block1, block2) {
-  const norm1 = normalizeCode(block1);
-  const norm2 = normalizeCode(block2);
-  const baseSimilarity = (0, import_core.similarityScore)(norm1, norm2);
-  const tokens1 = norm1.split(/[\s(){}[\];,]+/).filter(Boolean);
-  const tokens2 = norm2.split(/[\s(){}[\];,]+/).filter(Boolean);
-  const tokenSimilarity = (0, import_core.similarityScore)(tokens1.join(" "), tokens2.join(" "));
-  return baseSimilarity * 0.4 + tokenSimilarity * 0.6;
-}
 async function detectDuplicatePatterns(files, options) {
   const {
     minSimilarity,
     minLines,
-    maxBlocks = 500,
     batchSize = 100,
     approx = true,
     minSharedTokens = 8,
     maxCandidatesPerBlock = 100,
-    fastMode = true,
-    maxComparisons = 5e4,
-    // Cap at 50K comparisons by default
     streamResults = false
   } = options;
   const duplicates = [];
-  let allBlocks = files.flatMap(
+  const maxComparisons = approx ? Infinity : 5e5;
+  const allBlocks = files.flatMap(
     (file) => extractCodeBlocks(file.content, minLines).map((block) => ({
       content: block.content,
       startLine: block.startLine,
@@ -143,10 +131,9 @@ async function detectDuplicatePatterns(files, options) {
     }))
   );
   console.log(`Extracted ${allBlocks.length} code blocks for analysis`);
-  if (allBlocks.length > maxBlocks) {
-    console.log(`\u26A0\uFE0F  Limiting to ${maxBlocks} blocks (sorted by size) to prevent memory issues`);
-    console.log(`   Use --max-blocks to increase limit or --min-lines to filter smaller blocks`);
-    allBlocks = allBlocks.sort((a, b) => b.linesOfCode - a.linesOfCode).slice(0, maxBlocks);
+  if (!approx && allBlocks.length > 500) {
+    console.log(`\u26A0\uFE0F  Using --no-approx mode with ${allBlocks.length} blocks may be slow (O(B\xB2) complexity).`);
+    console.log(`   Consider using approximate mode (default) for better performance.`);
   }
   const stopwords = /* @__PURE__ */ new Set([
     "return",
@@ -236,10 +223,14 @@ async function detectDuplicatePatterns(files, options) {
     }
     if (approx && candidates) {
       for (const { j } of candidates) {
-        if (maxComparisons && comparisonsProcessed >= maxComparisons) break;
+        if (!approx && maxComparisons !== Infinity && comparisonsProcessed >= maxComparisons) {
+          console.log(`\u26A0\uFE0F  Comparison safety limit reached (${maxComparisons.toLocaleString()} comparisons in --no-approx mode).`);
+          console.log(`   This prevents excessive runtime on large repos. Consider using approximate mode (default) or --min-lines to reduce blocks.`);
+          break;
+        }
         comparisonsProcessed++;
         const block2 = allBlocks[j];
-        const similarity = fastMode ? jaccardSimilarity(blockTokens[i], blockTokens[j]) : calculateSimilarity(block1.content, block2.content);
+        const similarity = jaccardSimilarity(blockTokens[i], blockTokens[j]);
         if (similarity >= minSimilarity) {
           const duplicate = {
             file1: block1.file,
@@ -269,7 +260,7 @@ async function detectDuplicatePatterns(files, options) {
         comparisonsProcessed++;
         const block2 = allBlocks[j];
         if (block1.file === block2.file) continue;
-        const similarity = fastMode ? jaccardSimilarity(blockTokens[i], blockTokens[j]) : calculateSimilarity(block1.content, block2.content);
+        const similarity = jaccardSimilarity(blockTokens[i], blockTokens[j]);
         if (similarity >= minSimilarity) {
           const duplicate = {
             file1: block1.file,
@@ -319,16 +310,13 @@ function getRefactoringSuggestion(patternType, similarity) {
 }
 async function analyzePatterns(options) {
   const {
-    minSimilarity = 0.65,
-    // Lower default for fast Jaccard mode (Levenshtein would be 0.85+)
+    minSimilarity = 0.4,
+    // Jaccard similarity default (40% threshold)
     minLines = 5,
-    maxBlocks = 500,
     batchSize = 100,
     approx = true,
     minSharedTokens = 8,
     maxCandidatesPerBlock = 100,
-    fastMode = true,
-    maxComparisons = 5e4,
     streamResults = false,
     ...scanOptions
   } = options;
@@ -343,13 +331,10 @@ async function analyzePatterns(options) {
   const duplicates = await detectDuplicatePatterns(fileContents, {
     minSimilarity,
     minLines,
-    maxBlocks,
     batchSize,
     approx,
     minSharedTokens,
     maxCandidatesPerBlock,
-    fastMode,
-    maxComparisons,
     streamResults
   });
   for (const file of files) {
@@ -413,15 +398,21 @@ function generateSummary(results) {
     const typeMatch = issue.message.match(/^(\S+(?:-\S+)*) pattern/);
     const fileMatch = issue.message.match(/similar to (.+?) \(/);
     return {
-      file1: issue.location.file,
-      file2: fileMatch?.[1] || "unknown",
-      line1: issue.location.line,
-      line2: 0,
-      // Not available from Issue
-      endLine1: 0,
-      // Not available from Issue
-      endLine2: 0,
-      // Not available from Issue
+      files: [
+        {
+          path: issue.location.file,
+          startLine: issue.location.line,
+          endLine: 0
+          // Not available from Issue
+        },
+        {
+          path: fileMatch?.[1] || "unknown",
+          startLine: 0,
+          // Not available from Issue
+          endLine: 0
+          // Not available from Issue
+        }
+      ],
       similarity: similarityMatch ? parseInt(similarityMatch[1]) / 100 : 0,
       patternType: typeMatch?.[1] || "unknown",
       tokenCost: tokenMatch ? parseInt(tokenMatch[1]) : 0

package/dist/index.mjs

@@ -2,7 +2,7 @@ import {
   analyzePatterns,
   detectDuplicatePatterns,
   generateSummary
-} from "./chunk-N5DE7IYX.mjs";
+} from "./chunk-JKVKOXYR.mjs";
 export {
   analyzePatterns,
   detectDuplicatePatterns,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aiready/pattern-detect",
-  "version": "0.1.3",
+  "version": "0.2.0",
   "description": "Semantic duplicate pattern detection for AI-generated code - finds similar implementations that waste AI context tokens",
   "main": "./dist/index.js",
   "module": "./dist/index.mjs",
@@ -15,6 +15,13 @@
       "import": "./dist/index.mjs"
     }
   },
+  "scripts": {
+    "build": "tsup src/index.ts src/cli.ts --format cjs,esm --dts",
+    "dev": "tsup src/index.ts src/cli.ts --format cjs,esm --dts --watch",
+    "test": "vitest run",
+    "lint": "eslint src",
+    "clean": "rm -rf dist"
+  },
   "keywords": [
     "aiready",
     "duplicate-detection",
@@ -43,9 +50,9 @@
     "url": "https://github.com/caopengau/aiready-pattern-detect/issues"
   },
   "dependencies": {
+    "@aiready/core": "workspace:*",
     "commander": "^12.1.0",
-    "chalk": "^5.3.0",
-    "@aiready/core": "0.1.2"
+    "chalk": "^5.3.0"
   },
   "devDependencies": {
     "tsup": "^8.3.5",
@@ -62,12 +69,5 @@
   },
   "publishConfig": {
     "access": "public"
-  },
-  "scripts": {
-    "build": "tsup src/index.ts src/cli.ts --format cjs,esm --dts",
-    "dev": "tsup src/index.ts src/cli.ts --format cjs,esm --dts --watch",
-    "test": "vitest run",
-    "lint": "eslint src",
-    "clean": "rm -rf dist"
   }
-}
+}

package/dist/chunk-6VQTQRDW.mjs

@@ -1,245 +0,0 @@
-// src/index.ts
-import { scanFiles, readFileContent } from "@aiready/core";
-
-// src/detector.ts
-import { similarityScore, estimateTokens } from "@aiready/core";
-function categorizePattern(code) {
-  const lower = code.toLowerCase();
-  if (lower.includes("request") && lower.includes("response") || lower.includes("router.") || lower.includes("app.get") || lower.includes("app.post") || lower.includes("express") || lower.includes("ctx.body")) {
-    return "api-handler";
-  }
-  if (lower.includes("validate") || lower.includes("schema") || lower.includes("zod") || lower.includes("yup") || lower.includes("if") && lower.includes("throw")) {
-    return "validator";
-  }
-  if (lower.includes("return (") || lower.includes("jsx") || lower.includes("component") || lower.includes("props")) {
-    return "component";
-  }
-  if (lower.includes("class ") || lower.includes("this.")) {
-    return "class-method";
-  }
-  if (lower.includes("return ") && !lower.includes("this") && !lower.includes("new ")) {
-    return "utility";
-  }
-  if (lower.includes("function") || lower.includes("=>")) {
-    return "function";
-  }
-  return "unknown";
-}
-function extractCodeBlocks(content, minLines) {
-  const lines = content.split("\n");
-  const blocks = [];
-  let currentBlock = [];
-  let blockStart = 0;
-  let braceDepth = 0;
-  let inFunction = false;
-  for (let i = 0; i < lines.length; i++) {
-    const line = lines[i];
-    const trimmed = line.trim();
-    if (!inFunction && (trimmed.includes("function ") || trimmed.includes("=>") || trimmed.includes("async ") || /^(export\s+)?(async\s+)?function\s+/.test(trimmed) || /^(export\s+)?const\s+\w+\s*=\s*(async\s*)?\(/.test(trimmed))) {
-      inFunction = true;
-      blockStart = i;
-    }
-    for (const char of line) {
-      if (char === "{") braceDepth++;
-      if (char === "}") braceDepth--;
-    }
-    if (inFunction) {
-      currentBlock.push(line);
-    }
-    if (inFunction && braceDepth === 0 && currentBlock.length >= minLines) {
-      const blockContent = currentBlock.join("\n");
-      const linesOfCode = currentBlock.filter(
-        (l) => l.trim() && !l.trim().startsWith("//")
-      ).length;
-      blocks.push({
-        content: blockContent,
-        startLine: blockStart + 1,
-        patternType: categorizePattern(blockContent),
-        linesOfCode
-      });
-      currentBlock = [];
-      inFunction = false;
-    } else if (inFunction && braceDepth === 0) {
-      currentBlock = [];
-      inFunction = false;
-    }
-  }
-  return blocks;
-}
-function normalizeCode(code) {
-  return code.replace(/\/\/.*$/gm, "").replace(/\/\*[\s\S]*?\*\//g, "").replace(/"[^"]*"/g, '"STR"').replace(/'[^']*'/g, "'STR'").replace(/`[^`]*`/g, "`STR`").replace(/\b\d+\b/g, "NUM").replace(/\s+/g, " ").trim();
-}
-function calculateSimilarity(block1, block2) {
-  const norm1 = normalizeCode(block1);
-  const norm2 = normalizeCode(block2);
-  const baseSimilarity = similarityScore(norm1, norm2);
-  const tokens1 = norm1.split(/[\s(){}[\];,]+/).filter(Boolean);
-  const tokens2 = norm2.split(/[\s(){}[\];,]+/).filter(Boolean);
-  const tokenSimilarity = similarityScore(tokens1.join(" "), tokens2.join(" "));
-  return baseSimilarity * 0.4 + tokenSimilarity * 0.6;
-}
-async function detectDuplicatePatterns(files, options) {
-  const { minSimilarity, minLines, maxBlocks = 500, batchSize = 100 } = options;
-  const duplicates = [];
-  let allBlocks = files.flatMap(
-    (file) => extractCodeBlocks(file.content, minLines).map((block) => ({
-      ...block,
-      file: file.file,
-      normalized: normalizeCode(block.content),
-      tokenCost: estimateTokens(block.content)
-    }))
-  );
-  console.log(`Extracted ${allBlocks.length} code blocks for analysis`);
-  if (allBlocks.length > maxBlocks) {
-    console.log(`\u26A0\uFE0F  Limiting to ${maxBlocks} blocks (sorted by size) to prevent memory issues`);
-    console.log(`   Use --max-blocks to increase limit or --min-lines to filter smaller blocks`);
-    allBlocks = allBlocks.sort((a, b) => b.linesOfCode - a.linesOfCode).slice(0, maxBlocks);
-  }
-  const totalComparisons = allBlocks.length * (allBlocks.length - 1) / 2;
-  console.log(`Processing ${totalComparisons.toLocaleString()} comparisons in batches...`);
-  let comparisonsProcessed = 0;
-  const startTime = Date.now();
-  for (let i = 0; i < allBlocks.length; i++) {
-    if (i % batchSize === 0 && i > 0) {
-      const progress = (comparisonsProcessed / totalComparisons * 100).toFixed(1);
-      const elapsed = ((Date.now() - startTime) / 1e3).toFixed(1);
-      console.log(`   ${progress}% complete (${comparisonsProcessed.toLocaleString()}/${totalComparisons.toLocaleString()} comparisons, ${elapsed}s elapsed)`);
-      await new Promise((resolve) => setImmediate(resolve));
-    }
-    for (let j = i + 1; j < allBlocks.length; j++) {
-      comparisonsProcessed++;
-      const block1 = allBlocks[i];
-      const block2 = allBlocks[j];
-      if (block1.file === block2.file) continue;
-      const similarity = calculateSimilarity(block1.content, block2.content);
-      if (similarity >= minSimilarity) {
-        duplicates.push({
-          file1: block1.file,
-          file2: block2.file,
-          line1: block1.startLine,
-          line2: block2.startLine,
-          similarity,
-          snippet: block1.content.split("\n").slice(0, 5).join("\n") + "\n...",
-          patternType: block1.patternType,
-          tokenCost: block1.tokenCost + block2.tokenCost,
-          linesOfCode: block1.linesOfCode
-        });
-      }
-    }
-  }
-  return duplicates.sort(
-    (a, b) => b.similarity - a.similarity || b.tokenCost - a.tokenCost
-  );
-}
-
-// src/index.ts
-function getRefactoringSuggestion(patternType, similarity) {
-  const baseMessages = {
-    "api-handler": "Extract common middleware or create a base handler class",
-    validator: "Consolidate validation logic into shared schema validators (Zod/Yup)",
-    utility: "Move to a shared utilities file and reuse across modules",
-    "class-method": "Consider inheritance or composition to share behavior",
-    component: "Extract shared logic into a custom hook or HOC",
-    function: "Extract into a shared helper function",
-    unknown: "Extract common logic into a reusable module"
-  };
-  const urgency = similarity > 0.95 ? " (CRITICAL: Nearly identical code)" : similarity > 0.9 ? " (HIGH: Very similar, refactor soon)" : "";
-  return baseMessages[patternType] + urgency;
-}
-async function analyzePatterns(options) {
-  const { minSimilarity = 0.85, minLines = 5, maxBlocks = 500, batchSize = 100, ...scanOptions } = options;
-  const files = await scanFiles(scanOptions);
-  const results = [];
-  const fileContents = await Promise.all(
-    files.map(async (file) => ({
-      file,
-      content: await readFileContent(file)
-    }))
-  );
-  const duplicates = await detectDuplicatePatterns(fileContents, {
-    minSimilarity,
-    minLines,
-    maxBlocks,
-    batchSize
-  });
-  for (const file of files) {
-    const fileDuplicates = duplicates.filter(
-      (dup) => dup.file1 === file || dup.file2 === file
-    );
-    const issues = fileDuplicates.map((dup) => {
-      const otherFile = dup.file1 === file ? dup.file2 : dup.file1;
-      const severity = dup.similarity > 0.95 ? "critical" : dup.similarity > 0.9 ? "major" : "minor";
-      return {
-        type: "duplicate-pattern",
-        severity,
-        message: `${dup.patternType} pattern ${Math.round(dup.similarity * 100)}% similar to ${otherFile} (${dup.tokenCost} tokens wasted)`,
-        location: {
-          file,
-          line: dup.file1 === file ? dup.line1 : dup.line2
-        },
-        suggestion: getRefactoringSuggestion(dup.patternType, dup.similarity)
-      };
-    });
-    const totalTokenCost = fileDuplicates.reduce(
-      (sum, dup) => sum + dup.tokenCost,
-      0
-    );
-    results.push({
-      fileName: file,
-      issues,
-      metrics: {
-        tokenCost: totalTokenCost,
-        consistencyScore: Math.max(0, 1 - fileDuplicates.length * 0.1)
-      }
-    });
-  }
-  return results;
-}
-function generateSummary(results) {
-  const allIssues = results.flatMap((r) => r.issues);
-  const totalTokenCost = results.reduce(
-    (sum, r) => sum + (r.metrics.tokenCost || 0),
-    0
-  );
-  const patternsByType = {
-    "api-handler": 0,
-    validator: 0,
-    utility: 0,
-    "class-method": 0,
-    component: 0,
-    function: 0,
-    unknown: 0
-  };
-  allIssues.forEach((issue) => {
-    const match = issue.message.match(/^(\S+(?:-\S+)*) pattern/);
-    if (match) {
-      const type = match[1];
-      patternsByType[type] = (patternsByType[type] || 0) + 1;
-    }
-  });
-  const topDuplicates = allIssues.slice(0, 10).map((issue) => {
-    const similarityMatch = issue.message.match(/(\d+)% similar/);
-    const tokenMatch = issue.message.match(/\((\d+) tokens/);
-    const typeMatch = issue.message.match(/^(\S+(?:-\S+)*) pattern/);
-    const fileMatch = issue.message.match(/similar to (.+?) \(/);
-    return {
-      file1: issue.location.file,
-      file2: fileMatch?.[1] || "unknown",
-      similarity: similarityMatch ? parseInt(similarityMatch[1]) / 100 : 0,
-      patternType: typeMatch?.[1] || "unknown",
-      tokenCost: tokenMatch ? parseInt(tokenMatch[1]) : 0
-    };
-  });
-  return {
-    totalPatterns: allIssues.length,
-    totalTokenCost,
-    patternsByType,
-    topDuplicates
-  };
-}
-
-export {
-  detectDuplicatePatterns,
-  analyzePatterns,
-  generateSummary
-};