@aiready/pattern-detect 0.16.19 → 0.16.20

package/dist/analyzer-entry.js

@@ -182,144 +182,27 @@ function calculateSeverity(file1, file2, code, similarity, linesOfCode) {
   }
 }
 
-// src/detector.ts
+// src/core/normalizer.ts
 function normalizeCode(code, isPython = false) {
+  if (!code) return "";
   let normalized = code;
   if (isPython) {
     normalized = normalized.replace(/#.*/g, "");
   } else {
-    normalized = normalized.replace(/\/\/.*/g, "").replace(/\/\*[\s\S]*?\*\//g, "");
+    normalized = normalized.replace(/\/\/.*$/gm, "").replace(/\/\*[\s\S]*?\*\//g, "");
   }
-  return normalized.replace(/['"`]/g, '"').replace(/\s+/g, " ").trim().toLowerCase();
+  return normalized.replace(/"[^"]*"/g, '"STR"').replace(/'[^']*'/g, "'STR'").replace(/`[^`]*`/g, "`STR`").replace(/\b\d+\b/g, "NUM").replace(/\s+/g, " ").trim().toLowerCase();
 }
+
+// src/detector.ts
 function extractBlocks(file, content) {
-  const isPython = file.toLowerCase().endsWith(".py");
-  if (isPython) {
-    return extractBlocksPython(file, content);
-  }
-  const blocks = [];
-  const lines = content.split("\n");
-  const blockRegex = /^\s*(?:export\s+)?(?:async\s+)?(?:public\s+|private\s+|protected\s+|internal\s+|static\s+|readonly\s+|virtual\s+|abstract\s+|override\s+)*(function|class|interface|type|enum|record|struct|void|func|[a-zA-Z0-9_<>[]]+)\s+([a-zA-Z0-9_]+)(?:\s*\(|(?:\s+extends|\s+implements|\s+where)?\s*\{)|^\s*(?:export\s+)?const\s+([a-zA-Z0-9_]+)\s*=\s*[a-zA-Z0-9_.]+\.object\(|^\s*(app\.(?:get|post|put|delete|patch|use))\(/gm;
-  let match;
-  while ((match = blockRegex.exec(content)) !== null) {
-    const startLine = content.substring(0, match.index).split("\n").length;
-    let type;
-    let name;
-    if (match[1]) {
-      type = match[1];
-      name = match[2];
-    } else if (match[3]) {
-      type = "const";
-      name = match[3];
-    } else {
-      type = "handler";
-      name = match[4];
-    }
-    let endLine = -1;
-    let openBraces = 0;
-    let foundStart = false;
-    for (let i = match.index; i < content.length; i++) {
-      if (content[i] === "{") {
-        openBraces++;
-        foundStart = true;
-      } else if (content[i] === "}") {
-        openBraces--;
-      }
-      if (foundStart && openBraces === 0) {
-        endLine = content.substring(0, i + 1).split("\n").length;
-        break;
-      }
-    }
-    if (endLine === -1) {
-      const remaining = content.slice(match.index);
-      const nextLineMatch = remaining.indexOf("\n");
-      if (nextLineMatch !== -1) {
-        endLine = startLine;
-      } else {
-        endLine = lines.length;
-      }
-    }
-    endLine = Math.max(startLine, endLine);
-    const blockCode = lines.slice(startLine - 1, endLine).join("\n");
-    const tokens = (0, import_core2.estimateTokens)(blockCode);
-    blocks.push({
-      file,
-      startLine,
-      endLine,
-      code: blockCode,
-      tokens,
-      patternType: inferPatternType(type, name)
-    });
-  }
-  return blocks;
-}
-function extractBlocksPython(file, content) {
-  const blocks = [];
-  const lines = content.split("\n");
-  const blockRegex = /^\s*(?:async\s+)?(def|class)\s+([a-zA-Z0-9_]+)/gm;
-  let match;
-  while ((match = blockRegex.exec(content)) !== null) {
-    const startLinePos = content.substring(0, match.index).split("\n").length;
-    const startLineIdx = startLinePos - 1;
-    const initialIndent = lines[startLineIdx].search(/\S/);
-    let endLineIdx = startLineIdx;
-    for (let i = startLineIdx + 1; i < lines.length; i++) {
-      const line = lines[i];
-      if (line.trim().length === 0) {
-        endLineIdx = i;
-        continue;
-      }
-      const currentIndent = line.search(/\S/);
-      if (currentIndent <= initialIndent) {
-        break;
-      }
-      endLineIdx = i;
-    }
-    while (endLineIdx > startLineIdx && lines[endLineIdx].trim().length === 0) {
-      endLineIdx--;
-    }
-    const blockCode = lines.slice(startLineIdx, endLineIdx + 1).join("\n");
-    const tokens = (0, import_core2.estimateTokens)(blockCode);
-    blocks.push({
-      file,
-      startLine: startLinePos,
-      endLine: endLineIdx + 1,
-      code: blockCode,
-      tokens,
-      patternType: inferPatternType(match[1], match[2])
-    });
-  }
-  return blocks;
-}
-function inferPatternType(keyword, name) {
-  const n = name.toLowerCase();
-  if (keyword === "handler" || n.includes("handler") || n.includes("controller") || n.startsWith("app.")) {
-    return "api-handler";
-  }
-  if (n.includes("validate") || n.includes("schema")) return "validator";
-  if (n.includes("util") || n.includes("helper")) return "utility";
-  if (keyword === "class") return "class-method";
-  if (n.match(/^[A-Z]/)) return "component";
-  if (keyword === "function") return "function";
-  return "unknown";
+  return (0, import_core2.extractCodeBlocks)(file, content);
 }
 function calculateSimilarity(a, b) {
-  if (a === b) return 1;
-  const tokensA = a.split(/[^a-zA-Z0-9]+/).filter((t) => t.length > 0);
-  const tokensB = b.split(/[^a-zA-Z0-9]+/).filter((t) => t.length > 0);
-  if (tokensA.length === 0 || tokensB.length === 0) return 0;
-  const setA = new Set(tokensA);
-  const setB = new Set(tokensB);
-  const intersection = new Set([...setA].filter((x) => setB.has(x)));
-  const union = /* @__PURE__ */ new Set([...setA, ...setB]);
-  return intersection.size / union.size;
+  return (0, import_core2.calculateStringSimilarity)(a, b);
 }
 function calculateConfidence(similarity, tokens, lines) {
-  let confidence = similarity;
-  if (lines > 20) confidence += 0.05;
-  if (tokens > 200) confidence += 0.05;
-  if (lines < 5) confidence -= 0.1;
-  return Math.max(0, Math.min(1, confidence));
+  return (0, import_core2.calculateHeuristicConfidence)(similarity, tokens, lines);
 }
 async function detectDuplicatePatterns(fileContents, options) {
   const {

package/dist/analyzer-entry.mjs

@@ -2,8 +2,8 @@ import {
   analyzePatterns,
   generateSummary,
   getSmartDefaults
-} from "./chunk-WMOGJFME.mjs";
-import "./chunk-THF4RW63.mjs";
+} from "./chunk-DNZS4ESD.mjs";
+import "./chunk-VGMM3L3O.mjs";
 import "./chunk-I6ETJC7L.mjs";
 export {
   analyzePatterns,

package/dist/chunk-65UQ5J2J.mjs

@@ -0,0 +1,64 @@
+import {
+  analyzePatterns
+} from "./chunk-JBUZ6YHE.mjs";
+import {
+  calculatePatternScore
+} from "./chunk-WBBO35SC.mjs";
+
+// src/index.ts
+import { ToolRegistry, Severity } from "@aiready/core";
+
+// src/provider.ts
+import {
+  ToolName,
+  SpokeOutputSchema,
+  GLOBAL_SCAN_OPTIONS
+} from "@aiready/core";
+var PatternDetectProvider = {
+  id: ToolName.PatternDetect,
+  alias: ["patterns", "duplicates", "duplication"],
+  async analyze(options) {
+    const results = await analyzePatterns(options);
+    return SpokeOutputSchema.parse({
+      results: results.results,
+      summary: {
+        totalFiles: results.files.length,
+        totalIssues: results.results.reduce(
+          (sum, r) => sum + r.issues.length,
+          0
+        ),
+        duplicates: results.duplicates,
+        // Keep the raw duplicates for score calculation
+        clusters: results.clusters,
+        config: Object.fromEntries(
+          Object.entries(results.config).filter(
+            ([key]) => !GLOBAL_SCAN_OPTIONS.includes(key) || key === "rootDir"
+          )
+        )
+      },
+      metadata: {
+        toolName: ToolName.PatternDetect,
+        version: "0.12.5",
+        timestamp: (/* @__PURE__ */ new Date()).toISOString()
+      }
+    });
+  },
+  score(output, options) {
+    const duplicates = output.summary.duplicates || [];
+    const totalFiles = output.summary.totalFiles || output.results.length;
+    return calculatePatternScore(
+      duplicates,
+      totalFiles,
+      options.costConfig
+    );
+  },
+  defaultWeight: 22
+};
+
+// src/index.ts
+ToolRegistry.register(PatternDetectProvider);
+
+export {
+  PatternDetectProvider,
+  Severity
+};

package/dist/chunk-6JTVOBJX.mjs

@@ -0,0 +1,64 @@
+import {
+  analyzePatterns
+} from "./chunk-DNZS4ESD.mjs";
+import {
+  calculatePatternScore
+} from "./chunk-WBBO35SC.mjs";
+
+// src/index.ts
+import { ToolRegistry, Severity } from "@aiready/core";
+
+// src/provider.ts
+import {
+  ToolName,
+  SpokeOutputSchema,
+  GLOBAL_SCAN_OPTIONS
+} from "@aiready/core";
+var PatternDetectProvider = {
+  id: ToolName.PatternDetect,
+  alias: ["patterns", "duplicates", "duplication"],
+  async analyze(options) {
+    const results = await analyzePatterns(options);
+    return SpokeOutputSchema.parse({
+      results: results.results,
+      summary: {
+        totalFiles: results.files.length,
+        totalIssues: results.results.reduce(
+          (sum, r) => sum + r.issues.length,
+          0
+        ),
+        duplicates: results.duplicates,
+        // Keep the raw duplicates for score calculation
+        clusters: results.clusters,
+        config: Object.fromEntries(
+          Object.entries(results.config).filter(
+            ([key]) => !GLOBAL_SCAN_OPTIONS.includes(key) || key === "rootDir"
+          )
+        )
+      },
+      metadata: {
+        toolName: ToolName.PatternDetect,
+        version: "0.12.5",
+        timestamp: (/* @__PURE__ */ new Date()).toISOString()
+      }
+    });
+  },
+  score(output, options) {
+    const duplicates = output.summary.duplicates || [];
+    const totalFiles = output.summary.totalFiles || output.results.length;
+    return calculatePatternScore(
+      duplicates,
+      totalFiles,
+      options.costConfig
+    );
+  },
+  defaultWeight: 22
+};
+
+// src/index.ts
+ToolRegistry.register(PatternDetectProvider);
+
+export {
+  PatternDetectProvider,
+  Severity
+};

package/dist/chunk-BKRPSTT2.mjs

@@ -0,0 +1,64 @@
+import {
+  analyzePatterns
+} from "./chunk-XNPID6FU.mjs";
+import {
+  calculatePatternScore
+} from "./chunk-WBBO35SC.mjs";
+
+// src/index.ts
+import { ToolRegistry, Severity } from "@aiready/core";
+
+// src/provider.ts
+import {
+  ToolName,
+  SpokeOutputSchema,
+  GLOBAL_SCAN_OPTIONS
+} from "@aiready/core";
+var PatternDetectProvider = {
+  id: ToolName.PatternDetect,
+  alias: ["patterns", "duplicates", "duplication"],
+  async analyze(options) {
+    const results = await analyzePatterns(options);
+    return SpokeOutputSchema.parse({
+      results: results.results,
+      summary: {
+        totalFiles: results.files.length,
+        totalIssues: results.results.reduce(
+          (sum, r) => sum + r.issues.length,
+          0
+        ),
+        duplicates: results.duplicates,
+        // Keep the raw duplicates for score calculation
+        clusters: results.clusters,
+        config: Object.fromEntries(
+          Object.entries(results.config).filter(
+            ([key]) => !GLOBAL_SCAN_OPTIONS.includes(key) || key === "rootDir"
+          )
+        )
+      },
+      metadata: {
+        toolName: ToolName.PatternDetect,
+        version: "0.12.5",
+        timestamp: (/* @__PURE__ */ new Date()).toISOString()
+      }
+    });
+  },
+  score(output, options) {
+    const duplicates = output.summary.duplicates || [];
+    const totalFiles = output.summary.totalFiles || output.results.length;
+    return calculatePatternScore(
+      duplicates,
+      totalFiles,
+      options.costConfig
+    );
+  },
+  defaultWeight: 22
+};
+
+// src/index.ts
+ToolRegistry.register(PatternDetectProvider);
+
+export {
+  PatternDetectProvider,
+  Severity
+};

package/dist/chunk-CMWW24HW.mjs

@@ -0,0 +1,259 @@
+import {
+  calculateSeverity
+} from "./chunk-I6ETJC7L.mjs";
+
+// src/detector.ts
+import { estimateTokens } from "@aiready/core";
+
+// src/core/normalizer.ts
+function normalizeCode(code, isPython = false) {
+  if (!code) return "";
+  let normalized = code;
+  if (isPython) {
+    normalized = normalized.replace(/#.*/g, "");
+  } else {
+    normalized = normalized.replace(/\/\/.*$/gm, "").replace(/\/\*[\s\S]*?\*\//g, "");
+  }
+  return normalized.replace(/"[^"]*"/g, '"STR"').replace(/'[^']*'/g, "'STR'").replace(/`[^`]*`/g, "`STR`").replace(/\b\d+\b/g, "NUM").replace(/\s+/g, " ").trim();
+}
+
+// src/detector.ts
+function extractBlocks(file, content) {
+  const isPython = file.toLowerCase().endsWith(".py");
+  if (isPython) {
+    return extractBlocksPython(file, content);
+  }
+  const blocks = [];
+  const lines = content.split("\n");
+  const blockRegex = /^\s*(?:export\s+)?(?:async\s+)?(?:public\s+|private\s+|protected\s+|internal\s+|static\s+|readonly\s+|virtual\s+|abstract\s+|override\s+)*(function|class|interface|type|enum|record|struct|void|func|[a-zA-Z0-9_<>[]]+)\s+([a-zA-Z0-9_]+)(?:\s*\(|(?:\s+extends|\s+implements|\s+where)?\s*\{)|^\s*(?:export\s+)?const\s+([a-zA-Z0-9_]+)\s*=\s*[a-zA-Z0-9_.]+\.object\(|^\s*(app\.(?:get|post|put|delete|patch|use))\(/gm;
+  let match;
+  while ((match = blockRegex.exec(content)) !== null) {
+    const startLine = content.substring(0, match.index).split("\n").length;
+    let type;
+    let name;
+    if (match[1]) {
+      type = match[1];
+      name = match[2];
+    } else if (match[3]) {
+      type = "const";
+      name = match[3];
+    } else {
+      type = "handler";
+      name = match[4];
+    }
+    let endLine = -1;
+    let openBraces = 0;
+    let foundStart = false;
+    for (let i = match.index; i < content.length; i++) {
+      if (content[i] === "{") {
+        openBraces++;
+        foundStart = true;
+      } else if (content[i] === "}") {
+        openBraces--;
+      }
+      if (foundStart && openBraces === 0) {
+        endLine = content.substring(0, i + 1).split("\n").length;
+        break;
+      }
+    }
+    if (endLine === -1) {
+      const remaining = content.slice(match.index);
+      const nextLineMatch = remaining.indexOf("\n");
+      if (nextLineMatch !== -1) {
+        endLine = startLine;
+      } else {
+        endLine = lines.length;
+      }
+    }
+    endLine = Math.max(startLine, endLine);
+    const blockCode = lines.slice(startLine - 1, endLine).join("\n");
+    const tokens = estimateTokens(blockCode);
+    blocks.push({
+      file,
+      startLine,
+      endLine,
+      code: blockCode,
+      tokens,
+      patternType: inferPatternType(type, name)
+    });
+  }
+  return blocks;
+}
+function extractBlocksPython(file, content) {
+  const blocks = [];
+  const lines = content.split("\n");
+  const blockRegex = /^\s*(?:async\s+)?(def|class)\s+([a-zA-Z0-9_]+)/gm;
+  let match;
+  while ((match = blockRegex.exec(content)) !== null) {
+    const startLinePos = content.substring(0, match.index).split("\n").length;
+    const startLineIdx = startLinePos - 1;
+    const initialIndent = lines[startLineIdx].search(/\S/);
+    let endLineIdx = startLineIdx;
+    for (let i = startLineIdx + 1; i < lines.length; i++) {
+      const line = lines[i];
+      if (line.trim().length === 0) {
+        endLineIdx = i;
+        continue;
+      }
+      const currentIndent = line.search(/\S/);
+      if (currentIndent <= initialIndent) {
+        break;
+      }
+      endLineIdx = i;
+    }
+    while (endLineIdx > startLineIdx && lines[endLineIdx].trim().length === 0) {
+      endLineIdx--;
+    }
+    const blockCode = lines.slice(startLineIdx, endLineIdx + 1).join("\n");
+    const tokens = estimateTokens(blockCode);
+    blocks.push({
+      file,
+      startLine: startLinePos,
+      endLine: endLineIdx + 1,
+      code: blockCode,
+      tokens,
+      patternType: inferPatternType(match[1], match[2])
+    });
+  }
+  return blocks;
+}
+function inferPatternType(keyword, name) {
+  const n = name.toLowerCase();
+  if (keyword === "handler" || n.includes("handler") || n.includes("controller") || n.startsWith("app.")) {
+    return "api-handler";
+  }
+  if (n.includes("validate") || n.includes("schema")) return "validator";
+  if (n.includes("util") || n.includes("helper")) return "utility";
+  if (keyword === "class") return "class-method";
+  if (n.match(/^[A-Z]/)) return "component";
+  if (keyword === "function") return "function";
+  return "unknown";
+}
+function calculateSimilarity(a, b) {
+  if (a === b) return 1;
+  const tokensA = a.split(/[^a-zA-Z0-9]+/).filter((t) => t.length > 0);
+  const tokensB = b.split(/[^a-zA-Z0-9]+/).filter((t) => t.length > 0);
+  if (tokensA.length === 0 || tokensB.length === 0) return 0;
+  const setA = new Set(tokensA);
+  const setB = new Set(tokensB);
+  const intersection = new Set([...setA].filter((x) => setB.has(x)));
+  const union = /* @__PURE__ */ new Set([...setA, ...setB]);
+  return intersection.size / union.size;
+}
+function calculateConfidence(similarity, tokens, lines) {
+  let confidence = similarity;
+  if (lines > 20) confidence += 0.05;
+  if (tokens > 200) confidence += 0.05;
+  if (lines < 5) confidence -= 0.1;
+  return Math.max(0, Math.min(1, confidence));
+}
+async function detectDuplicatePatterns(fileContents, options) {
+  const {
+    minSimilarity,
+    minLines,
+    streamResults,
+    onProgress,
+    excludePatterns = [],
+    confidenceThreshold = 0,
+    ignoreWhitelist = []
+  } = options;
+  const allBlocks = [];
+  const excludeRegexes = excludePatterns.map((p) => new RegExp(p, "i"));
+  for (const { file, content } of fileContents) {
+    const blocks = extractBlocks(file, content);
+    for (const b of blocks) {
+      if (b.endLine - b.startLine + 1 < minLines) continue;
+      const isExcluded = excludeRegexes.some((regex) => regex.test(b.code));
+      if (isExcluded) continue;
+      allBlocks.push(b);
+    }
+  }
+  const duplicates = [];
+  const totalBlocks = allBlocks.length;
+  let comparisons = 0;
+  const totalComparisons = totalBlocks * (totalBlocks - 1) / 2;
+  if (onProgress) {
+    onProgress(
+      0,
+      totalComparisons,
+      `Starting duplicate detection on ${totalBlocks} blocks...`
+    );
+  }
+  for (let i = 0; i < allBlocks.length; i++) {
+    if (i % 50 === 0 && i > 0) {
+      await new Promise((resolve) => setImmediate(resolve));
+      if (onProgress) {
+        onProgress(
+          comparisons,
+          totalComparisons,
+          `Analyzing blocks (${i}/${totalBlocks})...`
+        );
+      }
+    }
+    const b1 = allBlocks[i];
+    const isPython1 = b1.file.toLowerCase().endsWith(".py");
+    const norm1 = normalizeCode(b1.code, isPython1);
+    for (let j = i + 1; j < allBlocks.length; j++) {
+      comparisons++;
+      const b2 = allBlocks[j];
+      if (b1.file === b2.file) continue;
+      const isWhitelisted = ignoreWhitelist.some((pattern) => {
+        return b1.file.includes(pattern) && b2.file.includes(pattern) || pattern === `${b1.file}::${b2.file}` || pattern === `${b2.file}::${b1.file}`;
+      });
+      if (isWhitelisted) continue;
+      const isPython2 = b2.file.toLowerCase().endsWith(".py");
+      const norm2 = normalizeCode(b2.code, isPython2);
+      const sim = calculateSimilarity(norm1, norm2);
+      if (sim >= minSimilarity) {
+        const confidence = calculateConfidence(
+          sim,
+          b1.tokens,
+          b1.endLine - b1.startLine + 1
+        );
+        if (confidence < confidenceThreshold) continue;
+        const { severity, reason, suggestion, matchedRule } = calculateSeverity(
+          b1.file,
+          b2.file,
+          b1.code,
+          sim,
+          b1.endLine - b1.startLine + 1
+        );
+        const dup = {
+          file1: b1.file,
+          line1: b1.startLine,
+          endLine1: b1.endLine,
+          file2: b2.file,
+          line2: b2.startLine,
+          endLine2: b2.endLine,
+          code1: b1.code,
+          code2: b2.code,
+          similarity: sim,
+          confidence,
+          patternType: b1.patternType,
+          tokenCost: b1.tokens + b2.tokens,
+          severity,
+          reason,
+          suggestion,
+          matchedRule
+        };
+        duplicates.push(dup);
+        if (streamResults)
+          console.log(
+            `[DUPLICATE] ${dup.file1}:${dup.line1} <-> ${dup.file2}:${dup.line2} (${Math.round(sim * 100)}%, conf: ${Math.round(confidence * 100)}%)`
+          );
+      }
+    }
+  }
+  if (onProgress) {
+    onProgress(
+      totalComparisons,
+      totalComparisons,
+      `Duplicate detection complete. Found ${duplicates.length} patterns.`
+    );
+  }
+  return duplicates.sort((a, b) => b.similarity - a.similarity);
+}
+
+export {
+  detectDuplicatePatterns
+};