dslop 1.5.0 → 1.5.2

package/CHANGELOG.md

@@ -1,5 +1,29 @@
 # Changelog
 
+## v1.5.2
+
+[compare changes](https://github.com/turf-sports/dslop/compare/v1.5.1...v1.5.2)
+
+### 📖 Documentation
+
+- Update README with AST-based detection details ([2bdb4f4](https://github.com/turf-sports/dslop/commit/2bdb4f4))
+
+### ❤️ Contributors
+
+- Siddharth Sharma <sharmasiddharthcs@gmail.com>
+
+## v1.5.1
+
+[compare changes](https://github.com/turf-sports/dslop/compare/v1.5.0...v1.5.1)
+
+### 🩹 Fixes
+
+- Resolve all lint errors ([0de0b6c](https://github.com/turf-sports/dslop/commit/0de0b6c))
+
+### ❤️ Contributors
+
+- Siddharth Sharma <sharmasiddharthcs@gmail.com>
+
 ## v1.5.0
 
 [compare changes](https://github.com/turf-sports/dslop/compare/v1.4.1...v1.5.0)

package/README.md

@@ -36,26 +36,42 @@ dslop --all --cross-package  # cross-package dupes (monorepos)
 
 ## How it works
 
-**Block extraction:** Sliding window over source files. Extracts overlapping blocks at sizes 4, 6, 9, 13... lines. For blocks <10 lines, step=1 (every line). Larger blocks use step=blockSize/2.
+dslop uses two detection methods in parallel:
 
-**Normalization:** Before hashing, code is normalized:
+### 1. AST-based detection (functions/classes)
+
+Parses TypeScript/JavaScript with Babel to extract functions and classes. Normalizes the AST by replacing all identifiers with generic placeholders (`$0`, `$1`, etc.), preserving only the code structure.
+
+**This catches:**
+- Functions with identical logic but different variable names
+- Renamed copies of existing functions
+- Structurally identical classes
+
+Example: `calculateSum(numbers)` and `computeTotal(items)` with the same loop structure will match.
+
+### 2. Text-based detection (code blocks)
+
+Sliding window over source files extracts overlapping blocks at sizes 4, 6, 9, 13... lines. Before hashing, code is normalized:
 - String literals → `"<STRING>"`
 - Numbers → `<NUMBER>`
 - Whitespace collapsed
 - Comments preserved (intentional - comments often indicate copy-paste)
 
-**Matching:** Normalized blocks are hashed. Exact hash matches = exact duplicates. For similar (non-exact) matches, uses character-level similarity on a sample of blocks per hash bucket.
+Exact hash matches = exact duplicates. For similar (non-exact) matches, uses character-level similarity.
+
+### Changed-line filtering (default mode)
+
+Parses `git diff` output to get exact line ranges of your changes. Only reports duplicates where your changed lines match code elsewhere in the codebase.
 
-**Declaration detection** (`--all` mode): Regex-based extraction of types, interfaces, functions, classes. Compares by name similarity (Levenshtein + word overlap) and content similarity.
+### Declaration detection (`--all` mode)
 
-**Changed-line filtering** (default mode): Parses `git diff` output to get exact line ranges. Only reports duplicates where your changed lines match code elsewhere.
+Regex-based extraction of types, interfaces, enums. Compares by name similarity (Levenshtein + word overlap) and content similarity.
 
 ## Limitations
 
-- **Text-based, not AST:** Doesn't understand code structure. A reformatted function won't match the original. Two semantically identical functions with different variable names won't match.
-- **TypeScript/JavaScript focused:** Default extensions are ts/tsx/js/jsx. Works on any text but tuned for JS-like syntax.
+- **TypeScript/JavaScript only for AST:** AST parsing uses Babel with TS/JSX plugins. Other languages fall back to text-based only.
 - **No cross-language:** Won't detect a Python function duplicated in TypeScript.
-- **Comments affect matching:** Intentional tradeoff. Copy-pasted code often includes comments.
+- **Comments affect text matching:** Intentional tradeoff. Copy-pasted code often includes comments.
 - **Declaration detection is regex:** Can miss edge cases like multi-line generics or decorators.
 - **Minimum 4 lines:** Shorter duplicates ignored to reduce noise. Use `-m 2` for stricter.
 - **Memory:** Loads all blocks in memory. Very large codebases (>1M lines) may be slow.

package/dist/index.cjs

@@ -43566,9 +43566,9 @@ var require_lib8 = __commonJS({
 });
 
 // index.ts
-var import_node_util = require("node:util");
 var import_node_child_process = require("node:child_process");
 var import_node_path4 = __toESM(require("node:path"), 1);
+var import_node_util = require("node:util");
 
 // src/constants.ts
 var DEFAULT_MIN_LINES = 4;
@@ -43751,6 +43751,7 @@ function extractDeclarations(content, filePath) {
   let lineStart = 0;
   for (let lineIndex = 0; lineIndex < lines.length; lineIndex++) {
     const line = lines[lineIndex];
+    if (!line) continue;
     const trimmed = line.trimStart();
     if (!QUICK_CHECK.test(trimmed)) {
       lineStart += line.length + 1;
@@ -43842,27 +43843,27 @@ function splitCamelCase(str) {
 function levenshteinDistance(a, b) {
   if (a.length === 0) return b.length;
   if (b.length === 0) return a.length;
-  const matrix = [];
-  for (let i = 0; i <= b.length; i++) {
-    matrix[i] = [i];
-  }
-  for (let j = 0; j <= a.length; j++) {
-    matrix[0][j] = j;
-  }
+  const matrix = Array.from(
+    { length: b.length + 1 },
+    (_, i) => Array.from({ length: a.length + 1 }, (_2, j) => i === 0 ? j : j === 0 ? i : 0)
+  );
   for (let i = 1; i <= b.length; i++) {
     for (let j = 1; j <= a.length; j++) {
+      const prevRow = matrix[i - 1];
+      const currRow = matrix[i];
+      if (!prevRow || !currRow) continue;
       if (b[i - 1] === a[j - 1]) {
-        matrix[i][j] = matrix[i - 1][j - 1];
+        currRow[j] = prevRow[j - 1] ?? 0;
       } else {
-        matrix[i][j] = Math.min(
-          matrix[i - 1][j - 1] + 1,
-          matrix[i][j - 1] + 1,
-          matrix[i - 1][j] + 1
+        currRow[j] = Math.min(
+          (prevRow[j - 1] ?? 0) + 1,
+          (currRow[j - 1] ?? 0) + 1,
+          (prevRow[j] ?? 0) + 1
         );
       }
     }
   }
-  return matrix[b.length][a.length];
+  return matrix[b.length]?.[a.length] ?? Math.max(a.length, b.length);
 }
 
 // src/detector.ts
@@ -44099,7 +44100,7 @@ function filterOverlappingBlocks(blocks) {
   return result;
 }
 function extractPackageInfo(filePath) {
-  const match2 = filePath.match(/(apps|packages|libs)\/([^\/]+)\/(.+)/);
+  const match2 = filePath.match(/(apps|packages|libs)\/([^/]+)\/(.+)/);
   if (match2) {
     return {
       type: match2[1],
@@ -44114,7 +44115,7 @@ function extractPackageInfo(filePath) {
     subPath: parts.slice(1).join("/")
   };
 }
-function inferSuggestedName(pattern, matches) {
+function inferSuggestedName(_pattern, matches) {
   const firstMatch = matches[0];
   if (!firstMatch) return void 0;
   const content = firstMatch.content;
@@ -44159,7 +44160,7 @@ function inferSuggestedName(pattern, matches) {
   }
   return void 0;
 }
-function generateRefactoringSuggestion(group, basePath) {
+function generateRefactoringSuggestion(group, _basePath) {
   const matches = group.matches;
   const packageInfos = matches.map((m) => extractPackageInfo(m.filePath));
   const uniquePackages = /* @__PURE__ */ new Map();
@@ -44171,7 +44172,7 @@ function generateRefactoringSuggestion(group, basePath) {
   }
   const packageList = Array.from(uniquePackages.values());
   const suggestedName = inferSuggestedName(group.pattern, matches);
-  if (packageList.length === 1) {
+  if (packageList.length === 1 && packageList[0]) {
     const pkg = packageList[0];
     const subPaths = packageInfos.map((p) => p.subPath);
     const commonPrefix = findCommonPrefix(subPaths);
@@ -44207,6 +44208,7 @@ function generateRefactoringSuggestion(group, basePath) {
     const sharedPackage = packages.find(
       (p) => p.name === "shared" || p.name === "common" || p.name === "utils"
     ) ?? packages[0];
+    if (!sharedPackage) return { targetLocation: "packages/shared/src", reason: "Duplicated across packages", confidence: "low", suggestedName };
     return {
       targetLocation: `packages/${sharedPackage.name}/src`,
       reason: `Already exists in ${sharedPackage.name}, also duplicated in ${apps.map((a) => a.name).join(", ")}`,
@@ -44291,6 +44293,7 @@ function findDeclarationDuplicates(declarations, minSimilarity) {
     for (let i = 0; i < typeDecls.length; i++) {
       if (processed.has(i)) continue;
       const declA = typeDecls[i];
+      if (!declA) continue;
       const matches = [{
         name: declA.name,
         filePath: declA.filePath,
@@ -44304,6 +44307,7 @@ function findDeclarationDuplicates(declarations, minSimilarity) {
       for (let j = i + 1; j < typeDecls.length; j++) {
         if (processed.has(j)) continue;
         const declB = typeDecls[j];
+        if (!declB) continue;
         if (declA.filePath === declB.filePath && Math.abs(declA.startLine - declB.startLine) < 5) continue;
         const nameSim = calculateNameSimilarity(declA.name, declB.name);
         const contentSim = calculateSimilarityFast(declA.normalized, declB.normalized);
@@ -44414,7 +44418,7 @@ function normalizeNode(node, idMap, counter) {
     if (!idMap.has(name)) {
       idMap.set(name, `$${counter.val++}`);
     }
-    return idMap.get(name);
+    return idMap.get(name) ?? name;
   }
   if (t.isStringLiteral(node)) return '"S"';
   if (t.isNumericLiteral(node)) return "N";
@@ -44424,7 +44428,7 @@ function normalizeNode(node, idMap, counter) {
   if (t.isFile(node)) return normalizeNode(node.program, idMap, counter);
   if (t.isProgram(node)) return node.body.map((n) => normalizeNode(n, idMap, counter)).join(";");
   if (t.isBlockStatement(node)) {
-    return "{" + node.body.map((n) => normalizeNode(n, idMap, counter)).join(";") + "}";
+    return `{${node.body.map((n) => normalizeNode(n, idMap, counter)).join(";")}}`;
   }
   if (t.isFunctionDeclaration(node) || t.isFunctionExpression(node)) {
     const params = node.params.map((p) => normalizeNode(p, idMap, counter)).join(",");
@@ -44439,20 +44443,20 @@ function normalizeNode(node, idMap, counter) {
     return `${async}(${params})=>${body}`;
   }
   if (t.isVariableDeclaration(node)) {
-    return node.kind + " " + node.declarations.map((d) => normalizeNode(d, idMap, counter)).join(",");
+    return `${node.kind} ${node.declarations.map((d) => normalizeNode(d, idMap, counter)).join(",")}`;
   }
   if (t.isVariableDeclarator(node)) {
     const id = normalizeNode(node.id, idMap, counter);
-    const init = node.init ? "=" + normalizeNode(node.init, idMap, counter) : "";
-    return id + init;
+    const init = node.init ? `=${normalizeNode(node.init, idMap, counter)}` : "";
+    return `${id}${init}`;
   }
   if (t.isReturnStatement(node)) {
-    return "return " + (node.argument ? normalizeNode(node.argument, idMap, counter) : "");
+    return `return ${node.argument ? normalizeNode(node.argument, idMap, counter) : ""}`;
   }
   if (t.isIfStatement(node)) {
     const test = normalizeNode(node.test, idMap, counter);
     const consequent = normalizeNode(node.consequent, idMap, counter);
-    const alternate = node.alternate ? " else " + normalizeNode(node.alternate, idMap, counter) : "";
+    const alternate = node.alternate ? ` else ${normalizeNode(node.alternate, idMap, counter)}` : "";
     return `if(${test})${consequent}${alternate}`;
   }
   if (t.isForStatement(node)) {
@@ -44474,7 +44478,7 @@ function normalizeNode(node, idMap, counter) {
   }
   if (t.isTryStatement(node)) {
     const block = normalizeNode(node.block, idMap, counter);
-    const handler = node.handler ? `catch(${normalizeNode(node.handler.param, idMap, counter)})${normalizeNode(node.handler.body, idMap, counter)}` : "";
+    const handler = node.handler ? `catch(${node.handler.param ? normalizeNode(node.handler.param, idMap, counter) : ""})${normalizeNode(node.handler.body, idMap, counter)}` : "";
     const finalizer = node.finalizer ? `finally${normalizeNode(node.finalizer, idMap, counter)}` : "";
     return `try${block}${handler}${finalizer}`;
   }
@@ -44541,7 +44545,7 @@ function normalizeNode(node, idMap, counter) {
     const key = normalizeNode(node.key, idMap, counter);
     const params = node.params.map((p) => normalizeNode(p, idMap, counter)).join(",");
     const body = normalizeNode(node.body, idMap, counter);
-    const kind = node.kind !== "method" ? node.kind + " " : "";
+    const kind = node.kind !== "method" ? `${node.kind} ` : "";
     const async = node.async ? "async " : "";
     const static_ = node.static ? "static " : "";
     return `${static_}${async}${kind}${key}(${params})${body}`;
@@ -44582,7 +44586,7 @@ function extractASTBlocks(content, filePath) {
     traverse(ast, {
       FunctionDeclaration(path5) {
         const node = path5.node;
-        if (!node.id || !node.loc) return;
+        if (!node.id || !node.loc || node.start === null || node.end === null) return;
         const idMap = /* @__PURE__ */ new Map();
         const counter = { val: 0 };
         const normalized = normalizeNode(node, idMap, counter);
@@ -44602,6 +44606,7 @@ function extractASTBlocks(content, filePath) {
         const node = path5.node;
         for (const decl of node.declarations) {
           if (!t.isIdentifier(decl.id) || !decl.init || !decl.loc) continue;
+          if (decl.start === null || decl.end === null) continue;
           if (!t.isArrowFunctionExpression(decl.init) && !t.isFunctionExpression(decl.init)) continue;
           const idMap = /* @__PURE__ */ new Map();
           const counter = { val: 0 };
@@ -44621,7 +44626,7 @@ function extractASTBlocks(content, filePath) {
       },
       ClassDeclaration(path5) {
         const node = path5.node;
-        if (!node.id || !node.loc) return;
+        if (!node.id || !node.loc || node.start === null || node.end === null) return;
         const idMap = /* @__PURE__ */ new Map();
         const counter = { val: 0 };
         const normalized = normalizeNode(node, idMap, counter);
@@ -44639,13 +44644,14 @@ function extractASTBlocks(content, filePath) {
       },
       TSTypeAliasDeclaration(path5) {
         const node = path5.node;
-        if (!node.loc) return;
+        if (!node.loc || node.start === null || node.end === null) return;
+        const nodeContent = content.slice(node.start, node.end);
         blocks.push({
           type: "type",
           name: node.id.name,
-          content: content.slice(node.start, node.end),
-          normalized: content.slice(node.start, node.end).replace(/\s+/g, " "),
-          hash: simpleHash2(content.slice(node.start, node.end).replace(/\s+/g, " ")),
+          content: nodeContent,
+          normalized: nodeContent.replace(/\s+/g, " "),
+          hash: simpleHash2(nodeContent.replace(/\s+/g, " ")),
           filePath,
           startLine: node.loc.start.line,
           endLine: node.loc.end.line,
@@ -44654,13 +44660,14 @@ function extractASTBlocks(content, filePath) {
       },
       TSInterfaceDeclaration(path5) {
         const node = path5.node;
-        if (!node.loc) return;
+        if (!node.loc || node.start === null || node.end === null) return;
+        const nodeContent = content.slice(node.start, node.end);
         blocks.push({
           type: "interface",
           name: node.id.name,
-          content: content.slice(node.start, node.end),
-          normalized: content.slice(node.start, node.end).replace(/\s+/g, " "),
-          hash: simpleHash2(content.slice(node.start, node.end).replace(/\s+/g, " ")),
+          content: nodeContent,
+          normalized: nodeContent.replace(/\s+/g, " "),
+          hash: simpleHash2(nodeContent.replace(/\s+/g, " ")),
           filePath,
           startLine: node.loc.start.line,
           endLine: node.loc.end.line,
@@ -44672,7 +44679,7 @@ function extractASTBlocks(content, filePath) {
   }
   return blocks;
 }
-function findASTDuplicates(blocks, minSimilarity) {
+function findASTDuplicates(blocks, _minSimilarity) {
   const groups = [];
   const hashGroups = /* @__PURE__ */ new Map();
   for (const block of blocks) {
@@ -44681,7 +44688,7 @@ function findASTDuplicates(blocks, minSimilarity) {
     hashGroups.set(block.hash, existing);
   }
   let groupId = 0;
-  for (const [_hash, matches] of hashGroups) {
+  for (const [, matches] of hashGroups) {
     if (matches.length < 2) continue;
     const uniqueMatches = matches.filter(
       (m, i) => !matches.slice(0, i).some(
@@ -44690,6 +44697,7 @@ function findASTDuplicates(blocks, minSimilarity) {
     );
     if (uniqueMatches.length < 2) continue;
     const firstMatch = uniqueMatches[0];
+    if (!firstMatch) continue;
     groups.push({
       id: groupId++,
       type: firstMatch.type,
@@ -44916,6 +44924,7 @@ function formatASTDuplicates(groups, basePath) {
   lines.push("");
   for (let i = 0; i < Math.min(groups.length, 20); i++) {
     const group = groups[i];
+    if (!group) continue;
     const typeLabel = AST_TYPE_LABELS[group.type] || group.type;
     lines.push(`${bold}${typeLabel} ${i + 1}${reset} \u2502 ${red}${bold}IDENTICAL${reset} \u2502 ${group.matches.length} occurrences`);
     lines.push("");
@@ -44951,6 +44960,10 @@ function formatASTDuplicates(groups, basePath) {
   return lines.join("\n");
 }
 
+// src/scanner.ts
+var import_promises2 = require("node:fs/promises");
+var import_node_path3 = __toESM(require("node:path"), 1);
+
 // node_modules/@isaacs/balanced-match/dist/esm/index.js
 var balanced = (a, b, str) => {
   const ma = a instanceof RegExp ? maybeMatch(a, str) : a;
@@ -51602,8 +51615,6 @@ var glob = Object.assign(glob_, {
 glob.glob = glob;
 
 // src/scanner.ts
-var import_node_path3 = __toESM(require("node:path"), 1);
-var import_promises2 = require("node:fs/promises");
 function simpleHash3(str) {
   let hash = 0;
   for (let i = 0; i < str.length; i++) {
@@ -51713,7 +51724,7 @@ async function scanDirectory(targetPath, options, enableAST = true) {
 }
 
 // index.ts
-var VERSION = process.env.npm_package_version || "1.5.0";
+var VERSION = process.env.npm_package_version || "1.5.2";
 function parseDiffOutput(diff, cwd) {
   const changes = /* @__PURE__ */ new Map();
   let currentFile = null;
@@ -51726,12 +51737,12 @@ function parseDiffOutput(diff, cwd) {
       }
     } else if (line.startsWith("@@") && currentFile) {
       const match2 = line.match(/@@ -\d+(?:,\d+)? \+(\d+)(?:,\d+)? @@/);
-      if (match2) {
+      if (match2?.[1]) {
         newLineNum = parseInt(match2[1], 10);
       }
     } else if (currentFile && newLineNum > 0) {
       if (line.startsWith("+") && !line.startsWith("+++")) {
-        const ranges = changes.get(currentFile);
+        const ranges = changes.get(currentFile) ?? [];
         const lastRange = ranges[ranges.length - 1];
         if (lastRange && lastRange.end === newLineNum - 1) {
           lastRange.end = newLineNum;
@@ -51752,10 +51763,9 @@ function getChangedLines(targetPath) {
   const allChanges = /* @__PURE__ */ new Map();
   const mergeChanges = (newChanges) => {
     for (const [file, ranges] of newChanges) {
-      if (!allChanges.has(file)) {
-        allChanges.set(file, []);
-      }
-      allChanges.get(file).push(...ranges);
+      const existing = allChanges.get(file) ?? [];
+      existing.push(...ranges);
+      allChanges.set(file, existing);
     }
   };
   const addFullFile = (filePath) => {
@@ -51885,7 +51895,7 @@ async function main() {
   }
   console.log(`
 Scanning ${targetPath}...`);
-  if (!scanAll) {
+  if (!scanAll && changedLines) {
     console.log(`  Mode: checking changed lines in ${changedLines.size} files`);
   } else {
     console.log(`  Mode: full codebase scan`);
@@ -51952,7 +51962,7 @@ Scanning ${targetPath}...`);
         return groups.filter((group) => {
           const packages = new Set(
             group.matches.map((m) => {
-              const match2 = m.filePath.match(/(?:apps|packages|libs)\/([^\/]+)/);
+              const match2 = m.filePath.match(/(?:apps|packages|libs)\/([^/]+)/);
               return match2 ? match2[1] : m.filePath.split("/")[0];
             })
           );

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dslop",
-  "version": "1.5.0",
+  "version": "1.5.2",
   "description": "Detect Similar/Duplicate Lines Of Programming - Find code duplication in your codebase",
   "type": "module",
   "main": "./dist/index.cjs",
@@ -44,6 +44,7 @@
     "node": ">=18"
   },
   "devDependencies": {
+    "@types/babel__traverse": "^7.28.0",
     "@types/bun": "latest",
     "changelogen": "^0.6.2",
     "esbuild": "^0.27.2"