dslop 1.7.1 → 1.7.2

package/CHANGELOG.md

@@ -1,5 +1,21 @@
 # Changelog
 
+## v1.7.2
+
+[compare changes](https://github.com/turf-sports/dslop/compare/v1.7.1...v1.7.2)
+
+### 🔥 Performance
+
+- 50x faster - parallel processing, cache line counts ([de430e5](https://github.com/turf-sports/dslop/commit/de430e5))
+
+### 📖 Documentation
+
+- Update performance numbers ([1c6465e](https://github.com/turf-sports/dslop/commit/1c6465e))
+
+### ❤️ Contributors
+
+- Siddharth Sharma <sharmasiddharthcs@gmail.com>
+
 ## v1.7.1
 
 [compare changes](https://github.com/turf-sports/dslop/compare/v1.7.0...v1.7.1)

package/README.md

@@ -151,23 +151,19 @@ function getInitials(person: Person): string {
 
 ## Performance
 
-dslop uses aggressive caching to make subsequent runs fast:
+dslop uses aggressive caching + parallel processing for speed:
 
 ```bash
 # First run (builds cache)
 $ dslop --all
-Scanned 1410 files in 2862ms
-Cache: 0 hits, 1410 misses (0% hit rate)
+Scanned 1410 files (373K lines) in 2.5s
 
-# Second run (uses cache)
+# Cached runs
 $ dslop --all
-Scanned 1410 files in 305ms
-Cache: 1410 hits, 0 misses (100% hit rate)
+Scanned 1410 files (373K lines) in 170ms  # 50x faster
 ```
 
-Cache is stored in `.dslop-cache` in your project root. Add it to `.gitignore`.
-
-Use `--no-cache` to bypass the cache.
+Cache is stored in `.dslop-cache`. Add it to `.gitignore`. Use `--no-cache` to bypass.
 
 ## Limitations
 

package/dist/index.cjs

@@ -50675,7 +50675,7 @@ glob.glob = glob;
 // src/cache.ts
 var import_promises2 = require("node:fs/promises");
 var import_node_path3 = __toESM(require("node:path"), 1);
-var CACHE_VERSION = 2;
+var CACHE_VERSION = 3;
 var CACHE_FILE = ".dslop-cache";
 var ASTCache = class {
   cache = { version: CACHE_VERSION, files: {} };
@@ -50699,70 +50699,55 @@ var ASTCache = class {
       this.cache = { version: CACHE_VERSION, files: {} };
     }
   }
-  async get(filePath) {
+  // Check cache with pre-fetched stat
+  get(filePath, mtime, size) {
     const cached = this.cache.files[filePath];
     if (!cached) {
       this.misses++;
       return null;
     }
-    try {
-      const fileStat = await (0, import_promises2.stat)(filePath);
-      if (fileStat.mtimeMs !== cached.mtime || fileStat.size !== cached.size) {
-        this.misses++;
-        return null;
-      }
-      this.hits++;
-      return cached.blocks.map((b) => ({
+    if (mtime !== cached.mtime || size !== cached.size) {
+      this.misses++;
+      return null;
+    }
+    this.hits++;
+    return {
+      lineCount: cached.lineCount,
+      blocks: cached.blocks.map((b) => ({
         type: b.type,
         name: b.name,
         content: "",
-        // Don't cache content - we can re-read if needed
         normalized: b.normalized,
         hash: b.hash,
         filePath,
         startLine: b.startLine,
         endLine: b.endLine,
         exported: b.exported
-      }));
-    } catch {
-      this.misses++;
-      return null;
-    }
+      }))
+    };
   }
-  async set(filePath, blocks) {
-    try {
-      const fileStat = await (0, import_promises2.stat)(filePath);
-      this.cache.files[filePath] = {
-        mtime: fileStat.mtimeMs,
-        size: fileStat.size,
-        blocks: blocks.map((b) => ({
-          type: b.type,
-          name: b.name,
-          hash: b.hash,
-          normalized: b.normalized,
-          startLine: b.startLine,
-          endLine: b.endLine,
-          exported: b.exported
-        }))
-      };
-      this.dirty = true;
-    } catch {
-    }
+  set(filePath, blocks, lineCount, mtime, size) {
+    this.cache.files[filePath] = {
+      mtime,
+      size,
+      lineCount,
+      blocks: blocks.map((b) => ({
+        type: b.type,
+        name: b.name,
+        hash: b.hash,
+        normalized: b.normalized,
+        startLine: b.startLine,
+        endLine: b.endLine,
+        exported: b.exported
+      }))
+    };
+    this.dirty = true;
   }
   async save() {
     if (!this.dirty) return;
     try {
       const dir = import_node_path3.default.dirname(this.cachePath);
       await (0, import_promises2.mkdir)(dir, { recursive: true });
-      const prunedFiles = {};
-      for (const [filePath, cached] of Object.entries(this.cache.files)) {
-        try {
-          await (0, import_promises2.stat)(filePath);
-          prunedFiles[filePath] = cached;
-        } catch {
-        }
-      }
-      this.cache.files = prunedFiles;
       await (0, import_promises2.writeFile)(this.cachePath, JSON.stringify(this.cache), "utf-8");
     } catch {
     }
@@ -50779,6 +50764,7 @@ var ASTCache = class {
 
 // src/scanner.ts
 var MAX_FILE_SIZE = 1024 * 1024;
+var PARALLEL_BATCH_SIZE = 50;
 function shouldIgnore(filePath, ignorePatterns) {
   const normalizedPath = filePath.toLowerCase();
   return ignorePatterns.some((pattern) => {
@@ -50786,6 +50772,25 @@ function shouldIgnore(filePath, ignorePatterns) {
     return normalizedPath.includes(`/${normalizedPattern}/`) || normalizedPath.includes(`\\${normalizedPattern}\\`) || normalizedPath.endsWith(`/${normalizedPattern}`) || normalizedPath.endsWith(`\\${normalizedPattern}`);
   });
 }
+async function processFile(filePath, cache) {
+  try {
+    const fileStat = await (0, import_promises3.stat)(filePath);
+    if (fileStat.size > MAX_FILE_SIZE) {
+      return null;
+    }
+    const cached = cache?.get(filePath, fileStat.mtimeMs, fileStat.size);
+    if (cached) {
+      return cached;
+    }
+    const content = await (0, import_promises3.readFile)(filePath, "utf-8");
+    const lineCount = content.split("\n").length;
+    const blocks = extractASTBlocks(content, filePath);
+    cache?.set(filePath, blocks, lineCount, fileStat.mtimeMs, fileStat.size);
+    return { blocks, lineCount };
+  } catch {
+    return null;
+  }
+}
 async function scanDirectory(targetPath, options, useCache = true) {
   const { extensions, ignorePatterns } = options;
   const absolutePath = import_node_path4.default.resolve(targetPath);
@@ -50797,41 +50802,28 @@ async function scanDirectory(targetPath, options, useCache = true) {
     await cache.load();
   }
   const pattern = extensions.length === 1 ? `**/*.${extensions[0]}` : `**/*.{${extensions.join(",")}}`;
-  const files = await glob(pattern, {
+  const allFiles = await glob(pattern, {
     cwd: absolutePath,
     absolute: true,
     nodir: true,
     ignore: ignorePatterns.map((p) => `**/${p}/**`)
   });
-  for (const filePath of files) {
-    if (shouldIgnore(filePath, ignorePatterns)) {
-      continue;
-    }
-    if (!filePath.endsWith(".ts") && !filePath.endsWith(".tsx") && !filePath.endsWith(".js") && !filePath.endsWith(".jsx")) {
-      continue;
-    }
-    try {
-      const fileStat = await (0, import_promises3.stat)(filePath);
-      if (fileStat.size > MAX_FILE_SIZE) {
-        continue;
-      }
-      const cachedAST = cache ? await cache.get(filePath) : null;
-      if (cachedAST) {
-        astBlocks.push(...cachedAST);
-        const content = await (0, import_promises3.readFile)(filePath, "utf-8");
-        totalLines += content.split("\n").length;
-        fileCount++;
-      } else {
-        const content = await (0, import_promises3.readFile)(filePath, "utf-8");
-        totalLines += content.split("\n").length;
+  const files = allFiles.filter((filePath) => {
+    if (shouldIgnore(filePath, ignorePatterns)) return false;
+    const ext2 = import_node_path4.default.extname(filePath);
+    return ext2 === ".ts" || ext2 === ".tsx" || ext2 === ".js" || ext2 === ".jsx";
+  });
+  for (let i = 0; i < files.length; i += PARALLEL_BATCH_SIZE) {
+    const batch = files.slice(i, i + PARALLEL_BATCH_SIZE);
+    const results = await Promise.all(
+      batch.map((filePath) => processFile(filePath, cache))
+    );
+    for (const result of results) {
+      if (result) {
+        astBlocks.push(...result.blocks);
+        totalLines += result.lineCount;
         fileCount++;
-        const fileAST = extractASTBlocks(content, filePath);
-        astBlocks.push(...fileAST);
-        if (cache) {
-          await cache.set(filePath, fileAST);
-        }
       }
-    } catch {
     }
   }
   if (cache) {
@@ -50846,7 +50838,7 @@ async function scanDirectory(targetPath, options, useCache = true) {
 }
 
 // index.ts
-var VERSION = process.env.npm_package_version || "1.7.1";
+var VERSION = process.env.npm_package_version || "1.7.2";
 var DEFAULT_EXTENSIONS = ["ts", "tsx", "js", "jsx"];
 var DEFAULT_IGNORE_PATTERNS = [
   "node_modules",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dslop",
-  "version": "1.7.1",
+  "version": "1.7.2",
   "description": "Detect Similar/Duplicate Lines Of Programming - Find code duplication in your codebase",
   "type": "module",
   "main": "./dist/index.cjs",